You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

302 lines
12 KiB

8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
7 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
7 years ago
8 years ago
8 years ago
8 years ago
9 years ago
6 years ago
6 years ago
9 years ago
8 years ago
  1. # -*- coding: utf-8 -*-
  2. import struct
  3. import array
  4. import six
  5. import warnings
  6. from pims.base_frames import Frame
  7. import numpy as np
  8. from nd2reader.common import get_version, read_chunk
  9. from nd2reader.exceptions import InvalidVersionError
  10. from nd2reader.label_map import LabelMap
  11. from nd2reader.raw_metadata import RawMetadata
  12. class Parser(object):
  13. """Parses ND2 files and creates a Metadata and driver object.
  14. """
  15. CHUNK_HEADER = 0xabeceda
  16. CHUNK_MAP_START = six.b("ND2 FILEMAP SIGNATURE NAME 0001!")
  17. CHUNK_MAP_END = six.b("ND2 CHUNK MAP SIGNATURE 0000001!")
  18. supported_file_versions = {(3, None): True}
  19. def __init__(self, fh):
  20. self._fh = fh
  21. self._label_map = None
  22. self._raw_metadata = None
  23. self.metadata = None
  24. # First check the file version
  25. self.supported = self._check_version_supported()
  26. # Parse the metadata
  27. self._parse_metadata()
  28. def calculate_image_properties(self, index):
  29. """Calculate FOV, channels and z_levels
  30. Args:
  31. index(int): the index (which is simply the order in which the image was acquired)
  32. Returns:
  33. tuple: tuple of the field of view, the channel and the z level
  34. """
  35. field_of_view = self._calculate_field_of_view(index)
  36. channel = self._calculate_channel(index)
  37. z_level = self._calculate_z_level(index)
  38. return field_of_view, channel, z_level
  39. def get_image(self, index):
  40. """
  41. Creates an Image object and adds its metadata, based on the index (which is simply the order in which the image
  42. was acquired). May return None if the ND2 contains multiple channels and not all were taken in each cycle (for
  43. example, if you take bright field images every minute, and GFP images every five minutes, there will be some
  44. indexes that do not contain an image. The reason for this is complicated, but suffice it to say that we hope to
  45. eliminate this possibility in future releases. For now, you'll need to check if your image is None if you're
  46. doing anything out of the ordinary.
  47. Args:
  48. index(int): the index (which is simply the order in which the image was acquired)
  49. Returns:
  50. Frame: the image
  51. """
  52. field_of_view, channel, z_level = self.calculate_image_properties(index)
  53. channel_offset = index % len(self.metadata["channels"])
  54. image_group_number = int(index / len(self.metadata["channels"]))
  55. frame_number = self._calculate_frame_number(image_group_number, field_of_view, z_level)
  56. try:
  57. timestamp, image = self._get_raw_image_data(image_group_number, channel_offset, self.metadata["height"],
  58. self.metadata["width"])
  59. except (TypeError):
  60. return Frame([], frame_no=frame_number, metadata=self._get_frame_metadata())
  61. else:
  62. return Frame(image, frame_no=frame_number, metadata=self._get_frame_metadata())
  63. def get_image_by_attributes(self, frame_number, field_of_view, channel, z_level, height, width):
  64. """Gets an image based on its attributes alone
  65. Args:
  66. frame_number: the frame number
  67. field_of_view: the field of view
  68. channel_name: the color channel name
  69. z_level: the z level
  70. height: the height of the image
  71. width: the width of the image
  72. Returns:
  73. Frame: the requested image
  74. """
  75. frame_number = 0 if frame_number is None else frame_number
  76. field_of_view = 0 if field_of_view is None else field_of_view
  77. channel = 0 if channel is None else channel
  78. z_level = 0 if z_level is None else z_level
  79. image_group_number = self._calculate_image_group_number(frame_number, field_of_view, z_level)
  80. try:
  81. timestamp, raw_image_data = self._get_raw_image_data(image_group_number, channel,
  82. height, width)
  83. except (TypeError):
  84. return Frame([], frame_no=frame_number, metadata=self._get_frame_metadata())
  85. else:
  86. return Frame(raw_image_data, frame_no=frame_number, metadata=self._get_frame_metadata())
  87. @staticmethod
  88. def get_dtype_from_metadata():
  89. """Determine the data type from the metadata.
  90. For now, always use float64 to prevent unexpected overflow errors when manipulating the data (calculating sums/
  91. means/etc.)
  92. """
  93. return np.float64
  94. def _check_version_supported(self):
  95. """Checks if the ND2 file version is supported by this reader.
  96. Returns:
  97. bool: True on supported
  98. """
  99. major_version, minor_version = get_version(self._fh)
  100. supported = self.supported_file_versions.get(
  101. (major_version, minor_version)) or self.supported_file_versions.get((major_version, None))
  102. if not supported:
  103. print("Warning: No parser is available for your current ND2 version (%d.%d). " % (
  104. major_version, minor_version) + "This might lead to unexpected behaviour.")
  105. return supported
  106. def _parse_metadata(self):
  107. """Reads all metadata and instantiates the Metadata object.
  108. """
  109. # Retrieve raw metadata from the label mapping
  110. self._label_map = self._build_label_map()
  111. self._raw_metadata = RawMetadata(self._fh, self._label_map)
  112. self.metadata = self._raw_metadata.__dict__
  113. self.acquisition_times = self._raw_metadata.acquisition_times
  114. def _build_label_map(self):
  115. """
  116. Every label ends with an exclamation point, however, we can't directly search for those to find all the labels
  117. as some of the bytes contain the value 33, which is the ASCII code for "!". So we iteratively find each label,
  118. grab the subsequent data (always 16 bytes long), advance to the next label and repeat.
  119. Returns:
  120. LabelMap: the computed label map
  121. """
  122. # go 8 bytes back from file end
  123. self._fh.seek(-8, 2)
  124. chunk_map_start_location = struct.unpack("Q", self._fh.read(8))[0]
  125. self._fh.seek(chunk_map_start_location)
  126. raw_text = self._fh.read(-1)
  127. return LabelMap(raw_text)
  128. def _calculate_field_of_view(self, index):
  129. """Determines what field of view was being imaged for a given image.
  130. Args:
  131. index(int): the index (which is simply the order in which the image was acquired)
  132. Returns:
  133. int: the field of view
  134. """
  135. images_per_cycle = len(self.metadata["z_levels"]) * len(self.metadata["channels"])
  136. return int((index - (index % images_per_cycle)) / images_per_cycle) % len(self.metadata["fields_of_view"])
  137. def _calculate_channel(self, index):
  138. """Determines what channel a particular image is.
  139. Args:
  140. index(int): the index (which is simply the order in which the image was acquired)
  141. Returns:
  142. string: the name of the color channel
  143. """
  144. return self.metadata["channels"][index % len(self.metadata["channels"])]
  145. def _calculate_z_level(self, index):
  146. """Determines the plane in the z-axis a given image was taken in.
  147. In the future, this will be replaced with the actual offset in micrometers.
  148. Args:
  149. index(int): the index (which is simply the order in which the image was acquired)
  150. Returns:
  151. The z level
  152. """
  153. return self.metadata["z_levels"][int(
  154. ((index - (index % len(self.metadata["channels"]))) / len(self.metadata["channels"])) % len(
  155. self.metadata["z_levels"]))]
  156. def _calculate_image_group_number(self, frame_number, fov, z_level):
  157. """
  158. Images are grouped together if they share the same time index, field of view, and z-level.
  159. Args:
  160. frame_number: the time index
  161. fov: the field of view number
  162. z_level: the z level number
  163. Returns:
  164. int: the image group number
  165. """
  166. z_length = len(self.metadata['z_levels'])
  167. z_length = z_length if z_length > 0 else 1
  168. fields_of_view = len(self.metadata["fields_of_view"])
  169. fields_of_view = fields_of_view if fields_of_view > 0 else 1
  170. return frame_number * fields_of_view * z_length + (fov * z_length + z_level)
  171. def _calculate_frame_number(self, image_group_number, field_of_view, z_level):
  172. """
  173. Images are in the same frame if they share the same group number and field of view and are taken sequentially.
  174. Args:
  175. image_group_number: the image group number (see _calculate_image_group_number)
  176. field_of_view: the field of view number
  177. z_level: the z level number
  178. Returns:
  179. """
  180. return (image_group_number - (field_of_view * len(self.metadata["z_levels"]) + z_level)) / (
  181. len(self.metadata["fields_of_view"]) * len(self.metadata["z_levels"]))
  182. @property
  183. def _channel_offset(self):
  184. """
  185. Image data is interleaved for each image set. That is, if there are four images in a set, the first image
  186. will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth.
  187. Returns:
  188. dict: the channel offset for each channel
  189. """
  190. return {channel: n for n, channel in enumerate(self.metadata["channels"])}
  191. def _get_raw_image_data(self, image_group_number, channel_offset, height, width):
  192. """Reads the raw bytes and the timestamp of an image.
  193. Args:
  194. image_group_number: the image group number (see _calculate_image_group_number)
  195. channel_offset: the number of the color channel
  196. height: the height of the image
  197. width: the width of the image
  198. Returns:
  199. """
  200. chunk = self._label_map.get_image_data_location(image_group_number)
  201. data = read_chunk(self._fh, chunk)
  202. # All images in the same image group share the same timestamp! So if you have complicated image data,
  203. # your timestamps may not be entirely accurate. Practically speaking though, they'll only be off by a few
  204. # seconds unless you're doing something super weird.
  205. timestamp = struct.unpack("d", data[:8])[0]
  206. image_group_data = array.array("H", data)
  207. image_data_start = 4 + channel_offset
  208. # The images for the various channels are interleaved within the same array. For example, the second image
  209. # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design
  210. # a data structure that way, please send the author of this library a message.
  211. number_of_true_channels = int(len(image_group_data[4:]) / (height * width))
  212. try:
  213. image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width))
  214. except ValueError:
  215. image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, int(round(len(image_group_data[image_data_start::number_of_true_channels])/height))))
  216. # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you
  217. # don't have the same number of images each cycle. We discovered this because we only took GFP images every
  218. # other cycle to reduce phototoxicity, but NIS Elements still allocated memory as if we were going to take
  219. # them every cycle.
  220. if np.any(image_data):
  221. return timestamp, image_data
  222. # If a blank "gap" image is encountered, generate an array of corresponding height and width to avoid
  223. # errors with ND2-files with missing frames. Array is filled with nan to reflect that data is missing.
  224. else:
  225. empty_frame = np.full((height, width), np.nan)
  226. warnings.warn('ND2 file contains gap frames which are represented by np.nan-filled arrays; to convert to zeros use e.g. np.nan_to_num(array)')
  227. return timestamp, image_data
  228. def _get_frame_metadata(self):
  229. """Get the metadata for one frame
  230. Returns:
  231. dict: a dictionary containing the parsed metadata
  232. """
  233. return self.metadata