You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

261 lines
10 KiB

8 years ago
8 years ago
8 years ago
9 years ago
9 years ago
9 years ago
8 years ago
  1. # -*- coding: utf-8 -*-
  2. import struct
  3. import array
  4. import six
  5. from pims import Frame
  6. import numpy as np
  7. from nd2reader.common import get_version, read_chunk
  8. from nd2reader.exceptions import InvalidVersionError, NoImageError
  9. from nd2reader.label_map import LabelMap
  10. from nd2reader.raw_metadata import RawMetadata
  11. class Parser(object):
  12. """ Parses ND2 files and creates a Metadata and driver object. """
  13. CHUNK_HEADER = 0xabeceda
  14. CHUNK_MAP_START = six.b("ND2 FILEMAP SIGNATURE NAME 0001!")
  15. CHUNK_MAP_END = six.b("ND2 CHUNK MAP SIGNATURE 0000001!")
  16. supported_file_versions = {(3, None): True}
  17. def __init__(self, fh):
  18. """
  19. :type fh: file
  20. """
  21. self._fh = fh
  22. self._label_map = None
  23. self._raw_metadata = None
  24. self.metadata = None
  25. # First check the file version
  26. self._check_version_supported()
  27. # Parse the metadata
  28. self._parse_metadata()
  29. def calculate_image_properties(self, index):
  30. """
  31. Calculate FOV, channels and z_levels
  32. :param index:
  33. :return:
  34. """
  35. field_of_view = self._calculate_field_of_view(index)
  36. channel = self._calculate_channel(index)
  37. z_level = self._calculate_z_level(index)
  38. return field_of_view, channel, z_level
  39. def get_image(self, index):
  40. """
  41. Creates an Image object and adds its metadata, based on the index (which is simply the order in which the image
  42. was acquired). May return None if the ND2 contains multiple channels and not all were taken in each cycle (for
  43. example, if you take bright field images every minute, and GFP images every five minutes, there will be some
  44. indexes that do not contain an image. The reason for this is complicated, but suffice it to say that we hope to
  45. eliminate this possibility in future releases. For now, you'll need to check if your image is None if you're
  46. doing anything out of the ordinary.
  47. :type index: int
  48. :rtype: Image or None
  49. """
  50. field_of_view, channel, z_level = self.calculate_image_properties(index)
  51. channel_offset = index % len(self.metadata["channels"])
  52. image_group_number = int(index / len(self.metadata["channels"]))
  53. frame_number = self._calculate_frame_number(image_group_number, field_of_view, z_level)
  54. try:
  55. timestamp, image = self._get_raw_image_data(image_group_number, channel_offset, self.metadata["height"],
  56. self.metadata["width"])
  57. except (TypeError, NoImageError):
  58. return Frame([], frame_no=frame_number, metadata=self._get_frame_metadata())
  59. else:
  60. return image
  61. def get_image_by_attributes(self, frame_number, field_of_view, channel_name, z_level, height, width):
  62. """
  63. Attempts to get Image based on attributes alone.
  64. :type frame_number: int
  65. :type field_of_view: int
  66. :type channel_name: str
  67. :type z_level: int
  68. :type height: int
  69. :type width: int
  70. :rtype: Image or None
  71. """
  72. image_group_number = self._calculate_image_group_number(frame_number, field_of_view, z_level)
  73. try:
  74. timestamp, raw_image_data = self._get_raw_image_data(image_group_number, self._channel_offset[channel_name],
  75. height, width)
  76. except (TypeError, NoImageError):
  77. return Frame([], frame_no=frame_number, metadata=self._get_frame_metadata())
  78. else:
  79. return raw_image_data
  80. @staticmethod
  81. def get_dtype_from_metadata():
  82. """
  83. Determine the data type from the metadata.
  84. For now, always use float64 to prevent unexpected overflow errors when manipulating the data (calculating sums/
  85. means/etc.)
  86. :return:
  87. """
  88. return np.float64
  89. def _check_version_supported(self):
  90. """
  91. Checks if the ND2 file version is supported by this reader.
  92. :return:
  93. """
  94. major_version, minor_version = get_version(self._fh)
  95. supported = self.supported_file_versions.get((major_version, minor_version)) or \
  96. self.supported_file_versions.get((major_version, None))
  97. if not supported:
  98. raise InvalidVersionError("No parser is available for that version.")
  99. return supported
  100. def _parse_metadata(self):
  101. """
  102. Reads all metadata and instantiates the Metadata object.
  103. """
  104. # Retrieve raw metadata from the label mapping
  105. self._label_map = self._build_label_map()
  106. self._raw_metadata = RawMetadata(self._fh, self._label_map)
  107. self.metadata = self._raw_metadata.__dict__
  108. def _build_label_map(self):
  109. """
  110. Every label ends with an exclamation point, however, we can't directly search for those to find all the labels
  111. as some of the bytes contain the value 33, which is the ASCII code for "!". So we iteratively find each label,
  112. grab the subsequent data (always 16 bytes long), advance to the next label and repeat.
  113. :rtype: LabelMap
  114. """
  115. self._fh.seek(-8, 2)
  116. chunk_map_start_location = struct.unpack("Q", self._fh.read(8))[0]
  117. self._fh.seek(chunk_map_start_location)
  118. raw_text = self._fh.read(-1)
  119. return LabelMap(raw_text)
  120. def _calculate_field_of_view(self, index):
  121. """
  122. Determines what field of view was being imaged for a given image.
  123. :type index: int
  124. :rtype: int
  125. """
  126. images_per_cycle = len(self.metadata["z_levels"]) * len(self.metadata["channels"])
  127. return int((index - (index % images_per_cycle)) / images_per_cycle) % len(self.metadata["fields_of_view"])
  128. def _calculate_channel(self, index):
  129. """
  130. Determines what channel a particular image is.
  131. :type index: int
  132. :rtype: str
  133. """
  134. return self.metadata["channels"][index % len(self.metadata["channels"])]
  135. def _calculate_z_level(self, index):
  136. """
  137. Determines the plane in the z-axis a given image was taken in. In the future, this will be replaced with the
  138. actual offset in micrometers.
  139. :type index: int
  140. :rtype: int
  141. """
  142. return self.metadata["z_levels"][int(
  143. ((index - (index % len(self.metadata["channels"]))) / len(self.metadata["channels"])) % len(
  144. self.metadata["z_levels"]))]
  145. def _calculate_image_group_number(self, frame_number, fov, z_level):
  146. """
  147. Images are grouped together if they share the same time index, field of view, and z-level.
  148. :type frame_number: int
  149. :type fov: int
  150. :type z_level: int
  151. :rtype: int
  152. """
  153. return frame_number * len(self.metadata["fields_of_view"]) * len(self.metadata["z_levels"]) + (
  154. fov * len(self.metadata["z_levels"]) + z_level)
  155. def _calculate_frame_number(self, image_group_number, field_of_view, z_level):
  156. """
  157. Images are in the same frame if they share the same group number and field of view and are taken sequentially.
  158. :type image_group_number: int
  159. :type field_of_view: int
  160. :type z_level: int
  161. :rtype: int
  162. """
  163. return (image_group_number - (field_of_view * len(self.metadata["z_levels"]) + z_level)) / (
  164. len(self.metadata["fields_of_view"]) * len(self.metadata["z_levels"]))
  165. @property
  166. def _channel_offset(self):
  167. """
  168. Image data is interleaved for each image set. That is, if there are four images in a set, the first image
  169. will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth.
  170. :rtype: dict
  171. """
  172. return {channel: n for n, channel in enumerate(self.metadata["channels"])}
  173. def _get_raw_image_data(self, image_group_number, channel_offset, height, width):
  174. """
  175. Reads the raw bytes and the timestamp of an image.
  176. :param image_group_number: groups are made of images with the same time index, field of view and z-level
  177. :type image_group_number: int
  178. :param channel_offset: the offset in the array where the bytes for this image are found
  179. :type channel_offset: int
  180. :rtype: (int, Image)
  181. :raises: NoImageError
  182. """
  183. chunk = self._label_map.get_image_data_location(image_group_number)
  184. data = read_chunk(self._fh, chunk)
  185. # print("data", data, "that was data")
  186. # All images in the same image group share the same timestamp! So if you have complicated image data,
  187. # your timestamps may not be entirely accurate. Practically speaking though, they'll only be off by a few
  188. # seconds unless you're doing something super weird.
  189. timestamp = struct.unpack("d", data[:8])[0]
  190. image_group_data = array.array("H", data)
  191. image_data_start = 4 + channel_offset
  192. # The images for the various channels are interleaved within the same array. For example, the second image
  193. # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design
  194. # a data structure that way, please send the author of this library a message.
  195. number_of_true_channels = int((len(image_group_data) - 4) / (height * width))
  196. image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width))
  197. # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you
  198. # don't have the same number of images each cycle. We discovered this because we only took GFP images every
  199. # other cycle to reduce phototoxicity, but NIS Elements still allocated memory as if we were going to take
  200. # them every cycle.
  201. if np.any(image_data):
  202. return timestamp, Frame(image_data, metadata=self._get_frame_metadata())
  203. raise NoImageError
  204. def _get_frame_metadata(self):
  205. """
  206. Get the metadata for one frame
  207. :return:
  208. """
  209. return self.metadata