You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

353 lines
13 KiB

9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
  1. # -*- coding: utf-8 -*-
  2. from datetime import datetime
  3. from nd2reader.model.metadata import Metadata, CameraSettings
  4. from nd2reader.model.label import LabelMap
  5. from nd2reader.parser.base import BaseParser
  6. from nd2reader.driver.v3 import V3Driver
  7. from nd2reader.common.v3 import read_chunk, read_array, read_metadata
  8. import re
  9. import six
  10. import struct
  11. import xmltodict
  12. def ignore_missing(func):
  13. def wrapper(*args, **kwargs):
  14. try:
  15. return func(*args, **kwargs)
  16. except:
  17. return None
  18. return wrapper
  19. class V3RawMetadata(object):
  20. def __init__(self, fh, label_map):
  21. self._fh = fh
  22. self._label_map = label_map
  23. @property
  24. @ignore_missing
  25. def image_text_info(self):
  26. return read_metadata(read_chunk(self._fh, self._label_map.image_text_info), 1)
  27. @property
  28. @ignore_missing
  29. def image_metadata_sequence(self):
  30. return read_metadata(read_chunk(self._fh, self._label_map.image_metadata_sequence), 1)
  31. @property
  32. @ignore_missing
  33. def image_calibration(self):
  34. return read_metadata(read_chunk(self._fh, self._label_map.image_calibration), 1)
  35. @property
  36. @ignore_missing
  37. def image_attributes(self):
  38. return read_metadata(read_chunk(self._fh, self._label_map.image_attributes), 1)
  39. @property
  40. @ignore_missing
  41. def x_data(self):
  42. return read_array(self._fh, 'double', self._label_map.x_data)
  43. @property
  44. @ignore_missing
  45. def y_data(self):
  46. return read_array(self._fh, 'double', self._label_map.y_data)
  47. @property
  48. @ignore_missing
  49. def z_data(self):
  50. return read_array(self._fh, 'double', self._label_map.z_data)
  51. @property
  52. @ignore_missing
  53. def roi_metadata(self):
  54. return read_metadata(read_chunk(self._fh, self._label_map.roi_metadata), 1)
  55. @property
  56. @ignore_missing
  57. def pfs_status(self):
  58. return read_array(self._fh, 'int', self._label_map.pfs_status)
  59. @property
  60. @ignore_missing
  61. def pfs_offset(self):
  62. return read_array(self._fh, 'int', self._label_map.pfs_offset)
  63. @property
  64. @ignore_missing
  65. def camera_exposure_time(self):
  66. return read_array(self._fh, 'double', self._label_map.camera_exposure_time)
  67. @property
  68. @ignore_missing
  69. def lut_data(self):
  70. return xmltodict.parse(read_chunk(self._fh, self._label_map.lut_data))
  71. @property
  72. @ignore_missing
  73. def grabber_settings(self):
  74. return xmltodict.parse(read_chunk(self._fh, self._label_map.grabber_settings))
  75. @property
  76. @ignore_missing
  77. def custom_data(self):
  78. return xmltodict.parse(read_chunk(self._fh, self._label_map.custom_data))
  79. @property
  80. @ignore_missing
  81. def app_info(self):
  82. return xmltodict.parse(read_chunk(self._fh, self._label_map.app_info))
  83. @property
  84. @ignore_missing
  85. def camera_temp(self):
  86. camera_temp = read_array(self._fh, 'double', self._label_map.camera_temp)
  87. if camera_temp:
  88. for temp in map(lambda x: round(x * 100.0, 2), camera_temp):
  89. yield temp
  90. @property
  91. @ignore_missing
  92. def acquisition_times(self):
  93. acquisition_times = read_array(self._fh, 'double', self._label_map.acquisition_times)
  94. if acquisition_times:
  95. for acquisition_time in map(lambda x: x / 1000.0, acquisition_times):
  96. yield acquisition_time
  97. @property
  98. @ignore_missing
  99. def image_metadata(self):
  100. if self._label_map.image_metadata:
  101. return read_metadata(read_chunk(self._fh, self._label_map.image_metadata), 1)
  102. class V3Parser(BaseParser):
  103. """ Parses ND2 files and creates a Metadata and driver object. """
  104. CHUNK_HEADER = 0xabeceda
  105. CHUNK_MAP_START = six.b("ND2 FILEMAP SIGNATURE NAME 0001!")
  106. CHUNK_MAP_END = six.b("ND2 CHUNK MAP SIGNATURE 0000001!")
  107. def __init__(self, fh):
  108. """
  109. :type fh: file
  110. """
  111. if six.PY3:
  112. super().__init__(fh)
  113. else:
  114. super(V3Parser, self).__init__(fh)
  115. self._label_map = self._build_label_map()
  116. self.raw_metadata = V3RawMetadata(self._fh, self._label_map)
  117. self._parse_camera_metadata()
  118. self._parse_metadata()
  119. @property
  120. def driver(self):
  121. """
  122. Provides an object that knows how to look up and read images based on an index.
  123. """
  124. return V3Driver(self.metadata, self._label_map, self._fh)
  125. def _parse_camera_metadata(self):
  126. """
  127. Gets parsed data about the physical cameras used to produce images and throws them in a dictionary.
  128. """
  129. self.camera_metadata = {}
  130. for camera_setting in self._parse_camera_settings():
  131. self.camera_metadata[camera_setting.channel_name] = camera_setting
  132. def _parse_metadata(self):
  133. """
  134. Reads all metadata and instantiates the Metadata object.
  135. """
  136. height = self.raw_metadata.image_attributes[six.b('SLxImageAttributes')][six.b('uiHeight')]
  137. width = self.raw_metadata.image_attributes[six.b('SLxImageAttributes')][six.b('uiWidth')]
  138. date = self._parse_date(self.raw_metadata)
  139. fields_of_view = self._parse_fields_of_view(self.raw_metadata)
  140. frames = self._parse_frames(self.raw_metadata)
  141. z_levels = self._parse_z_levels(self.raw_metadata)
  142. total_images_per_channel = self._parse_total_images_per_channel(self.raw_metadata)
  143. channels = self._parse_channels(self.raw_metadata)
  144. pixel_microns = self.raw_metadata.image_calibration.get(six.b('SLxCalibration'), {}).get(six.b('dCalibration'))
  145. self.metadata = Metadata(height, width, channels, date, fields_of_view, frames, z_levels, total_images_per_channel, pixel_microns)
  146. def _parse_camera_settings(self):
  147. """
  148. Looks up information in the raw metadata about the camera(s) and puts it into a CameraSettings object.
  149. Duplicate cameras can be returned if the same one was used for multiple channels.
  150. """
  151. for n, camera in enumerate(self.raw_metadata.image_metadata_sequence[six.b('SLxPictureMetadata')][six.b('sPicturePlanes')][six.b('sSampleSetting')].values()):
  152. name = camera[six.b('pCameraSetting')][six.b('CameraUserName')]
  153. id = camera[six.b('pCameraSetting')][six.b('CameraUniqueName')]
  154. exposure = camera[six.b('dExposureTime')]
  155. x_binning = camera[six.b('pCameraSetting')][six.b('FormatFast')][six.b('fmtDesc')][six.b('dBinningX')]
  156. y_binning = camera[six.b('pCameraSetting')][six.b('FormatFast')][six.b('fmtDesc')][six.b('dBinningY')]
  157. optical_configs = camera[six.b('sOpticalConfigs')]
  158. # This definitely is not working right. It seems to be totally inconsistent in each of the sample ND2s that I have.
  159. # Fixing one breaks another.
  160. if six.b('') in optical_configs.keys():
  161. channel_name = optical_configs[six.b('')][six.b('sOpticalConfigName')]
  162. yield CameraSettings(name, id, exposure, x_binning, y_binning, channel_name)
  163. else:
  164. channel_names = [channel[six.b('Name')] for key, channel in camera[six.b('pCameraSetting')][six.b('Metadata')][six.b('Channels')].items()]
  165. for channel_name in channel_names:
  166. yield CameraSettings(name, id, exposure, x_binning, y_binning, channel_name)
  167. def _parse_date(self, raw_metadata):
  168. """
  169. The date and time when acquisition began.
  170. :type raw_metadata: V3RawMetadata
  171. :rtype: datetime.datetime() or None
  172. """
  173. for line in raw_metadata.image_text_info[six.b('SLxImageTextInfo')].values():
  174. line = line.decode("utf8")
  175. absolute_start_12 = None
  176. absolute_start_24 = None
  177. # ND2s seem to randomly switch between 12- and 24-hour representations.
  178. try:
  179. absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S")
  180. except (TypeError, ValueError):
  181. pass
  182. try:
  183. absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p")
  184. except (TypeError, ValueError):
  185. pass
  186. if not absolute_start_12 and not absolute_start_24:
  187. continue
  188. return absolute_start_12 if absolute_start_12 else absolute_start_24
  189. return None
  190. def _parse_channels(self, raw_metadata):
  191. """
  192. These are labels created by the NIS Elements user. Typically they may a short description of the filter cube
  193. used (e.g. "bright field", "GFP", etc.)
  194. :type raw_metadata: V3RawMetadata
  195. :rtype: list
  196. """
  197. channels = []
  198. metadata = raw_metadata.image_metadata_sequence[six.b('SLxPictureMetadata')][six.b('sPicturePlanes')]
  199. try:
  200. validity = raw_metadata.image_metadata[six.b('SLxExperiment')][six.b('ppNextLevelEx')][six.b('')][0][six.b('ppNextLevelEx')][six.b('')][0][six.b('pItemValid')]
  201. except (KeyError, TypeError):
  202. # If none of the channels have been deleted, there is no validity list, so we just make one
  203. validity = [True for _ in metadata]
  204. # Channel information is contained in dictionaries with the keys a0, a1...an where the number
  205. # indicates the order in which the channel is stored. So by sorting the dicts alphabetically
  206. # we get the correct order.
  207. for (label, chan), valid in zip(sorted(metadata[six.b('sPlaneNew')].items()), validity):
  208. if not valid:
  209. continue
  210. channels.append(chan[six.b('sDescription')].decode("utf8"))
  211. return channels
  212. def _parse_fields_of_view(self, raw_metadata):
  213. """
  214. The metadata contains information about fields of view, but it contains it even if some fields
  215. of view were cropped. We can't find anything that states which fields of view are actually
  216. in the image data, so we have to calculate it. There probably is something somewhere, since
  217. NIS Elements can figure it out, but we haven't found it yet.
  218. :type raw_metadata: V3RawMetadata
  219. :rtype: list
  220. """
  221. return self._parse_dimension(r""".*?XY\((\d+)\).*?""", raw_metadata)
  222. def _parse_frames(self, raw_metadata):
  223. """
  224. The number of cycles.
  225. :type raw_metadata: V3RawMetadata
  226. :rtype: list
  227. """
  228. return self._parse_dimension(r""".*?T'?\((\d+)\).*?""", raw_metadata)
  229. def _parse_z_levels(self, raw_metadata):
  230. """
  231. The different levels in the Z-plane. Just a sequence from 0 to n.
  232. :type raw_metadata: V3RawMetadata
  233. :rtype: list
  234. """
  235. return self._parse_dimension(r""".*?Z\((\d+)\).*?""", raw_metadata)
  236. def _parse_dimension_text(self, raw_metadata):
  237. """
  238. While there are metadata values that represent a lot of what we want to capture, they seem to be unreliable.
  239. Sometimes certain elements don't exist, or change their data type randomly. However, the human-readable text
  240. is always there and in the same exact format, so we just parse that instead.
  241. :type raw_metadata: V3RawMetadata
  242. :rtype: str
  243. """
  244. for line in raw_metadata.image_text_info[six.b('SLxImageTextInfo')].values():
  245. if six.b("Dimensions:") in line:
  246. metadata = line
  247. break
  248. else:
  249. return six.b("")
  250. for line in metadata.split(six.b("\r\n")):
  251. if line.startswith(six.b("Dimensions:")):
  252. dimension_text = line
  253. break
  254. else:
  255. return six.b("")
  256. return dimension_text
  257. def _parse_dimension(self, pattern, raw_metadata):
  258. """
  259. :param pattern: a valid regex pattern
  260. :type pattern: str
  261. :type raw_metadata: V3RawMetadata
  262. :rtype: list of int
  263. """
  264. dimension_text = self._parse_dimension_text(raw_metadata)
  265. if six.PY3:
  266. dimension_text = dimension_text.decode("utf8")
  267. match = re.match(pattern, dimension_text)
  268. if not match:
  269. return [0]
  270. count = int(match.group(1))
  271. return list(range(count))
  272. def _parse_total_images_per_channel(self, raw_metadata):
  273. """
  274. The total number of images per channel. Warning: this may be inaccurate as it includes "gap" images.
  275. :type raw_metadata: V3RawMetadata
  276. :rtype: int
  277. """
  278. return raw_metadata.image_attributes[six.b('SLxImageAttributes')][six.b('uiSequenceCount')]
  279. def _build_label_map(self):
  280. """
  281. Every label ends with an exclamation point, however, we can't directly search for those to find all the labels
  282. as some of the bytes contain the value 33, which is the ASCII code for "!". So we iteratively find each label,
  283. grab the subsequent data (always 16 bytes long), advance to the next label and repeat.
  284. :rtype: LabelMap
  285. """
  286. self._fh.seek(-8, 2)
  287. chunk_map_start_location = struct.unpack("Q", self._fh.read(8))[0]
  288. self._fh.seek(chunk_map_start_location)
  289. raw_text = self._fh.read(-1)
  290. return LabelMap(raw_text)