You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

548 lines
17 KiB

8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
9 years ago
9 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
  1. import re
  2. from nd2reader.common import read_chunk, read_array, read_metadata, parse_date, get_from_dict_if_exists
  3. import xmltodict
  4. import six
  5. import numpy as np
  6. class RawMetadata(object):
  7. """RawMetadata class parses and stores the raw metadata that is read from the binary file in dict format.
  8. """
  9. def __init__(self, fh, label_map):
  10. self._fh = fh
  11. self._label_map = label_map
  12. self._metadata_parsed = None
  13. @property
  14. def __dict__(self):
  15. """Returns the parsed metadata in dictionary form.
  16. Returns:
  17. dict: the parsed metadata
  18. """
  19. return self.get_parsed_metadata()
  20. def get_parsed_metadata(self):
  21. """Returns the parsed metadata in dictionary form.
  22. Returns:
  23. dict: the parsed metadata
  24. """
  25. if self._metadata_parsed is not None:
  26. return self._metadata_parsed
  27. frames_per_channel = self._parse_total_images_per_channel()
  28. self._metadata_parsed = {
  29. "height": self._parse_if_not_none(self.image_attributes, self._parse_height),
  30. "width": self._parse_if_not_none(self.image_attributes, self._parse_width),
  31. "date": self._parse_if_not_none(self.image_text_info, self._parse_date),
  32. "fields_of_view": self._parse_fields_of_view(),
  33. "frames": self._parse_frames(),
  34. "z_levels": self._parse_z_levels(),
  35. "total_images_per_channel": frames_per_channel,
  36. "channels": self._parse_channels(),
  37. "pixel_microns": self._parse_if_not_none(self.image_calibration, self._parse_calibration),
  38. }
  39. self._set_default_if_not_empty('fields_of_view')
  40. self._set_default_if_not_empty('frames')
  41. self._metadata_parsed['num_frames'] = len(self._metadata_parsed['frames'])
  42. self._parse_roi_metadata()
  43. self._parse_experiment_metadata()
  44. return self._metadata_parsed
  45. def _set_default_if_not_empty(self, entry):
  46. if len(self._metadata_parsed[entry]) == 0 and self._metadata_parsed['total_images_per_channel'] > 0:
  47. # if the file is not empty, we always have one of this entry
  48. self._metadata_parsed[entry] = [0]
  49. @staticmethod
  50. def _parse_if_not_none(to_check, callback):
  51. if to_check is not None:
  52. return callback()
  53. return None
  54. def _parse_width_or_height(self, key):
  55. return self.image_attributes[six.b('SLxImageAttributes')][six.b(key)]
  56. def _parse_height(self):
  57. return self._parse_width_or_height('uiHeight')
  58. def _parse_width(self):
  59. return self._parse_width_or_height('uiWidth')
  60. def _parse_date(self):
  61. return parse_date(self.image_text_info[six.b('SLxImageTextInfo')])
  62. def _parse_calibration(self):
  63. return self.image_calibration.get(six.b('SLxCalibration'), {}).get(six.b('dCalibration'))
  64. def _parse_frames(self):
  65. """The number of cycles.
  66. Returns:
  67. list: list of all the frame numbers
  68. """
  69. return self._parse_dimension(r""".*?T'?\((\d+)\).*?""")
  70. def _parse_channels(self):
  71. """These are labels created by the NIS Elements user. Typically they may a short description of the filter cube
  72. used (e.g. 'bright field', 'GFP', etc.)
  73. Returns:
  74. list: the color channels
  75. """
  76. if self.image_metadata_sequence is None:
  77. return []
  78. metadata = self.image_metadata_sequence[six.b('SLxPictureMetadata')][six.b('sPicturePlanes')]
  79. channels = self._process_channels_metadata(metadata)
  80. return channels
  81. def _process_channels_metadata(self, metadata):
  82. try:
  83. validity = self.image_metadata[six.b('SLxExperiment')][six.b('ppNextLevelEx')][six.b('')][0][
  84. six.b('ppNextLevelEx')][six.b('')][0][six.b('pItemValid')]
  85. except (KeyError, TypeError):
  86. # If none of the channels have been deleted, there is no validity list, so we just make one
  87. validity = [True for _ in metadata]
  88. # Channel information is contained in dictionaries with the keys a0, a1...an where the number
  89. # indicates the order in which the channel is stored. So by sorting the dicts alphabetically
  90. # we get the correct order.
  91. channels = []
  92. for (label, chan), valid in zip(sorted(metadata[six.b('sPlaneNew')].items()), validity):
  93. if not valid:
  94. continue
  95. channels.append(chan[six.b('sDescription')].decode("utf8"))
  96. return channels
  97. def _parse_fields_of_view(self):
  98. """The metadata contains information about fields of view, but it contains it even if some fields
  99. of view were cropped. We can't find anything that states which fields of view are actually
  100. in the image data, so we have to calculate it. There probably is something somewhere, since
  101. NIS Elements can figure it out, but we haven't found it yet.
  102. """
  103. return self._parse_dimension(r""".*?XY\((\d+)\).*?""")
  104. def _parse_z_levels(self):
  105. """The different levels in the Z-plane.
  106. Returns:
  107. list: the z levels, just a sequence from 0 to n.
  108. """
  109. return self._parse_dimension(r""".*?Z\((\d+)\).*?""")
  110. def _parse_dimension_text(self):
  111. """While there are metadata values that represent a lot of what we want to capture, they seem to be unreliable.
  112. Sometimes certain elements don't exist, or change their data type randomly. However, the human-readable text
  113. is always there and in the same exact format, so we just parse that instead.
  114. """
  115. dimension_text = six.b("")
  116. if self.image_text_info is None:
  117. return dimension_text
  118. textinfo = self.image_text_info[six.b('SLxImageTextInfo')].values()
  119. for line in textinfo:
  120. entry = self._parse_dimension_text_line(line)
  121. if entry is not None:
  122. return entry
  123. return dimension_text
  124. @staticmethod
  125. def _parse_dimension_text_line(line):
  126. if six.b("Dimensions:") in line:
  127. entries = line.split(six.b("\r\n"))
  128. for entry in entries:
  129. if entry.startswith(six.b("Dimensions:")):
  130. return entry
  131. return None
  132. def _parse_dimension(self, pattern):
  133. dimension_text = self._parse_dimension_text()
  134. if dimension_text is None:
  135. return []
  136. if six.PY3:
  137. dimension_text = dimension_text.decode("utf8")
  138. match = re.match(pattern, dimension_text)
  139. if not match:
  140. return []
  141. count = int(match.group(1))
  142. return list(range(count))
  143. def _parse_total_images_per_channel(self):
  144. """The total number of images per channel.
  145. Warning: this may be inaccurate as it includes 'gap' images.
  146. """
  147. if self.image_attributes is None:
  148. return 0
  149. return self.image_attributes[six.b('SLxImageAttributes')][six.b('uiSequenceCount')]
  150. def _parse_roi_metadata(self):
  151. """Parse the raw ROI metadata.
  152. """
  153. if self.roi_metadata is None or not six.b('RoiMetadata_v1') in self.roi_metadata:
  154. return
  155. raw_roi_data = self.roi_metadata[six.b('RoiMetadata_v1')]
  156. if not six.b('m_vectGlobal_Size') in raw_roi_data:
  157. return
  158. number_of_rois = raw_roi_data[six.b('m_vectGlobal_Size')]
  159. roi_objects = []
  160. for i in range(number_of_rois):
  161. current_roi = raw_roi_data[six.b('m_vectGlobal_%d' % i)]
  162. roi_objects.append(self._parse_roi(current_roi))
  163. self._metadata_parsed['rois'] = roi_objects
  164. def _parse_roi(self, raw_roi_dict):
  165. """Extract the vector animation parameters from the ROI.
  166. This includes the position and size at the given timepoints.
  167. Args:
  168. raw_roi_dict: dictionary of raw roi metadata
  169. Returns:
  170. dict: the parsed ROI metadata
  171. """
  172. number_of_timepoints = raw_roi_dict[six.b('m_vectAnimParams_Size')]
  173. roi_dict = {
  174. "timepoints": [],
  175. "positions": [],
  176. "sizes": [],
  177. "shape": self._parse_roi_shape(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiShapeType')]),
  178. "type": self._parse_roi_type(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiInterpType')])
  179. }
  180. for i in range(number_of_timepoints):
  181. roi_dict = self._parse_vect_anim(roi_dict, raw_roi_dict[six.b('m_vectAnimParams_%d' % i)])
  182. # convert to NumPy arrays
  183. roi_dict["timepoints"] = np.array(roi_dict["timepoints"], dtype=np.float)
  184. roi_dict["positions"] = np.array(roi_dict["positions"], dtype=np.float)
  185. roi_dict["sizes"] = np.array(roi_dict["sizes"], dtype=np.float)
  186. return roi_dict
  187. @staticmethod
  188. def _parse_roi_shape(shape):
  189. if shape == 3:
  190. return 'rectangle'
  191. elif shape == 9:
  192. return 'circle'
  193. return None
  194. @staticmethod
  195. def _parse_roi_type(type_no):
  196. if type_no == 4:
  197. return 'stimulation'
  198. elif type_no == 3:
  199. return 'reference'
  200. elif type_no == 2:
  201. return 'background'
  202. return None
  203. def _parse_vect_anim(self, roi_dict, animation_dict):
  204. """
  205. Parses a ROI vector animation object and adds it to the global list of timepoints and positions.
  206. Args:
  207. roi_dict: the raw roi dictionary
  208. animation_dict: the raw animation dictionary
  209. Returns:
  210. dict: the parsed metadata
  211. """
  212. roi_dict["timepoints"].append(animation_dict[six.b('m_dTimeMs')])
  213. image_width = self._metadata_parsed["width"] * self._metadata_parsed["pixel_microns"]
  214. image_height = self._metadata_parsed["height"] * self._metadata_parsed["pixel_microns"]
  215. # positions are taken from the center of the image as a fraction of the half width/height of the image
  216. position = np.array((0.5 * image_width * (1 + animation_dict[six.b('m_dCenterX')]),
  217. 0.5 * image_height * (1 + animation_dict[six.b('m_dCenterY')]),
  218. animation_dict[six.b('m_dCenterZ')]))
  219. roi_dict["positions"].append(position)
  220. size_dict = animation_dict[six.b('m_sBoxShape')]
  221. # sizes are fractions of the half width/height of the image
  222. roi_dict["sizes"].append((size_dict[six.b('m_dSizeX')] * 0.25 * image_width,
  223. size_dict[six.b('m_dSizeY')] * 0.25 * image_height,
  224. size_dict[six.b('m_dSizeZ')]))
  225. return roi_dict
  226. def _parse_experiment_metadata(self):
  227. """Parse the metadata of the ND experiment
  228. """
  229. if self.image_metadata is None or six.b('SLxExperiment') not in self.image_metadata:
  230. return
  231. raw_data = self.image_metadata[six.b('SLxExperiment')]
  232. experimental_data = {
  233. 'description': 'unknown',
  234. 'loops': []
  235. }
  236. if six.b('wsApplicationDesc') in raw_data:
  237. experimental_data['description'] = raw_data[six.b('wsApplicationDesc')].decode('utf8')
  238. if six.b('uLoopPars') in raw_data:
  239. experimental_data['loops'] = self._parse_loop_data(raw_data[six.b('uLoopPars')])
  240. self._metadata_parsed['experiment'] = experimental_data
  241. @staticmethod
  242. def _get_loops_from_data(loop_data):
  243. loops = [loop_data]
  244. if six.b('uiPeriodCount') in loop_data and loop_data[six.b('uiPeriodCount')] > 0:
  245. # special ND experiment
  246. if six.b('pPeriod') not in loop_data:
  247. return []
  248. # take the first dictionary element, it contains all loop data
  249. loops = loop_data[six.b('pPeriod')][list(loop_data[six.b('pPeriod')].keys())[0]]
  250. return loops
  251. def _parse_loop_data(self, loop_data):
  252. """Parse the experimental loop data
  253. Args:
  254. loop_data: dictionary of experiment loops
  255. Returns:
  256. list: list of the parsed loops
  257. """
  258. loops = self._get_loops_from_data(loop_data)
  259. # take into account the absolute time in ms
  260. time_offset = 0
  261. parsed_loops = []
  262. for loop in loops:
  263. # duration of this loop
  264. duration = get_from_dict_if_exists('dDuration', loop) or 0
  265. # uiLoopType == 6 is a stimulation loop
  266. is_stimulation = get_from_dict_if_exists('uiLoopType', loop) == 6
  267. # sampling interval in ms
  268. interval = get_from_dict_if_exists('dAvgPeriodDiff', loop)
  269. parsed_loop = {
  270. 'start': time_offset,
  271. 'duration': duration,
  272. 'stimulation': is_stimulation,
  273. 'sampling_interval': interval
  274. }
  275. parsed_loops.append(parsed_loop)
  276. # increase the time offset
  277. time_offset += duration
  278. return parsed_loops
  279. @property
  280. def image_text_info(self):
  281. """Textual image information
  282. Returns:
  283. dict: containing the textual image info
  284. """
  285. return read_metadata(read_chunk(self._fh, self._label_map.image_text_info), 1)
  286. @property
  287. def image_metadata_sequence(self):
  288. """Image metadata of the sequence
  289. Returns:
  290. dict: containing the metadata
  291. """
  292. return read_metadata(read_chunk(self._fh, self._label_map.image_metadata_sequence), 1)
  293. @property
  294. def image_calibration(self):
  295. """The amount of pixels per micron.
  296. Returns:
  297. dict: pixels per micron
  298. """
  299. return read_metadata(read_chunk(self._fh, self._label_map.image_calibration), 1)
  300. @property
  301. def image_attributes(self):
  302. """Image attributes
  303. Returns:
  304. dict: containing the image attributes
  305. """
  306. return read_metadata(read_chunk(self._fh, self._label_map.image_attributes), 1)
  307. @property
  308. def x_data(self):
  309. """X data
  310. Returns:
  311. dict: x_data
  312. """
  313. return read_array(self._fh, 'double', self._label_map.x_data)
  314. @property
  315. def y_data(self):
  316. """Y data
  317. Returns:
  318. dict: y_data
  319. """
  320. return read_array(self._fh, 'double', self._label_map.y_data)
  321. @property
  322. def z_data(self):
  323. """Z data
  324. Returns:
  325. dict: z_data
  326. """
  327. return read_array(self._fh, 'double', self._label_map.z_data)
  328. @property
  329. def roi_metadata(self):
  330. """Contains information about the defined ROIs: shape, position and type (reference/background/stimulation).
  331. Returns:
  332. dict: ROI metadata dictionary
  333. """
  334. return read_metadata(read_chunk(self._fh, self._label_map.roi_metadata), 1)
  335. @property
  336. def pfs_status(self):
  337. """Perfect focus system (PFS) status
  338. Returns:
  339. dict: Perfect focus system (PFS) status
  340. """
  341. return read_array(self._fh, 'int', self._label_map.pfs_status)
  342. @property
  343. def pfs_offset(self):
  344. """Perfect focus system (PFS) offset
  345. Returns:
  346. dict: Perfect focus system (PFS) offset
  347. """
  348. return read_array(self._fh, 'int', self._label_map.pfs_offset)
  349. @property
  350. def camera_exposure_time(self):
  351. """Exposure time information
  352. Returns:
  353. dict: Camera exposure time
  354. """
  355. return read_array(self._fh, 'double', self._label_map.camera_exposure_time)
  356. @property
  357. def lut_data(self):
  358. """LUT information
  359. Returns:
  360. dict: LUT information
  361. """
  362. return xmltodict.parse(read_chunk(self._fh, self._label_map.lut_data))
  363. @property
  364. def grabber_settings(self):
  365. """Grabber settings
  366. Returns:
  367. dict: Acquisition settings
  368. """
  369. return xmltodict.parse(read_chunk(self._fh, self._label_map.grabber_settings))
  370. @property
  371. def custom_data(self):
  372. """Custom user data
  373. Returns:
  374. dict: custom user data
  375. """
  376. return xmltodict.parse(read_chunk(self._fh, self._label_map.custom_data))
  377. @property
  378. def app_info(self):
  379. """NIS elements application info
  380. Returns:
  381. dict: (Version) information of the NIS Elements application
  382. """
  383. return xmltodict.parse(read_chunk(self._fh, self._label_map.app_info))
  384. @property
  385. def camera_temp(self):
  386. """Camera temperature
  387. Yields:
  388. float: the temperature
  389. """
  390. camera_temp = read_array(self._fh, 'double', self._label_map.camera_temp)
  391. if camera_temp:
  392. for temp in map(lambda x: round(x * 100.0, 2), camera_temp):
  393. yield temp
  394. @property
  395. def acquisition_times(self):
  396. """Acquisition times
  397. Yields:
  398. float: the acquisition time
  399. """
  400. acquisition_times = read_array(self._fh, 'double', self._label_map.acquisition_times)
  401. if acquisition_times:
  402. for acquisition_time in map(lambda x: x / 1000.0, acquisition_times):
  403. yield acquisition_time
  404. @property
  405. def image_metadata(self):
  406. """Image metadata
  407. Returns:
  408. dict: Extra image metadata
  409. """
  410. if self._label_map.image_metadata:
  411. return read_metadata(read_chunk(self._fh, self._label_map.image_metadata), 1)