# -*- coding: utf-8 -*- import array from datetime import datetime from nd2reader.model.metadata import Metadata from nd2reader.model.label import LabelMap from nd2reader.parser.base import BaseParser from nd2reader.driver.v3 import V3Driver from nd2reader.common.v3 import read_chunk import re import six import struct class V3Parser(BaseParser): """ Parses ND2 files and creates a Metadata and driver object. """ CHUNK_HEADER = 0xabeceda CHUNK_MAP_START = six.b("ND2 FILEMAP SIGNATURE NAME 0001!") CHUNK_MAP_END = six.b("ND2 CHUNK MAP SIGNATURE 0000001!") def __init__(self, fh): """ :type fh: file """ self._fh = fh self._metadata = None self._label_map = None @property def metadata(self): """ :rtype: Metadata """ if not self._metadata: self._parse_metadata() return self._metadata @property def driver(self): return V3Driver(self.metadata, self._label_map, self._fh) def _build_metadata_dict(self): self._label_map = self._build_label_map() raw_data = {"image_text_info": read_chunk(self._fh, self._label_map.image_text_info), "image_metadata_sequence": read_chunk(self._fh, self._label_map.image_metadata_sequence), # "image_data": read_chunk(self._fh, self._label_map.image_data), "image_calibration": read_chunk(self._fh, self._label_map.image_calibration), "image_attributes": read_chunk(self._fh, self._label_map.image_attributes), # "x_data": read_chunk(self._fh, self._label_map.x_data), # "y_data": read_chunk(self._fh, self._label_map.y_data), # "z_data": read_chunk(self._fh, self._label_map.z_data), # "roi_metadata": read_chunk(self._fh, self._label_map.roi_metadata), # "pfs_status": read_chunk(self._fh, self._label_map.pfs_status), # "pfs_offset": read_chunk(self._fh, self._label_map.pfs_offset), # "guid": read_chunk(self._fh, self._label_map.guid), # "description": read_chunk(self._fh, self._label_map.description), # "camera_exposure_time": read_chunk(self._fh, self._label_map.camera_exposure_time), # "camera_temp": read_chunk(self._fh, self._label_map.camera_temp), # "acquisition_times": read_chunk(self._fh, self._label_map.acquisition_times), # "acquisition_times_2": read_chunk(self._fh, self._label_map.acquisition_times_2), # "acquisition_frames": read_chunk(self._fh, self._label_map.acquisition_frames), # "lut_data": read_chunk(self._fh, self._label_map.lut_data), # "grabber_settings": read_chunk(self._fh, self._label_map.grabber_settings), # "custom_data": read_chunk(self._fh, self._label_map.custom_data), # "app_info": read_chunk(self._fh, self._label_map.app_info) } if self._label_map.image_metadata: raw_data["image_metadata"] = read_chunk(self._fh, self._label_map.image_metadata) return {key: self._read_metadata(data, 1) for key, data in raw_data.items()} def _parse_metadata(self): """ Reads all metadata and instantiates the Metadata object. """ metadata_dict = self._build_metadata_dict() height = metadata_dict['image_attributes'][six.b('SLxImageAttributes')][six.b('uiHeight')] width = metadata_dict['image_attributes'][six.b('SLxImageAttributes')][six.b('uiWidth')] channels = self._parse_channels(metadata_dict) date = self._parse_date(metadata_dict) fields_of_view = self._parse_fields_of_view(metadata_dict) frames = self._parse_frames(metadata_dict) z_levels = self._parse_z_levels(metadata_dict) total_images_per_channel = self._parse_total_images_per_channel(metadata_dict) self._metadata = Metadata(height, width, channels, date, fields_of_view, frames, z_levels, total_images_per_channel) def _parse_date(self, metadata_dict): """ The date and time when acquisition began. :type metadata_dict: dict :rtype: datetime.datetime() or None """ for line in metadata_dict['image_text_info'][six.b('SLxImageTextInfo')].values(): line = line.decode("utf8") absolute_start_12 = None absolute_start_24 = None # ND2s seem to randomly switch between 12- and 24-hour representations. try: absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") except (TypeError, ValueError): pass try: absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") except (TypeError, ValueError): pass if not absolute_start_12 and not absolute_start_24: continue return absolute_start_12 if absolute_start_12 else absolute_start_24 return None def _parse_channels(self, metadata_dict): """ These are labels created by the NIS Elements user. Typically they may a short description of the filter cube used (e.g. "bright field", "GFP", etc.) :type metadata_dict: dict :rtype: list """ channels = [] metadata = metadata_dict['image_metadata_sequence'][six.b('SLxPictureMetadata')][six.b('sPicturePlanes')] try: validity = metadata_dict['image_metadata'][six.b('SLxExperiment')][six.b('ppNextLevelEx')][six.b('')][0][six.b('ppNextLevelEx')][six.b('')][0][six.b('pItemValid')] except KeyError: # If none of the channels have been deleted, there is no validity list, so we just make one validity = [True for _ in metadata] # Channel information is contained in dictionaries with the keys a0, a1...an where the number # indicates the order in which the channel is stored. So by sorting the dicts alphabetically # we get the correct order. for (label, chan), valid in zip(sorted(metadata[six.b('sPlaneNew')].items()), validity): if not valid: continue channels.append(chan[six.b('sDescription')].decode("utf8")) return channels def _parse_fields_of_view(self, metadata_dict): """ The metadata contains information about fields of view, but it contains it even if some fields of view were cropped. We can't find anything that states which fields of view are actually in the image data, so we have to calculate it. There probably is something somewhere, since NIS Elements can figure it out, but we haven't found it yet. :type metadata_dict: dict :rtype: list """ return self._parse_dimension(r""".*?XY\((\d+)\).*?""", metadata_dict) def _parse_frames(self, metadata_dict): """ The number of cycles. :type metadata_dict: dict :rtype: list """ return self._parse_dimension(r""".*?T'?\((\d+)\).*?""", metadata_dict) def _parse_z_levels(self, metadata_dict): """ The different levels in the Z-plane. Just a sequence from 0 to n. :type metadata_dict: dict :rtype: list """ return self._parse_dimension(r""".*?Z\((\d+)\).*?""", metadata_dict) def _parse_dimension_text(self, metadata_dict): """ While there are metadata values that represent a lot of what we want to capture, they seem to be unreliable. Sometimes certain elements don't exist, or change their data type randomly. However, the human-readable text is always there and in the same exact format, so we just parse that instead. :type metadata_dict: dict :rtype: str """ for line in metadata_dict['image_text_info'][six.b('SLxImageTextInfo')].values(): if six.b("Dimensions:") in line: metadata = line break else: return six.b("") for line in metadata.split(six.b("\r\n")): if line.startswith(six.b("Dimensions:")): dimension_text = line break else: return six.b("") return dimension_text def _parse_dimension(self, pattern, metadata_dict): """ :param pattern: a valid regex pattern :type pattern: str :type metadata_dict: dict :rtype: list of int """ dimension_text = self._parse_dimension_text(metadata_dict) if six.PY3: dimension_text = dimension_text.decode("utf8") match = re.match(pattern, dimension_text) if not match: return [0] count = int(match.group(1)) return list(range(count)) def _parse_total_images_per_channel(self, metadata_dict): """ The total number of images per channel. Warning: this may be inaccurate as it includes "gap" images. :type metadata_dict: dict :rtype: int """ return metadata_dict['image_attributes'][six.b('SLxImageAttributes')][six.b('uiSequenceCount')] def _build_label_map(self): """ Every label ends with an exclamation point, however, we can't directly search for those to find all the labels as some of the bytes contain the value 33, which is the ASCII code for "!". So we iteratively find each label, grab the subsequent data (always 16 bytes long), advance to the next label and repeat. :rtype: LabelMap """ self._fh.seek(-8, 2) chunk_map_start_location = struct.unpack("Q", self._fh.read(8))[0] self._fh.seek(chunk_map_start_location) raw_text = self._fh.read(-1) return LabelMap(raw_text) def _parse_unsigned_char(self, data): return struct.unpack("B", data.read(1))[0] def _parse_unsigned_int(self, data): return struct.unpack("I", data.read(4))[0] def _parse_unsigned_long(self, data): return struct.unpack("Q", data.read(8))[0] def _parse_double(self, data): return struct.unpack("d", data.read(8))[0] def _parse_string(self, data): value = data.read(2) while not value.endswith(six.b("\x00\x00")): # the string ends at the first instance of \x00\x00 value += data.read(2) return value.decode("utf16")[:-1].encode("utf8") def _parse_char_array(self, data): array_length = struct.unpack("Q", data.read(8))[0] return array.array("B", data.read(array_length)) def _parse_metadata_item(self, data): """ Reads hierarchical data, analogous to a Python dict. """ new_count, length = struct.unpack("