From 9548b92957e90c09c00f99689bc62be270be8c3e Mon Sep 17 00:00:00 2001 From: jim Date: Sat, 7 Nov 2015 11:24:02 -0600 Subject: [PATCH 1/3] #123: created label map object, which parses the pointers to locations in the file where data is stored --- nd2reader/model/label.py | 121 +++++++++++++++++++++++++++++++++++++++ nd2reader/parser/v3.py | 41 +++++++------ 2 files changed, 144 insertions(+), 18 deletions(-) create mode 100644 nd2reader/model/label.py diff --git a/nd2reader/model/label.py b/nd2reader/model/label.py new file mode 100644 index 0000000..adbac16 --- /dev/null +++ b/nd2reader/model/label.py @@ -0,0 +1,121 @@ +import six +import struct +from collections import namedtuple +import re + + +data_location = namedtuple("DataLocation", ["location", "length"]) + + +class LabelMap(object): + """ + """ + def __init__(self, raw_binary_data): + self._data = raw_binary_data + + def _get_location(self, label): + try: + label_location = self._data.index(label) + len(label) + return self._parse_data_location(label_location) + except ValueError: + return None + + def _parse_data_location(self, label_location): + location, length = struct.unpack("QQ", self._data[label_location: label_location + 16]) + return data_location(location=location, length=length) + + @property + def image_text_info(self): + return self._get_location(six.b("ImageTextInfoLV!")) + + @property + def image_metadata_sequence(self): + # there is always only one of these, even though it has a pipe followed by a zero, which is how they do indexes + return self._get_location(six.b("ImageMetadataSeqLV|0!")) + + @property + def image_data(self): + image_data = {} + regex = re.compile(six.b("""ImageDataSeq\|(\d+)!""")) + for match in regex.finditer(self._data): + if match: + print(match.start(), match.end()) + location = self._parse_data_location(match.end()) + image_data[int(match.group(1))] = location + return image_data + + @property + def image_calibration(self): + return self._get_location(six.b("ImageCalibrationLV|0!")) + + @property + def image_attributes(self): + return self._get_location(six.b("ImageAttributesLV!")) + + @property + def x_data(self): + return self._get_location(six.b("CustomData|X!")) + + @property + def y_data(self): + return self._get_location(six.b("CustomData|Y!")) + + @property + def z_data(self): + return self._get_location(six.b("CustomData|Z!")) + + @property + def roi_metadata(self): + return self._get_location(six.b("CustomData|RoiMetadata_v1!")) + + @property + def pfs_status(self): + return self._get_location(six.b("CustomData|PFS_STATUS!")) + + @property + def pfs_offset(self): + return self._get_location(six.b("CustomData|PFS_OFFSET!")) + + @property + def guid(self): + return self._get_location(six.b("CustomData|GUIDStore!")) + + @property + def description(self): + return self._get_location(six.b("CustomData|CustomDescriptionV1_0!")) + + @property + def camera_exposure_time(self): + return self._get_location(six.b("CustomData|Camera_ExposureTime1!")) + + @property + def camera_temp(self): + return self._get_location(six.b("CustomData|CameraTemp1!")) + + @property + def acquisition_times(self): + return self._get_location(six.b("CustomData|AcqTimesCache!")) + + @property + def acquisition_times_2(self): + return self._get_location(six.b("CustomData|AcqTimes2Cache!")) + + @property + def acquisition_frames(self): + return self._get_location(six.b("CustomData|AcqFramesCache!")) + + @property + def lut_data(self): + return self._get_location(six.b("CustomDataVar|LUTDataV1_0!")) + + @property + def grabber_settings(self): + return self._get_location(six.b("CustomDataVar|GrabberCameraSettingsV1_0!")) + + @property + def custom_data(self): + return self._get_location(six.b("CustomDataVar|CustomDataV2_0!")) + + @property + def app_info(self): + return self._get_location(six.b("CustomDataVar|AppInfo_V1_0!")) diff --git a/nd2reader/parser/v3.py b/nd2reader/parser/v3.py index 147902a..e831f62 100644 --- a/nd2reader/parser/v3.py +++ b/nd2reader/parser/v3.py @@ -3,6 +3,7 @@ import array from datetime import datetime from nd2reader.model.metadata import Metadata +from nd2reader.model.label import LabelMap from nd2reader.parser.base import BaseParser from nd2reader.driver.v3 import V3Driver from nd2reader.common.v3 import read_chunk @@ -40,11 +41,7 @@ class V3Parser(BaseParser): def driver(self): return V3Driver(self.metadata, self._label_map, self._fh) - def _parse_metadata(self): - """ - Reads all metadata and instantiates the Metadata object. - - """ + def _build_metadata_dict(self): metadata_dict = {} self._label_map = self._build_label_map() for label in self._label_map.keys(): @@ -52,7 +49,14 @@ class V3Parser(BaseParser): data = read_chunk(self._fh, self._label_map[label]) stop = label.index(six.b("LV")) metadata_dict[label[:stop]] = self._read_metadata(data, 1) + return metadata_dict + def _parse_metadata(self): + """ + Reads all metadata and instantiates the Metadata object. + + """ + metadata_dict = self._build_metadata_dict() height = metadata_dict[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiHeight')] width = metadata_dict[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiWidth')] channels = self._parse_channels(metadata_dict) @@ -208,23 +212,24 @@ class V3Parser(BaseParser): :rtype: dict """ - label_map = {} + # label_map = {} self._fh.seek(-8, 2) chunk_map_start_location = struct.unpack("Q", self._fh.read(8))[0] self._fh.seek(chunk_map_start_location) raw_text = self._fh.read(-1) - label_start = raw_text.index(V3Parser.CHUNK_MAP_START) + 32 - - while True: - data_start = raw_text.index(six.b("!"), label_start) + 1 - key = raw_text[label_start: data_start] - location, length = struct.unpack("QQ", raw_text[data_start: data_start + 16]) - if key == V3Parser.CHUNK_MAP_END: - # We've reached the end of the chunk map - break - label_map[key] = location - label_start = data_start + 16 - return label_map + # label_start = raw_text.index(V3Parser.CHUNK_MAP_START) + 32 + return LabelMap(raw_text) + + # while True: + # data_start = raw_text.index(six.b("!"), label_start) + 1 + # key = raw_text[label_start: data_start] + # location, length = struct.unpack("QQ", raw_text[data_start: data_start + 16]) + # if key == V3Parser.CHUNK_MAP_END: + # # We've reached the end of the chunk map + # break + # label_map[key] = location + # label_start = data_start + 16 + # return label_map def _parse_unsigned_char(self, data): return struct.unpack("B", data.read(1))[0] From e9ff0915bffc6a0dab7726b72c3130cd5a6f7f0a Mon Sep 17 00:00:00 2001 From: jim Date: Sat, 7 Nov 2015 15:16:23 -0600 Subject: [PATCH 2/3] #123 switched to the new LabelMap object, added support for another metadata field --- nd2reader/model/label.py | 11 ++++----- nd2reader/parser/v3.py | 49 +++++++++++++++++++++++----------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/nd2reader/model/label.py b/nd2reader/model/label.py index adbac16..daa6a16 100644 --- a/nd2reader/model/label.py +++ b/nd2reader/model/label.py @@ -1,12 +1,8 @@ import six import struct -from collections import namedtuple import re -data_location = namedtuple("DataLocation", ["location", "length"]) - - class LabelMap(object): """ """ @@ -22,12 +18,16 @@ class LabelMap(object): def _parse_data_location(self, label_location): location, length = struct.unpack("QQ", self._data[label_location: label_location + 16]) - return data_location(location=location, length=length) + return location @property def image_text_info(self): return self._get_location(six.b("ImageTextInfoLV!")) + @property + def image_metadata(self): + return self._get_location(six.b("ImageMetadataLV!")) + @property def image_metadata_sequence(self): # there is always only one of these, even though it has a pipe followed by a zero, which is how they do indexes @@ -39,7 +39,6 @@ class LabelMap(object): regex = re.compile(six.b("""ImageDataSeq\|(\d+)!""")) for match in regex.finditer(self._data): if match: - print(match.start(), match.end()) location = self._parse_data_location(match.end()) image_data[int(match.group(1))] = location return image_data diff --git a/nd2reader/parser/v3.py b/nd2reader/parser/v3.py index e831f62..4e11927 100644 --- a/nd2reader/parser/v3.py +++ b/nd2reader/parser/v3.py @@ -42,14 +42,34 @@ class V3Parser(BaseParser): return V3Driver(self.metadata, self._label_map, self._fh) def _build_metadata_dict(self): - metadata_dict = {} self._label_map = self._build_label_map() - for label in self._label_map.keys(): - if label.endswith(six.b("LV!")) or six.b("LV|") in label: - data = read_chunk(self._fh, self._label_map[label]) - stop = label.index(six.b("LV")) - metadata_dict[label[:stop]] = self._read_metadata(data, 1) - return metadata_dict + raw_data = {"image_text_info": read_chunk(self._fh, self._label_map.image_text_info), + "image_metadata_sequence": read_chunk(self._fh, self._label_map.image_metadata_sequence), + # "image_data": read_chunk(self._fh, self._label_map.image_data), + "image_calibration": read_chunk(self._fh, self._label_map.image_calibration), + "image_attributes": read_chunk(self._fh, self._label_map.image_attributes), + # "x_data": read_chunk(self._fh, self._label_map.x_data), + # "y_data": read_chunk(self._fh, self._label_map.y_data), + # "z_data": read_chunk(self._fh, self._label_map.z_data), + # "roi_metadata": read_chunk(self._fh, self._label_map.roi_metadata), + # "pfs_status": read_chunk(self._fh, self._label_map.pfs_status), + # "pfs_offset": read_chunk(self._fh, self._label_map.pfs_offset), + # "guid": read_chunk(self._fh, self._label_map.guid), + # "description": read_chunk(self._fh, self._label_map.description), + # "camera_exposure_time": read_chunk(self._fh, self._label_map.camera_exposure_time), + # "camera_temp": read_chunk(self._fh, self._label_map.camera_temp), + # "acquisition_times": read_chunk(self._fh, self._label_map.acquisition_times), + # "acquisition_times_2": read_chunk(self._fh, self._label_map.acquisition_times_2), + # "acquisition_frames": read_chunk(self._fh, self._label_map.acquisition_frames), + # "lut_data": read_chunk(self._fh, self._label_map.lut_data), + # "grabber_settings": read_chunk(self._fh, self._label_map.grabber_settings), + # "custom_data": read_chunk(self._fh, self._label_map.custom_data), + # "app_info": read_chunk(self._fh, self._label_map.app_info) + } + if self._label_map.image_metadata: + raw_data["image_metadata"] = read_chunk(self._fh, self._label_map.image_metadata) + + return {key: self._read_metadata(data, 1) for key, data in raw_data.items()} def _parse_metadata(self): """ @@ -209,28 +229,15 @@ class V3Parser(BaseParser): as some of the bytes contain the value 33, which is the ASCII code for "!". So we iteratively find each label, grab the subsequent data (always 16 bytes long), advance to the next label and repeat. - :rtype: dict + :rtype: LabelMap """ - # label_map = {} self._fh.seek(-8, 2) chunk_map_start_location = struct.unpack("Q", self._fh.read(8))[0] self._fh.seek(chunk_map_start_location) raw_text = self._fh.read(-1) - # label_start = raw_text.index(V3Parser.CHUNK_MAP_START) + 32 return LabelMap(raw_text) - # while True: - # data_start = raw_text.index(six.b("!"), label_start) + 1 - # key = raw_text[label_start: data_start] - # location, length = struct.unpack("QQ", raw_text[data_start: data_start + 16]) - # if key == V3Parser.CHUNK_MAP_END: - # # We've reached the end of the chunk map - # break - # label_map[key] = location - # label_start = data_start + 16 - # return label_map - def _parse_unsigned_char(self, data): return struct.unpack("B", data.read(1))[0] From 4943a854f1679be57d18ee96407ce913260da4c0 Mon Sep 17 00:00:00 2001 From: jim Date: Tue, 17 Nov 2015 10:41:07 -0600 Subject: [PATCH 3/3] resolves #123: the label map and raw metadata are now accessible, so we can more easily compare what they contain to the information in the XML (which we're still not parsing) --- functional_tests/single.py | 7 +++++++ nd2reader/driver/v3.py | 2 +- nd2reader/model/label.py | 18 +++++++++--------- nd2reader/parser/v3.py | 14 +++++++------- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/functional_tests/single.py b/functional_tests/single.py index cc2f5c4..d884248 100644 --- a/functional_tests/single.py +++ b/functional_tests/single.py @@ -25,6 +25,13 @@ class FunctionalTests(unittest.TestCase): def test_length(self): self.assertEqual(len(self.nd2), 1) + def test_actual_length(self): + count = 0 + for image in self.nd2: + if image is not None: + count += 1 + self.assertEqual(len(self.nd2), count) + def test_frames(self): self.assertEqual(len(self.nd2.frames), 1) diff --git a/nd2reader/driver/v3.py b/nd2reader/driver/v3.py index 29746ae..8c00d4d 100644 --- a/nd2reader/driver/v3.py +++ b/nd2reader/driver/v3.py @@ -130,7 +130,7 @@ class V3Driver(object): :raises: NoImageError """ - chunk = self._label_map[six.b("ImageDataSeq|%d!" % image_group_number)] + chunk = self._label_map.get_image_data_location(image_group_number) data = read_chunk(self._file_handle, chunk) # All images in the same image group share the same timestamp! So if you have complicated image data, # your timestamps may not be entirely accurate. Practically speaking though, they'll only be off by a few diff --git a/nd2reader/model/label.py b/nd2reader/model/label.py index daa6a16..a3ebd80 100644 --- a/nd2reader/model/label.py +++ b/nd2reader/model/label.py @@ -8,6 +8,7 @@ class LabelMap(object): """ def __init__(self, raw_binary_data): self._data = raw_binary_data + self._image_data = {} def _get_location(self, label): try: @@ -33,15 +34,14 @@ class LabelMap(object): # there is always only one of these, even though it has a pipe followed by a zero, which is how they do indexes return self._get_location(six.b("ImageMetadataSeqLV|0!")) - @property - def image_data(self): - image_data = {} - regex = re.compile(six.b("""ImageDataSeq\|(\d+)!""")) - for match in regex.finditer(self._data): - if match: - location = self._parse_data_location(match.end()) - image_data[int(match.group(1))] = location - return image_data + def get_image_data_location(self, index): + if not self._image_data: + regex = re.compile(six.b("""ImageDataSeq\|(\d+)!""")) + for match in regex.finditer(self._data): + if match: + location = self._parse_data_location(match.end()) + self._image_data[int(match.group(1))] = location + return self._image_data[index] @property def image_calibration(self): diff --git a/nd2reader/parser/v3.py b/nd2reader/parser/v3.py index 4e11927..4673f39 100644 --- a/nd2reader/parser/v3.py +++ b/nd2reader/parser/v3.py @@ -77,8 +77,8 @@ class V3Parser(BaseParser): """ metadata_dict = self._build_metadata_dict() - height = metadata_dict[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiHeight')] - width = metadata_dict[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiWidth')] + height = metadata_dict['image_attributes'][six.b('SLxImageAttributes')][six.b('uiHeight')] + width = metadata_dict['image_attributes'][six.b('SLxImageAttributes')][six.b('uiWidth')] channels = self._parse_channels(metadata_dict) date = self._parse_date(metadata_dict) fields_of_view = self._parse_fields_of_view(metadata_dict) @@ -95,7 +95,7 @@ class V3Parser(BaseParser): :rtype: datetime.datetime() or None """ - for line in metadata_dict[six.b('ImageTextInfo')][six.b('SLxImageTextInfo')].values(): + for line in metadata_dict['image_text_info'][six.b('SLxImageTextInfo')].values(): line = line.decode("utf8") absolute_start_12 = None absolute_start_24 = None @@ -123,9 +123,9 @@ class V3Parser(BaseParser): """ channels = [] - metadata = metadata_dict[six.b('ImageMetadataSeq')][six.b('SLxPictureMetadata')][six.b('sPicturePlanes')] + metadata = metadata_dict['image_metadata_sequence'][six.b('SLxPictureMetadata')][six.b('sPicturePlanes')] try: - validity = metadata_dict[six.b('ImageMetadata')][six.b('SLxExperiment')][six.b('ppNextLevelEx')][six.b('')][0][six.b('ppNextLevelEx')][six.b('')][0][six.b('pItemValid')] + validity = metadata_dict['image_metadata'][six.b('SLxExperiment')][six.b('ppNextLevelEx')][six.b('')][0][six.b('ppNextLevelEx')][six.b('')][0][six.b('pItemValid')] except KeyError: # If none of the channels have been deleted, there is no validity list, so we just make one validity = [True for _ in metadata] @@ -181,7 +181,7 @@ class V3Parser(BaseParser): :rtype: str """ - for line in metadata_dict[six.b('ImageTextInfo')][six.b('SLxImageTextInfo')].values(): + for line in metadata_dict['image_text_info'][six.b('SLxImageTextInfo')].values(): if six.b("Dimensions:") in line: metadata = line break @@ -221,7 +221,7 @@ class V3Parser(BaseParser): :rtype: int """ - return metadata_dict[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiSequenceCount')] + return metadata_dict['image_attributes'][six.b('SLxImageAttributes')][six.b('uiSequenceCount')] def _build_label_map(self): """