From 9f3e4c1b35dc2e73e04049d00119109040552eb5 Mon Sep 17 00:00:00 2001 From: Ruben Verweij Date: Fri, 7 Jul 2017 12:22:03 +0200 Subject: [PATCH] Extend artificial to also write the label_map + data to file for more extensive unit testing --- nd2reader/artificial.py | 182 ++++++++++++++++++++++++++++--------- nd2reader/raw_metadata.py | 21 ++++- tests/test_label_map.py | 3 +- tests/test_raw_metadata.py | 15 ++- 4 files changed, 166 insertions(+), 55 deletions(-) diff --git a/nd2reader/artificial.py b/nd2reader/artificial.py index 68b5cdb..3a0a8f9 100644 --- a/nd2reader/artificial.py +++ b/nd2reader/artificial.py @@ -3,15 +3,22 @@ import six import numpy as np import struct +from nd2reader.common import check_or_make_dir +from os import path class ArtificialND2(object): """Artificial ND2 class (for testing purposes) """ + header = 0xabeceda + relative_offset = 0 def __init__(self, file, version=(3, 0)): - self._fh = open(file, 'wb') - self.write_version(version) + self.version = version + self.raw_text, self.locations, self.data = None, None, None + check_or_make_dir(path.dirname(file)) + self._fh = open(file, 'w+b', 0) + self.write_file() def __enter__(self): return self @@ -35,18 +42,32 @@ class ArtificialND2(object): if self._fh is not None: self._fh.close() - def write_version(self, version=(3, 0)): + def write_file(self): + self.write_version() + self.raw_text, self.locations, self.data = self.write_label_map() + + def write_version(self): """Write file header """ # write 16 empty bytes self._fh.write(bytearray(16)) # write version info - version_info = six.b('ND2 FILE SIGNATURE CHUNK NAME01!Ver%s.%s' % version) - self._fh.write(version_info) + self._fh.write(self._get_version_string()) + + def _get_version_string(self): + return six.b('ND2 FILE SIGNATURE CHUNK NAME01!Ver%s.%s' % self.version) + + def _get_version_byte_length(self): + return 16 + len(self._get_version_string()) + + def write_label_map(self): + raw_text, locations, data = self.create_label_map_bytes() + self._fh.write(raw_text) - @staticmethod - def create_label_map_bytes(): + return raw_text, locations, data + + def create_label_map_bytes(self): """Construct a binary label map Returns: @@ -54,40 +75,117 @@ class ArtificialND2(object): """ raw_text = six.b('') - labels = { - 'image_attributes': "ImageAttributesLV!", - 'image_text_info': "ImageTextInfoLV!", - 'image_metadata': "ImageMetadataLV!", - 'image_metadata_sequence': "ImageMetadataSeqLV|0!", - 'image_calibration': "ImageCalibrationLV|0!", - 'x_data': "CustomData|X!", - 'y_data': "CustomData|Y!", - 'z_data': "CustomData|Z!", - 'roi_metadata': "CustomData|RoiMetadata_v1!", - 'pfs_status': "CustomData|PFS_STATUS!", - 'pfs_offset': "CustomData|PFS_OFFSET!", - 'guid': "CustomData|GUIDStore!", - 'description': "CustomData|CustomDescriptionV1_0!", - 'camera_exposure_time': "CustomData|Camera_ExposureTime1!", - 'camera_temp': "CustomData|CameraTemp1!", - 'acquisition_times': "CustomData|AcqTimesCache!", - 'acquisition_times_2': "CustomData|AcqTimes2Cache!", - 'acquisition_frames': "CustomData|AcqFramesCache!", - 'lut_data': "CustomDataVar|LUTDataV1_0!", - 'grabber_settings': "CustomDataVar|GrabberCameraSettingsV1_0!", - 'custom_data': "CustomDataVar|CustomDataV2_0!", - 'app_info': "CustomDataVar|AppInfo_V1_0!", - 'image_frame_0': "ImageDataSeq|0!" - } - data = {} + labels = [ + 'image_attributes', + 'image_text_info', + 'image_metadata', + 'image_metadata_sequence', + 'image_calibration', + 'x_data', + 'y_data', + 'z_data', + 'roi_metadata', + 'pfs_status', + 'pfs_offset', + 'guid', + 'description', + 'camera_exposure_time', + 'camera_temp', + 'acquisition_times', + 'acquisition_times_2', + 'acquisition_frames', + 'lut_data', + 'grabber_settings', + 'custom_data', + 'app_info', + 'image_frame_0' + ] + file_labels = [ + "ImageAttributesLV!", + "ImageTextInfoLV!", + "ImageMetadataLV!", + "ImageMetadataSeqLV|0!", + "ImageCalibrationLV|0!", + "CustomData|X!", + "CustomData|Y!", + "CustomData|Z!", + "CustomData|RoiMetadata_v1!", + "CustomData|PFS_STATUS!", + "CustomData|PFS_OFFSET!", + "CustomData|GUIDStore!", + "CustomData|CustomDescriptionV1_0!", + "CustomData|Camera_ExposureTime1!", + "CustomData|CameraTemp1!", + "CustomData|AcqTimesCache!", + "CustomData|AcqTimes2Cache!", + "CustomData|AcqFramesCache!", + "CustomDataVar|LUTDataV1_0!", + "CustomDataVar|GrabberCameraSettingsV1_0!", + "CustomDataVar|CustomDataV2_0!", + "CustomDataVar|AppInfo_V1_0!", + "ImageDataSeq|0!" + ] + + file_data, file_data_dict = self._get_file_data(labels) + + locations = {} # generate random positions and lengths - lengths = np.random.random_integers(1, 999, len(labels)) - positions = np.subtract(np.cumsum(lengths), lengths[0]) - - for length, pos, label in zip(lengths, positions, labels): - raw_text += six.b(labels[label]) - raw_text += struct.pack('QQ', pos, length) - data[label] = (pos, length) - - return raw_text, data + version_length = self._get_version_byte_length() + + # calculate data length + label_length = np.sum([len(six.b(l)) + 16 for l in file_labels]) + + # write label map + cur_pos = version_length + label_length + for label, file_label, data in zip(labels, file_labels, file_data): + raw_text += six.b(file_label) + data_length = len(data) + raw_text += struct.pack('QQ', cur_pos, data_length) + locations[label] = (cur_pos, data_length) + cur_pos += data_length + + # write data + raw_text += six.b('').join(file_data) + + return raw_text, locations, file_data_dict + + def _pack_data_with_metadata(self, data): + data = struct.pack('I', data) + raw_data = struct.pack("IIQ", self.header, self.relative_offset, len(data)) + raw_data += data + return raw_data + + def _get_file_data(self, labels): + file_data = [ + 7, # ImageAttributesLV!", + 7, # ImageTextInfoLV!", + 7, # ImageMetadataLV!", + 7, # ImageMetadataSeqLV|0!", + 7, # ImageCalibrationLV|0!", + 7, # CustomData|X!", + 7, # CustomData|Y!", + 7, # CustomData|Z!", + 7, # CustomData|RoiMetadata_v1!", + 7, # CustomData|PFS_STATUS!", + 7, # CustomData|PFS_OFFSET!", + 7, # CustomData|GUIDStore!", + 7, # CustomData|CustomDescriptionV1_0!", + 7, # CustomData|Camera_ExposureTime1!", + 7, # CustomData|CameraTemp1!", + 7, # CustomData|AcqTimesCache!", + 7, # CustomData|AcqTimes2Cache!", + 7, # CustomData|AcqFramesCache!", + 7, # CustomDataVar|LUTDataV1_0!", + 7, # CustomDataVar|GrabberCameraSettingsV1_0!", + 7, # CustomDataVar|CustomDataV2_0!", + 7, # CustomDataVar|AppInfo_V1_0!", + 7, # ImageDataSeq|0!" + ] + + file_data_dict = {l: d for l, d in zip(labels, file_data)} + + # convert to bytes + file_data = [self._pack_data_with_metadata(d) for d in file_data] + + return file_data, file_data_dict diff --git a/nd2reader/raw_metadata.py b/nd2reader/raw_metadata.py index f5dbec4..f03587d 100644 --- a/nd2reader/raw_metadata.py +++ b/nd2reader/raw_metadata.py @@ -383,9 +383,24 @@ class RawMetadata(object): interval = get_from_dict_if_exists('dAvgPeriodDiff', loop) if interval is None or interval <= 0: # In some cases, both keys are not saved. Then try to calculate it. - number_of_loops = get_from_dict_if_exists('uiCount', loop) - number_of_loops = number_of_loops if number_of_loops is not None and number_of_loops > 0 else 1 - interval = duration / number_of_loops + interval = RawMetadata._guess_sampling_from_loops(duration, loop) + return interval + + @staticmethod + def _guess_sampling_from_loops(duration, loop): + """ In some cases, both keys are not saved. Then try to calculate it. + + Args: + duration: the total duration of the loop + loop: the raw loop data + + Returns: + float: the guessed sampling interval in milliseconds + + """ + number_of_loops = get_from_dict_if_exists('uiCount', loop) + number_of_loops = number_of_loops if number_of_loops is not None and number_of_loops > 0 else 1 + interval = duration / number_of_loops return interval @property diff --git a/tests/test_label_map.py b/tests/test_label_map.py index c9a2369..1f58fd6 100644 --- a/tests/test_label_map.py +++ b/tests/test_label_map.py @@ -5,7 +5,8 @@ from nd2reader.artificial import ArtificialND2 class TestLabelMap(unittest.TestCase): def setUp(self): - self.raw_text, self.locations = ArtificialND2.create_label_map_bytes() + self.nd2 = ArtificialND2('test_data/test_nd2_label_map001.nd2') + self.raw_text, self.locations = self.nd2.raw_text, self.nd2.locations self.label_map = LabelMap(self.raw_text) def test_image_data_location(self): diff --git a/tests/test_raw_metadata.py b/tests/test_raw_metadata.py index 635341b..0269ecb 100644 --- a/tests/test_raw_metadata.py +++ b/tests/test_raw_metadata.py @@ -6,11 +6,11 @@ from nd2reader.raw_metadata import RawMetadata class TestRawMetadata(unittest.TestCase): - def setUp(self): - self.raw_text, self.locations = ArtificialND2.create_label_map_bytes() + self.nd2 = ArtificialND2('test_data/test_nd2_raw_metadata001.nd2') + self.raw_text, self.locations, self.file_data = self.nd2.raw_text, self.nd2.locations, self.nd2.data self.label_map = LabelMap(self.raw_text) - self.metadata = RawMetadata(None, self.label_map) + self.metadata = RawMetadata(self.nd2.file_handle, self.label_map) def test_parse_roi_shape(self): self.assertEqual(self.metadata._parse_roi_shape(3), 'rectangle') @@ -26,7 +26,7 @@ class TestRawMetadata(unittest.TestCase): def test_dict(self): self.assertTrue(type(self.metadata.__dict__) is dict) - def test_parsed_metadata(self): + def test_parsed_metadata_has_all_keys(self): metadata = self.metadata.get_parsed_metadata() self.assertTrue(type(metadata) is dict) required_keys = ["height", "width", "date", "fields_of_view", "frames", "z_levels", "total_images_per_channel", @@ -34,8 +34,5 @@ class TestRawMetadata(unittest.TestCase): for required in required_keys: self.assertTrue(required in metadata) - # it should now be stored, see if that dict is returned - metadata['height'] = 1 - self.metadata._metadata_parsed['height'] = 1 - second_metadata = self.metadata.get_parsed_metadata() - self.assertDictEqual(metadata, second_metadata) + def test_pfs_status(self): + self.assertEqual(self.file_data['pfs_status'], self.metadata.pfs_status[0])