Extend artificial to also write the label_map + data to file for more extensive unit testing

nd2reader/artificial.py

@@ -3,15 +3,22 @@
 import six
 import numpy as np
 import struct
+from nd2reader.common import check_or_make_dir
+from os import path
 
 
 class ArtificialND2(object):
     """Artificial ND2 class (for testing purposes)
     """
+    header = 0xabeceda
+    relative_offset = 0
 
     def __init__(self, file, version=(3, 0)):
-        self._fh = open(file, 'wb')
-        self.write_version(version)
+        self.version = version
+        self.raw_text, self.locations, self.data = None, None, None
+        check_or_make_dir(path.dirname(file))
+        self._fh = open(file, 'w+b', 0)
+        self.write_file()
 
     def __enter__(self):
         return self
@@ -35,18 +42,32 @@ class ArtificialND2(object):
         if self._fh is not None:
             self._fh.close()
 
-    def write_version(self, version=(3, 0)):
+    def write_file(self):
+        self.write_version()
+        self.raw_text, self.locations, self.data = self.write_label_map()
+
+    def write_version(self):
         """Write file header
         """
         # write 16 empty bytes
         self._fh.write(bytearray(16))
 
         # write version info
-        version_info = six.b('ND2 FILE SIGNATURE CHUNK NAME01!Ver%s.%s' % version)
-        self._fh.write(version_info)
+        self._fh.write(self._get_version_string())
+
+    def _get_version_string(self):
+        return six.b('ND2 FILE SIGNATURE CHUNK NAME01!Ver%s.%s' % self.version)
+
+    def _get_version_byte_length(self):
+        return 16 + len(self._get_version_string())
+
+    def write_label_map(self):
+        raw_text, locations, data = self.create_label_map_bytes()
+        self._fh.write(raw_text)
 
-    @staticmethod
-    def create_label_map_bytes():
+        return raw_text, locations, data
+
+    def create_label_map_bytes(self):
         """Construct a binary label map
 
         Returns:
@@ -54,40 +75,117 @@ class ArtificialND2(object):
 
         """
         raw_text = six.b('')
-        labels = {
-            'image_attributes': "ImageAttributesLV!",
-            'image_text_info': "ImageTextInfoLV!",
-            'image_metadata': "ImageMetadataLV!",
-            'image_metadata_sequence': "ImageMetadataSeqLV|0!",
-            'image_calibration': "ImageCalibrationLV|0!",
-            'x_data': "CustomData|X!",
-            'y_data': "CustomData|Y!",
-            'z_data': "CustomData|Z!",
-            'roi_metadata': "CustomData|RoiMetadata_v1!",
-            'pfs_status': "CustomData|PFS_STATUS!",
-            'pfs_offset': "CustomData|PFS_OFFSET!",
-            'guid': "CustomData|GUIDStore!",
-            'description': "CustomData|CustomDescriptionV1_0!",
-            'camera_exposure_time': "CustomData|Camera_ExposureTime1!",
-            'camera_temp': "CustomData|CameraTemp1!",
-            'acquisition_times': "CustomData|AcqTimesCache!",
-            'acquisition_times_2': "CustomData|AcqTimes2Cache!",
-            'acquisition_frames': "CustomData|AcqFramesCache!",
-            'lut_data': "CustomDataVar|LUTDataV1_0!",
-            'grabber_settings': "CustomDataVar|GrabberCameraSettingsV1_0!",
-            'custom_data': "CustomDataVar|CustomDataV2_0!",
-            'app_info': "CustomDataVar|AppInfo_V1_0!",
-            'image_frame_0': "ImageDataSeq|0!"
-        }
-        data = {}
+        labels = [
+            'image_attributes',
+            'image_text_info',
+            'image_metadata',
+            'image_metadata_sequence',
+            'image_calibration',
+            'x_data',
+            'y_data',
+            'z_data',
+            'roi_metadata',
+            'pfs_status',
+            'pfs_offset',
+            'guid',
+            'description',
+            'camera_exposure_time',
+            'camera_temp',
+            'acquisition_times',
+            'acquisition_times_2',
+            'acquisition_frames',
+            'lut_data',
+            'grabber_settings',
+            'custom_data',
+            'app_info',
+            'image_frame_0'
+        ]
+        file_labels = [
+            "ImageAttributesLV!",
+            "ImageTextInfoLV!",
+            "ImageMetadataLV!",
+            "ImageMetadataSeqLV|0!",
+            "ImageCalibrationLV|0!",
+            "CustomData|X!",
+            "CustomData|Y!",
+            "CustomData|Z!",
+            "CustomData|RoiMetadata_v1!",
+            "CustomData|PFS_STATUS!",
+            "CustomData|PFS_OFFSET!",
+            "CustomData|GUIDStore!",
+            "CustomData|CustomDescriptionV1_0!",
+            "CustomData|Camera_ExposureTime1!",
+            "CustomData|CameraTemp1!",
+            "CustomData|AcqTimesCache!",
+            "CustomData|AcqTimes2Cache!",
+            "CustomData|AcqFramesCache!",
+            "CustomDataVar|LUTDataV1_0!",
+            "CustomDataVar|GrabberCameraSettingsV1_0!",
+            "CustomDataVar|CustomDataV2_0!",
+            "CustomDataVar|AppInfo_V1_0!",
+            "ImageDataSeq|0!"
+        ]
+
+        file_data, file_data_dict = self._get_file_data(labels)
+
+        locations = {}
 
         # generate random positions and lengths
-        lengths = np.random.random_integers(1, 999, len(labels))
-        positions = np.subtract(np.cumsum(lengths), lengths[0])
-
-        for length, pos, label in zip(lengths, positions, labels):
-            raw_text += six.b(labels[label])
-            raw_text += struct.pack('QQ', pos, length)
-            data[label] = (pos, length)
-
-        return raw_text, data
+        version_length = self._get_version_byte_length()
+
+        # calculate data length
+        label_length = np.sum([len(six.b(l)) + 16 for l in file_labels])
+
+        # write label map
+        cur_pos = version_length + label_length
+        for label, file_label, data in zip(labels, file_labels, file_data):
+            raw_text += six.b(file_label)
+            data_length = len(data)
+            raw_text += struct.pack('QQ', cur_pos, data_length)
+            locations[label] = (cur_pos, data_length)
+            cur_pos += data_length
+
+        # write data
+        raw_text += six.b('').join(file_data)
+
+        return raw_text, locations, file_data_dict
+
+    def _pack_data_with_metadata(self, data):
+        data = struct.pack('I', data)
+        raw_data = struct.pack("IIQ", self.header, self.relative_offset, len(data))
+        raw_data += data
+        return raw_data
+
+    def _get_file_data(self, labels):
+        file_data = [
+            7,  # ImageAttributesLV!",
+            7,  # ImageTextInfoLV!",
+            7,  # ImageMetadataLV!",
+            7,  # ImageMetadataSeqLV|0!",
+            7,  # ImageCalibrationLV|0!",
+            7,  # CustomData|X!",
+            7,  # CustomData|Y!",
+            7,  # CustomData|Z!",
+            7,  # CustomData|RoiMetadata_v1!",
+            7,  # CustomData|PFS_STATUS!",
+            7,  # CustomData|PFS_OFFSET!",
+            7,  # CustomData|GUIDStore!",
+            7,  # CustomData|CustomDescriptionV1_0!",
+            7,  # CustomData|Camera_ExposureTime1!",
+            7,  # CustomData|CameraTemp1!",
+            7,  # CustomData|AcqTimesCache!",
+            7,  # CustomData|AcqTimes2Cache!",
+            7,  # CustomData|AcqFramesCache!",
+            7,  # CustomDataVar|LUTDataV1_0!",
+            7,  # CustomDataVar|GrabberCameraSettingsV1_0!",
+            7,  # CustomDataVar|CustomDataV2_0!",
+            7,  # CustomDataVar|AppInfo_V1_0!",
+            7,  # ImageDataSeq|0!"
+        ]
+
+        file_data_dict = {l: d for l, d in zip(labels, file_data)}
+
+        # convert to bytes
+        file_data = [self._pack_data_with_metadata(d) for d in file_data]
+
+        return file_data, file_data_dict

nd2reader/raw_metadata.py

@@ -383,9 +383,24 @@ class RawMetadata(object):
             interval = get_from_dict_if_exists('dAvgPeriodDiff', loop)
         if interval is None or interval <= 0:
             # In some cases, both keys are not saved. Then try to calculate it.
-            number_of_loops = get_from_dict_if_exists('uiCount', loop)
-            number_of_loops = number_of_loops if number_of_loops is not None and number_of_loops > 0 else 1
-            interval = duration / number_of_loops
+            interval = RawMetadata._guess_sampling_from_loops(duration, loop)
+        return interval
+
+    @staticmethod
+    def _guess_sampling_from_loops(duration, loop):
+        """ In some cases, both keys are not saved. Then try to calculate it.
+        
+        Args:
+            duration: the total duration of the loop
+            loop: the raw loop data
+
+        Returns:
+            float: the guessed sampling interval in milliseconds
+
+        """
+        number_of_loops = get_from_dict_if_exists('uiCount', loop)
+        number_of_loops = number_of_loops if number_of_loops is not None and number_of_loops > 0 else 1
+        interval = duration / number_of_loops
         return interval
 
     @property

tests/test_label_map.py

@@ -5,7 +5,8 @@ from nd2reader.artificial import ArtificialND2
 
 class TestLabelMap(unittest.TestCase):
     def setUp(self):
-        self.raw_text, self.locations = ArtificialND2.create_label_map_bytes()
+        self.nd2 = ArtificialND2('test_data/test_nd2_label_map001.nd2')
+        self.raw_text, self.locations = self.nd2.raw_text, self.nd2.locations
         self.label_map = LabelMap(self.raw_text)
 
     def test_image_data_location(self):

tests/test_raw_metadata.py

@@ -6,11 +6,11 @@ from nd2reader.raw_metadata import RawMetadata
 
 
 class TestRawMetadata(unittest.TestCase):
-
     def setUp(self):
-        self.raw_text, self.locations = ArtificialND2.create_label_map_bytes()
+        self.nd2 = ArtificialND2('test_data/test_nd2_raw_metadata001.nd2')
+        self.raw_text, self.locations, self.file_data = self.nd2.raw_text, self.nd2.locations, self.nd2.data
         self.label_map = LabelMap(self.raw_text)
-        self.metadata = RawMetadata(None, self.label_map)
+        self.metadata = RawMetadata(self.nd2.file_handle, self.label_map)
 
     def test_parse_roi_shape(self):
         self.assertEqual(self.metadata._parse_roi_shape(3), 'rectangle')
@@ -26,7 +26,7 @@ class TestRawMetadata(unittest.TestCase):
     def test_dict(self):
         self.assertTrue(type(self.metadata.__dict__) is dict)
 
-    def test_parsed_metadata(self):
+    def test_parsed_metadata_has_all_keys(self):
         metadata = self.metadata.get_parsed_metadata()
         self.assertTrue(type(metadata) is dict)
         required_keys = ["height", "width", "date", "fields_of_view", "frames", "z_levels", "total_images_per_channel",
@@ -34,8 +34,5 @@ class TestRawMetadata(unittest.TestCase):
         for required in required_keys:
             self.assertTrue(required in metadata)
 
-        # it should now be stored, see if that dict is returned
-        metadata['height'] = 1
-        self.metadata._metadata_parsed['height'] = 1
-        second_metadata = self.metadata.get_parsed_metadata()
-        self.assertDictEqual(metadata, second_metadata)
+    def test_pfs_status(self):
+        self.assertEqual(self.file_data['pfs_status'], self.metadata.pfs_status[0])