Merge branch 'master' of github.com:rbnvrw/nd2reader

docs

@@ -1 +1 @@
-Subproject commit 1b43456084689bad958984cf8a0cad7f63b52286
+Subproject commit 6a12543461144fc164942c79a0b1457fd6284779

nd2reader/__init__.py

@@ -1,4 +1,4 @@
 from nd2reader.reader import ND2Reader
 from nd2reader.legacy import Nd2
 
-__version__ = '3.0.9'
+__version__ = '3.1.0'

nd2reader/artificial.py

@@ -6,6 +6,27 @@ import struct
 from nd2reader.common import check_or_make_dir
 from os import path
 
+global_labels = ['image_attributes', 'image_text_info', 'image_metadata',
+                 'image_metadata_sequence', 'image_calibration', 'x_data',
+                 'y_data', 'z_data', 'roi_metadata', 'pfs_status', 'pfs_offset',
+                 'guid', 'description', 'camera_exposure_time', 'camera_temp',
+                 'acquisition_times', 'acquisition_times_2',
+                 'acquisition_frames', 'lut_data', 'grabber_settings',
+                 'custom_data', 'app_info', 'image_frame_0']
+
+global_file_labels = ["ImageAttributesLV!", "ImageTextInfoLV!",
+                      "ImageMetadataLV!", "ImageMetadataSeqLV|0!",
+                      "ImageCalibrationLV|0!", "CustomData|X!", "CustomData|Y!",
+                      "CustomData|Z!", "CustomData|RoiMetadata_v1!",
+                      "CustomData|PFS_STATUS!", "CustomData|PFS_OFFSET!",
+                      "CustomData|GUIDStore!", "CustomData|CustomDescriptionV1_0!",
+                      "CustomData|Camera_ExposureTime1!", "CustomData|CameraTemp1!",
+                      "CustomData|AcqTimesCache!", "CustomData|AcqTimes2Cache!",
+                      "CustomData|AcqFramesCache!", "CustomDataVar|LUTDataV1_0!",
+                      "CustomDataVar|GrabberCameraSettingsV1_0!",
+                      "CustomDataVar|CustomDataV2_0!", "CustomDataVar|AppInfo_V1_0!",
+                      "ImageDataSeq|0!"]
+
 
 class ArtificialND2(object):
     """Artificial ND2 class (for testing purposes)
@@ -105,56 +126,8 @@ class ArtificialND2(object):
 
         """
         raw_text = six.b('')
-        labels = [
-            'image_attributes',
-            'image_text_info',
-            'image_metadata',
-            'image_metadata_sequence',
-            'image_calibration',
-            'x_data',
-            'y_data',
-            'z_data',
-            'roi_metadata',
-            'pfs_status',
-            'pfs_offset',
-            'guid',
-            'description',
-            'camera_exposure_time',
-            'camera_temp',
-            'acquisition_times',
-            'acquisition_times_2',
-            'acquisition_frames',
-            'lut_data',
-            'grabber_settings',
-            'custom_data',
-            'app_info',
-            'image_frame_0'
-        ]
-        file_labels = [
-            "ImageAttributesLV!",
-            "ImageTextInfoLV!",
-            "ImageMetadataLV!",
-            "ImageMetadataSeqLV|0!",
-            "ImageCalibrationLV|0!",
-            "CustomData|X!",
-            "CustomData|Y!",
-            "CustomData|Z!",
-            "CustomData|RoiMetadata_v1!",
-            "CustomData|PFS_STATUS!",
-            "CustomData|PFS_OFFSET!",
-            "CustomData|GUIDStore!",
-            "CustomData|CustomDescriptionV1_0!",
-            "CustomData|Camera_ExposureTime1!",
-            "CustomData|CameraTemp1!",
-            "CustomData|AcqTimesCache!",
-            "CustomData|AcqTimes2Cache!",
-            "CustomData|AcqFramesCache!",
-            "CustomDataVar|LUTDataV1_0!",
-            "CustomDataVar|GrabberCameraSettingsV1_0!",
-            "CustomDataVar|CustomDataV2_0!",
-            "CustomDataVar|AppInfo_V1_0!",
-            "ImageDataSeq|0!"
-        ]
+        labels = global_labels
+        file_labels = global_file_labels
 
         file_data, file_data_dict = self._get_file_data(labels)
 
@@ -242,42 +215,42 @@ class ArtificialND2(object):
 
         return raw_data
 
+    @staticmethod
+    def _get_slx_img_attrib():
+        return {'uiWidth': 128,
+                'uiWidthBytes': 256,
+                'uiHeight': 128,
+                'uiComp': 1,
+                'uiBpcInMemory': 16,
+                'uiBpcSignificant': 12,
+                'uiSequenceCount': 70,
+                'uiTileWidth': 128,
+                'uiTileHeight': 128,
+                'eCompression': 2,
+                'dCompressionParam': -1.0,
+                'ePixelType': 1,
+                'uiVirtualComponents': 1
+                }
+
+    @staticmethod
+    def _get_slx_picture_metadata():
+        return {'sPicturePlanes':
+                {
+                    'sPlaneNew': {
+                        # channels are numbered a0, a1, ..., aN
+                        'a0': {
+                            'sDescription': 'TRITC'
+                            }
+                        }
+                    }
+                }
+
     def _get_file_data(self, labels):
         file_data = [
-            {
-                'SLxImageAttributes':
-                    {
-                        'uiWidth': 128,
-                        'uiWidthBytes': 256,
-                        'uiHeight': 128,
-                        'uiComp': 1,
-                        'uiBpcInMemory': 16,
-                        'uiBpcSignificant': 12,
-                        'uiSequenceCount': 70,
-                        'uiTileWidth': 128,
-                        'uiTileHeight': 128,
-                        'eCompression': 2,
-                        'dCompressionParam': -1.0,
-                        'ePixelType': 1,
-                        'uiVirtualComponents': 1
-                    }
-            },  # ImageAttributesLV!",
+            {'SLxImageAttributes': self._get_slx_img_attrib()},  # ImageAttributesLV!",
             7,  # ImageTextInfoLV!",
             7,  # ImageMetadataLV!",
-            {
-                'SLxPictureMetadata':
-                    {
-                        'sPicturePlanes':
-                            {
-                                'sPlaneNew': {
-                                    # channels are numbered a0, a1, ..., aN
-                                    'a0': {
-                                        'sDescription': 'TRITC'
-                                    }
-                                }
-                            }
-                    }
-            },  # ImageMetadataSeqLV|0!",
+            {'SLxPictureMetadata': self._get_slx_picture_metadata()},  # ImageMetadataSeqLV|0!",
             7,  # ImageCalibrationLV|0!",
             7,  # CustomData|X!",
             7,  # CustomData|Y!",
@@ -289,9 +262,9 @@ class ArtificialND2(object):
             7,  # CustomData|CustomDescriptionV1_0!",
             7,  # CustomData|Camera_ExposureTime1!",
             7,  # CustomData|CameraTemp1!",
-            7,  # CustomData|AcqTimesCache!",
-            7,  # CustomData|AcqTimes2Cache!",
-            7,  # CustomData|AcqFramesCache!",
+            [0],  # CustomData|AcqTimesCache!",
+            [0],  # CustomData|AcqTimes2Cache!",
+            [0],  # CustomData|AcqFramesCache!",
             7,  # CustomDataVar|LUTDataV1_0!",
             7,  # CustomDataVar|GrabberCameraSettingsV1_0!",
             7,  # CustomDataVar|CustomDataV2_0!",

nd2reader/common_raw_metadata.py

@@ -0,0 +1,101 @@
+import six
+import warnings
+
+from nd2reader.common import get_from_dict_if_exists
+
+
+def parse_if_not_none(to_check, callback):
+    if to_check is not None:
+        return callback()
+    return None
+
+
+def parse_dimension_text_line(line):
+    if six.b("Dimensions:") in line:
+        entries = line.split(six.b("\r\n"))
+        for entry in entries:
+            if entry.startswith(six.b("Dimensions:")):
+                return entry
+    return None
+
+
+def parse_roi_shape(shape):
+    if shape == 3:
+        return 'rectangle'
+    elif shape == 9:
+        return 'circle'
+
+    return None
+
+
+def parse_roi_type(type_no):
+    if type_no == 4:
+        return 'stimulation'
+    elif type_no == 3:
+        return 'reference'
+    elif type_no == 2:
+        return 'background'
+
+    return None
+
+
+def get_loops_from_data(loop_data):
+    loops = [loop_data]
+    if six.b('uiPeriodCount') in loop_data and loop_data[six.b('uiPeriodCount')] > 0:
+        # special ND experiment
+        if six.b('pPeriod') not in loop_data:
+            return []
+
+        # take the first dictionary element, it contains all loop data
+        loops = loop_data[six.b('pPeriod')][list(loop_data[six.b('pPeriod')].keys())[0]]
+    return loops
+
+
+def guess_sampling_from_loops(duration, loop):
+    """ In some cases, both keys are not saved. Then try to calculate it.
+    
+    Args:
+        duration: the total duration of the loop
+        loop: the raw loop data
+
+    Returns:
+        float: the guessed sampling interval in milliseconds
+
+    """
+    number_of_loops = get_from_dict_if_exists('uiCount', loop)
+    number_of_loops = number_of_loops if number_of_loops is not None and number_of_loops > 0 else 1
+    interval = duration / number_of_loops
+    return interval
+
+
+def determine_sampling_interval(duration, loop):
+    """Determines the loop sampling interval in milliseconds
+
+    Args:
+        duration: loop duration in milliseconds
+        loop: loop dictionary
+
+    Returns:
+        float: the sampling interval in milliseconds
+
+    """
+    interval = get_from_dict_if_exists('dPeriod', loop)
+    avg_interval = get_from_dict_if_exists('dAvgPeriodDiff', loop)
+
+    if interval is None or interval <= 0:
+        interval = avg_interval
+    else:
+        avg_interval_set = avg_interval is not None and avg_interval > 0
+
+        if round(avg_interval) != round(interval) and avg_interval_set:
+            message = ("Reported average frame interval (%.1f ms) doesn't"
+                       " match the set interval (%.1f ms). Using the average"
+                       " now.")
+            warnings.warn(message % (avg_interval, interval), RuntimeWarning)
+            interval = avg_interval
+
+    if interval is None or interval <= 0:
+        # In some cases, both keys are not saved. Then try to calculate it.
+        interval = guess_sampling_from_loops(duration, loop)
+
+    return interval

nd2reader/parser.py

@@ -265,8 +265,11 @@ class Parser(object):
         # The images for the various channels are interleaved within the same array. For example, the second image
         # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design
         # a data structure that way, please send the author of this library a message.
-        number_of_true_channels = int((len(image_group_data) - 4) / (height * width))
-        image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width))
+        number_of_true_channels = int(len(image_group_data[4:]) / (height * width))
+        try:
+            image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width))
+        except ValueError:
+            image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, int(round(len(image_group_data[image_data_start::number_of_true_channels])/height))))
 
         # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you
         # don't have the same number of images each cycle. We discovered this because we only took GFP images every

nd2reader/raw_metadata.py

@@ -1,9 +1,11 @@
 import re
-from nd2reader.common import read_chunk, read_array, read_metadata, parse_date, get_from_dict_if_exists
 import xmltodict
 import six
 import numpy as np
 
+from nd2reader.common import read_chunk, read_array, read_metadata, parse_date, get_from_dict_if_exists
+from nd2reader.common_raw_metadata import parse_dimension_text_line, parse_if_not_none, parse_roi_shape, parse_roi_type, get_loops_from_data, determine_sampling_interval
+
 
 class RawMetadata(object):
     """RawMetadata class parses and stores the raw metadata that is read from the binary file in dict format.
@@ -35,15 +37,15 @@ class RawMetadata(object):
 
         frames_per_channel = self._parse_total_images_per_channel()
         self._metadata_parsed = {
-            "height": self._parse_if_not_none(self.image_attributes, self._parse_height),
-            "width": self._parse_if_not_none(self.image_attributes, self._parse_width),
-            "date": self._parse_if_not_none(self.image_text_info, self._parse_date),
+            "height": parse_if_not_none(self.image_attributes, self._parse_height),
+            "width": parse_if_not_none(self.image_attributes, self._parse_width),
+            "date": parse_if_not_none(self.image_text_info, self._parse_date),
             "fields_of_view": self._parse_fields_of_view(),
             "frames": self._parse_frames(),
             "z_levels": self._parse_z_levels(),
             "total_images_per_channel": frames_per_channel,
             "channels": self._parse_channels(),
-            "pixel_microns": self._parse_if_not_none(self.image_calibration, self._parse_calibration),
+            "pixel_microns": parse_if_not_none(self.image_calibration, self._parse_calibration),
         }
 
         self._set_default_if_not_empty('fields_of_view')
@@ -63,12 +65,6 @@ class RawMetadata(object):
             # if the file is not empty, we always have one of this entry
             self._metadata_parsed[entry] = [0]
 
-    @staticmethod
-    def _parse_if_not_none(to_check, callback):
-        if to_check is not None:
-            return callback()
-        return None
-
     def _parse_width_or_height(self, key):
         try:
             length = self.image_attributes[six.b('SLxImageAttributes')][six.b(key)]
@@ -180,21 +176,12 @@ class RawMetadata(object):
             return dimension_text
 
         for line in textinfo:
-            entry = self._parse_dimension_text_line(line)
+            entry = parse_dimension_text_line(line)
             if entry is not None:
                 return entry
 
         return dimension_text
 
-    @staticmethod
-    def _parse_dimension_text_line(line):
-        if six.b("Dimensions:") in line:
-            entries = line.split(six.b("\r\n"))
-            for entry in entries:
-                if entry.startswith(six.b("Dimensions:")):
-                    return entry
-        return None
-
     def _parse_dimension(self, pattern):
         dimension_text = self._parse_dimension_text()
         if dimension_text is None:
@@ -261,8 +248,8 @@ class RawMetadata(object):
             "timepoints": [],
             "positions": [],
             "sizes": [],
-            "shape": self._parse_roi_shape(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiShapeType')]),
-            "type": self._parse_roi_type(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiInterpType')])
+            "shape": parse_roi_shape(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiShapeType')]),
+            "type": parse_roi_type(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiInterpType')])
         }
         for i in range(number_of_timepoints):
             roi_dict = self._parse_vect_anim(roi_dict, raw_roi_dict[six.b('m_vectAnimParams_%d' % i)])
@@ -274,26 +261,6 @@ class RawMetadata(object):
 
         return roi_dict
 
-    @staticmethod
-    def _parse_roi_shape(shape):
-        if shape == 3:
-            return 'rectangle'
-        elif shape == 9:
-            return 'circle'
-
-        return None
-
-    @staticmethod
-    def _parse_roi_type(type_no):
-        if type_no == 4:
-            return 'stimulation'
-        elif type_no == 3:
-            return 'reference'
-        elif type_no == 2:
-            return 'background'
-
-        return None
-
     def _parse_vect_anim(self, roi_dict, animation_dict):
         """
         Parses a ROI vector animation object and adds it to the global list of timepoints and positions.
@@ -345,18 +312,6 @@ class RawMetadata(object):
         if six.b('uLoopPars') in raw_data:
             self._metadata_parsed['experiment']['loops'] = self._parse_loop_data(raw_data[six.b('uLoopPars')])
 
-    @staticmethod
-    def _get_loops_from_data(loop_data):
-        loops = [loop_data]
-        if six.b('uiPeriodCount') in loop_data and loop_data[six.b('uiPeriodCount')] > 0:
-            # special ND experiment
-            if six.b('pPeriod') not in loop_data:
-                return []
-
-            # take the first dictionary element, it contains all loop data
-            loops = loop_data[six.b('pPeriod')][list(loop_data[six.b('pPeriod')].keys())[0]]
-        return loops
-
     def _parse_loop_data(self, loop_data):
         """Parse the experimental loop data
 
@@ -367,7 +322,7 @@ class RawMetadata(object):
             list: list of the parsed loops
 
         """
-        loops = self._get_loops_from_data(loop_data)
+        loops = get_loops_from_data(loop_data)
 
         # take into account the absolute time in ms
         time_offset = 0
@@ -377,7 +332,7 @@ class RawMetadata(object):
         for loop in loops:
             # duration of this loop
             duration = get_from_dict_if_exists('dDuration', loop) or 0
-            interval = self._determine_sampling_interval(duration, loop)
+            interval = determine_sampling_interval(duration, loop)
 
             # if duration is not saved, infer it
             duration = self.get_duration_from_interval_and_loops(duration, interval, loop)
@@ -418,43 +373,6 @@ class RawMetadata(object):
 
         return duration
 
-    @staticmethod
-    def _determine_sampling_interval(duration, loop):
-        """Determines the loop sampling interval in milliseconds
-
-        Args:
-            duration: loop duration in milliseconds
-            loop: loop dictionary
-
-        Returns:
-            float: the sampling interval in milliseconds
-
-        """
-        interval = get_from_dict_if_exists('dPeriod', loop)
-        if interval is None or interval <= 0:
-            # Use a fallback if it is still not found
-            interval = get_from_dict_if_exists('dAvgPeriodDiff', loop)
-        if interval is None or interval <= 0:
-            # In some cases, both keys are not saved. Then try to calculate it.
-            interval = RawMetadata._guess_sampling_from_loops(duration, loop)
-        return interval
-
-    @staticmethod
-    def _guess_sampling_from_loops(duration, loop):
-        """ In some cases, both keys are not saved. Then try to calculate it.
-        
-        Args:
-            duration: the total duration of the loop
-            loop: the raw loop data
-
-        Returns:
-            float: the guessed sampling interval in milliseconds
-
-        """
-        number_of_loops = get_from_dict_if_exists('uiCount', loop)
-        number_of_loops = number_of_loops if number_of_loops is not None and number_of_loops > 0 else 1
-        interval = duration / number_of_loops
-        return interval
 
     @property
     def image_text_info(self):

nd2reader/reader.py

@@ -185,26 +185,9 @@ class ND2Reader(FramesSequenceND):
             np.ndarray: an array of times in milliseconds.
 
         """
-        if self._timesteps is not None and len(timesteps) > 0:
+        if self._timesteps is not None and len(self._timesteps) > 0:
             return self._timesteps
 
-        timesteps = np.array([])
-        current_time = 0.0
-
-        for loop in self.metadata['experiment']['loops']:
-            if loop['stimulation']:
-                continue
-
-            if loop['sampling_interval'] == 0:
-                # This is a loop were no data is acquired
-                current_time += loop['duration']
-                continue
-
-            timesteps = np.concatenate(
-                (timesteps, np.arange(current_time, current_time + loop['duration'], loop['sampling_interval'])))
-            current_time += loop['duration']
-
-        # if experiment did not finish, number of timesteps is wrong. Take correct amount of leading timesteps.
-        self._timesteps = timesteps[:self.metadata['num_frames']]
+        self._timesteps = np.array(list(self._parser._raw_metadata.acquisition_times), dtype=np.float) * 1000.0
 
         return self._timesteps

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 
-VERSION = '3.0.9'
+VERSION = '3.1.0'
 
 if __name__ == '__main__':
     setup(

sphinx/conf.py

@@ -44,9 +44,9 @@ author = 'Ruben Verweij'
 # built documents.
 #
 # The short X.Y version.
-version = '3.0.9'
+version = '3.1.0'
 # The full version, including alpha/beta/rc tags.
-release = '3.0.9'
+release = '3.1.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

tests/test_raw_metadata.py

@@ -4,6 +4,7 @@ import six
 from nd2reader.artificial import ArtificialND2
 from nd2reader.label_map import LabelMap
 from nd2reader.raw_metadata import RawMetadata
+from nd2reader.common_raw_metadata import parse_roi_shape, parse_roi_type, parse_dimension_text_line
 
 
 class TestRawMetadata(unittest.TestCase):
@@ -14,15 +15,20 @@ class TestRawMetadata(unittest.TestCase):
         self.metadata = RawMetadata(self.nd2.file_handle, self.label_map)
 
     def test_parse_roi_shape(self):
-        self.assertEqual(self.metadata._parse_roi_shape(3), 'rectangle')
-        self.assertEqual(self.metadata._parse_roi_shape(9), 'circle')
-        self.assertIsNone(self.metadata._parse_roi_shape(-1))
+        self.assertEqual(parse_roi_shape(3), 'rectangle')
+        self.assertEqual(parse_roi_shape(9), 'circle')
+        self.assertIsNone(parse_roi_shape(-1))
 
     def test_parse_roi_type(self):
-        self.assertEqual(self.metadata._parse_roi_type(3), 'reference')
-        self.assertEqual(self.metadata._parse_roi_type(2), 'background')
-        self.assertEqual(self.metadata._parse_roi_type(4), 'stimulation')
-        self.assertIsNone(self.metadata._parse_roi_type(-1))
+        self.assertEqual(parse_roi_type(3), 'reference')
+        self.assertEqual(parse_roi_type(2), 'background')
+        self.assertEqual(parse_roi_type(4), 'stimulation')
+        self.assertIsNone(parse_roi_type(-1))
+
+    def test_parse_dimension_text(self):
+        line = six.b('Metadata:\r\nDimensions: T(443) x \xce\xbb(1)\r\nCamera Name: Andor Zyla VSC-01537')
+        self.assertEqual(parse_dimension_text_line(line), six.b('Dimensions: T(443) x \xce\xbb(1)'))
+        self.assertIsNone(parse_dimension_text_line(six.b('Dim: nothing')))
 
     def test_dict(self):
         self.assertTrue(type(self.metadata.__dict__) is dict)