diff --git a/docs b/docs index 1b43456..6a12543 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1b43456084689bad958984cf8a0cad7f63b52286 +Subproject commit 6a12543461144fc164942c79a0b1457fd6284779 diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index e29913c..d50c9b3 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -1,4 +1,4 @@ from nd2reader.reader import ND2Reader from nd2reader.legacy import Nd2 -__version__ = '3.0.9' +__version__ = '3.1.0' diff --git a/nd2reader/artificial.py b/nd2reader/artificial.py index df83c18..861c613 100644 --- a/nd2reader/artificial.py +++ b/nd2reader/artificial.py @@ -6,6 +6,27 @@ import struct from nd2reader.common import check_or_make_dir from os import path +global_labels = ['image_attributes', 'image_text_info', 'image_metadata', + 'image_metadata_sequence', 'image_calibration', 'x_data', + 'y_data', 'z_data', 'roi_metadata', 'pfs_status', 'pfs_offset', + 'guid', 'description', 'camera_exposure_time', 'camera_temp', + 'acquisition_times', 'acquisition_times_2', + 'acquisition_frames', 'lut_data', 'grabber_settings', + 'custom_data', 'app_info', 'image_frame_0'] + +global_file_labels = ["ImageAttributesLV!", "ImageTextInfoLV!", + "ImageMetadataLV!", "ImageMetadataSeqLV|0!", + "ImageCalibrationLV|0!", "CustomData|X!", "CustomData|Y!", + "CustomData|Z!", "CustomData|RoiMetadata_v1!", + "CustomData|PFS_STATUS!", "CustomData|PFS_OFFSET!", + "CustomData|GUIDStore!", "CustomData|CustomDescriptionV1_0!", + "CustomData|Camera_ExposureTime1!", "CustomData|CameraTemp1!", + "CustomData|AcqTimesCache!", "CustomData|AcqTimes2Cache!", + "CustomData|AcqFramesCache!", "CustomDataVar|LUTDataV1_0!", + "CustomDataVar|GrabberCameraSettingsV1_0!", + "CustomDataVar|CustomDataV2_0!", "CustomDataVar|AppInfo_V1_0!", + "ImageDataSeq|0!"] + class ArtificialND2(object): """Artificial ND2 class (for testing purposes) @@ -105,56 +126,8 @@ class ArtificialND2(object): """ raw_text = six.b('') - labels = [ - 'image_attributes', - 'image_text_info', - 'image_metadata', - 'image_metadata_sequence', - 'image_calibration', - 'x_data', - 'y_data', - 'z_data', - 'roi_metadata', - 'pfs_status', - 'pfs_offset', - 'guid', - 'description', - 'camera_exposure_time', - 'camera_temp', - 'acquisition_times', - 'acquisition_times_2', - 'acquisition_frames', - 'lut_data', - 'grabber_settings', - 'custom_data', - 'app_info', - 'image_frame_0' - ] - file_labels = [ - "ImageAttributesLV!", - "ImageTextInfoLV!", - "ImageMetadataLV!", - "ImageMetadataSeqLV|0!", - "ImageCalibrationLV|0!", - "CustomData|X!", - "CustomData|Y!", - "CustomData|Z!", - "CustomData|RoiMetadata_v1!", - "CustomData|PFS_STATUS!", - "CustomData|PFS_OFFSET!", - "CustomData|GUIDStore!", - "CustomData|CustomDescriptionV1_0!", - "CustomData|Camera_ExposureTime1!", - "CustomData|CameraTemp1!", - "CustomData|AcqTimesCache!", - "CustomData|AcqTimes2Cache!", - "CustomData|AcqFramesCache!", - "CustomDataVar|LUTDataV1_0!", - "CustomDataVar|GrabberCameraSettingsV1_0!", - "CustomDataVar|CustomDataV2_0!", - "CustomDataVar|AppInfo_V1_0!", - "ImageDataSeq|0!" - ] + labels = global_labels + file_labels = global_file_labels file_data, file_data_dict = self._get_file_data(labels) @@ -242,42 +215,42 @@ class ArtificialND2(object): return raw_data + @staticmethod + def _get_slx_img_attrib(): + return {'uiWidth': 128, + 'uiWidthBytes': 256, + 'uiHeight': 128, + 'uiComp': 1, + 'uiBpcInMemory': 16, + 'uiBpcSignificant': 12, + 'uiSequenceCount': 70, + 'uiTileWidth': 128, + 'uiTileHeight': 128, + 'eCompression': 2, + 'dCompressionParam': -1.0, + 'ePixelType': 1, + 'uiVirtualComponents': 1 + } + + @staticmethod + def _get_slx_picture_metadata(): + return {'sPicturePlanes': + { + 'sPlaneNew': { + # channels are numbered a0, a1, ..., aN + 'a0': { + 'sDescription': 'TRITC' + } + } + } + } + def _get_file_data(self, labels): file_data = [ - { - 'SLxImageAttributes': - { - 'uiWidth': 128, - 'uiWidthBytes': 256, - 'uiHeight': 128, - 'uiComp': 1, - 'uiBpcInMemory': 16, - 'uiBpcSignificant': 12, - 'uiSequenceCount': 70, - 'uiTileWidth': 128, - 'uiTileHeight': 128, - 'eCompression': 2, - 'dCompressionParam': -1.0, - 'ePixelType': 1, - 'uiVirtualComponents': 1 - } - }, # ImageAttributesLV!", + {'SLxImageAttributes': self._get_slx_img_attrib()}, # ImageAttributesLV!", 7, # ImageTextInfoLV!", 7, # ImageMetadataLV!", - { - 'SLxPictureMetadata': - { - 'sPicturePlanes': - { - 'sPlaneNew': { - # channels are numbered a0, a1, ..., aN - 'a0': { - 'sDescription': 'TRITC' - } - } - } - } - }, # ImageMetadataSeqLV|0!", + {'SLxPictureMetadata': self._get_slx_picture_metadata()}, # ImageMetadataSeqLV|0!", 7, # ImageCalibrationLV|0!", 7, # CustomData|X!", 7, # CustomData|Y!", @@ -289,9 +262,9 @@ class ArtificialND2(object): 7, # CustomData|CustomDescriptionV1_0!", 7, # CustomData|Camera_ExposureTime1!", 7, # CustomData|CameraTemp1!", - 7, # CustomData|AcqTimesCache!", - 7, # CustomData|AcqTimes2Cache!", - 7, # CustomData|AcqFramesCache!", + [0], # CustomData|AcqTimesCache!", + [0], # CustomData|AcqTimes2Cache!", + [0], # CustomData|AcqFramesCache!", 7, # CustomDataVar|LUTDataV1_0!", 7, # CustomDataVar|GrabberCameraSettingsV1_0!", 7, # CustomDataVar|CustomDataV2_0!", diff --git a/nd2reader/common_raw_metadata.py b/nd2reader/common_raw_metadata.py new file mode 100644 index 0000000..a195563 --- /dev/null +++ b/nd2reader/common_raw_metadata.py @@ -0,0 +1,101 @@ +import six +import warnings + +from nd2reader.common import get_from_dict_if_exists + + +def parse_if_not_none(to_check, callback): + if to_check is not None: + return callback() + return None + + +def parse_dimension_text_line(line): + if six.b("Dimensions:") in line: + entries = line.split(six.b("\r\n")) + for entry in entries: + if entry.startswith(six.b("Dimensions:")): + return entry + return None + + +def parse_roi_shape(shape): + if shape == 3: + return 'rectangle' + elif shape == 9: + return 'circle' + + return None + + +def parse_roi_type(type_no): + if type_no == 4: + return 'stimulation' + elif type_no == 3: + return 'reference' + elif type_no == 2: + return 'background' + + return None + + +def get_loops_from_data(loop_data): + loops = [loop_data] + if six.b('uiPeriodCount') in loop_data and loop_data[six.b('uiPeriodCount')] > 0: + # special ND experiment + if six.b('pPeriod') not in loop_data: + return [] + + # take the first dictionary element, it contains all loop data + loops = loop_data[six.b('pPeriod')][list(loop_data[six.b('pPeriod')].keys())[0]] + return loops + + +def guess_sampling_from_loops(duration, loop): + """ In some cases, both keys are not saved. Then try to calculate it. + + Args: + duration: the total duration of the loop + loop: the raw loop data + + Returns: + float: the guessed sampling interval in milliseconds + + """ + number_of_loops = get_from_dict_if_exists('uiCount', loop) + number_of_loops = number_of_loops if number_of_loops is not None and number_of_loops > 0 else 1 + interval = duration / number_of_loops + return interval + + +def determine_sampling_interval(duration, loop): + """Determines the loop sampling interval in milliseconds + + Args: + duration: loop duration in milliseconds + loop: loop dictionary + + Returns: + float: the sampling interval in milliseconds + + """ + interval = get_from_dict_if_exists('dPeriod', loop) + avg_interval = get_from_dict_if_exists('dAvgPeriodDiff', loop) + + if interval is None or interval <= 0: + interval = avg_interval + else: + avg_interval_set = avg_interval is not None and avg_interval > 0 + + if round(avg_interval) != round(interval) and avg_interval_set: + message = ("Reported average frame interval (%.1f ms) doesn't" + " match the set interval (%.1f ms). Using the average" + " now.") + warnings.warn(message % (avg_interval, interval), RuntimeWarning) + interval = avg_interval + + if interval is None or interval <= 0: + # In some cases, both keys are not saved. Then try to calculate it. + interval = guess_sampling_from_loops(duration, loop) + + return interval diff --git a/nd2reader/parser.py b/nd2reader/parser.py index c413676..8a71b0b 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -265,8 +265,11 @@ class Parser(object): # The images for the various channels are interleaved within the same array. For example, the second image # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design # a data structure that way, please send the author of this library a message. - number_of_true_channels = int((len(image_group_data) - 4) / (height * width)) - image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width)) + number_of_true_channels = int(len(image_group_data[4:]) / (height * width)) + try: + image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width)) + except ValueError: + image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, int(round(len(image_group_data[image_data_start::number_of_true_channels])/height)))) # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you # don't have the same number of images each cycle. We discovered this because we only took GFP images every diff --git a/nd2reader/raw_metadata.py b/nd2reader/raw_metadata.py index b0d3485..6d9918e 100644 --- a/nd2reader/raw_metadata.py +++ b/nd2reader/raw_metadata.py @@ -1,9 +1,11 @@ import re -from nd2reader.common import read_chunk, read_array, read_metadata, parse_date, get_from_dict_if_exists import xmltodict import six import numpy as np +from nd2reader.common import read_chunk, read_array, read_metadata, parse_date, get_from_dict_if_exists +from nd2reader.common_raw_metadata import parse_dimension_text_line, parse_if_not_none, parse_roi_shape, parse_roi_type, get_loops_from_data, determine_sampling_interval + class RawMetadata(object): """RawMetadata class parses and stores the raw metadata that is read from the binary file in dict format. @@ -35,15 +37,15 @@ class RawMetadata(object): frames_per_channel = self._parse_total_images_per_channel() self._metadata_parsed = { - "height": self._parse_if_not_none(self.image_attributes, self._parse_height), - "width": self._parse_if_not_none(self.image_attributes, self._parse_width), - "date": self._parse_if_not_none(self.image_text_info, self._parse_date), + "height": parse_if_not_none(self.image_attributes, self._parse_height), + "width": parse_if_not_none(self.image_attributes, self._parse_width), + "date": parse_if_not_none(self.image_text_info, self._parse_date), "fields_of_view": self._parse_fields_of_view(), "frames": self._parse_frames(), "z_levels": self._parse_z_levels(), "total_images_per_channel": frames_per_channel, "channels": self._parse_channels(), - "pixel_microns": self._parse_if_not_none(self.image_calibration, self._parse_calibration), + "pixel_microns": parse_if_not_none(self.image_calibration, self._parse_calibration), } self._set_default_if_not_empty('fields_of_view') @@ -63,12 +65,6 @@ class RawMetadata(object): # if the file is not empty, we always have one of this entry self._metadata_parsed[entry] = [0] - @staticmethod - def _parse_if_not_none(to_check, callback): - if to_check is not None: - return callback() - return None - def _parse_width_or_height(self, key): try: length = self.image_attributes[six.b('SLxImageAttributes')][six.b(key)] @@ -180,21 +176,12 @@ class RawMetadata(object): return dimension_text for line in textinfo: - entry = self._parse_dimension_text_line(line) + entry = parse_dimension_text_line(line) if entry is not None: return entry return dimension_text - @staticmethod - def _parse_dimension_text_line(line): - if six.b("Dimensions:") in line: - entries = line.split(six.b("\r\n")) - for entry in entries: - if entry.startswith(six.b("Dimensions:")): - return entry - return None - def _parse_dimension(self, pattern): dimension_text = self._parse_dimension_text() if dimension_text is None: @@ -261,8 +248,8 @@ class RawMetadata(object): "timepoints": [], "positions": [], "sizes": [], - "shape": self._parse_roi_shape(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiShapeType')]), - "type": self._parse_roi_type(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiInterpType')]) + "shape": parse_roi_shape(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiShapeType')]), + "type": parse_roi_type(raw_roi_dict[six.b('m_sInfo')][six.b('m_uiInterpType')]) } for i in range(number_of_timepoints): roi_dict = self._parse_vect_anim(roi_dict, raw_roi_dict[six.b('m_vectAnimParams_%d' % i)]) @@ -274,26 +261,6 @@ class RawMetadata(object): return roi_dict - @staticmethod - def _parse_roi_shape(shape): - if shape == 3: - return 'rectangle' - elif shape == 9: - return 'circle' - - return None - - @staticmethod - def _parse_roi_type(type_no): - if type_no == 4: - return 'stimulation' - elif type_no == 3: - return 'reference' - elif type_no == 2: - return 'background' - - return None - def _parse_vect_anim(self, roi_dict, animation_dict): """ Parses a ROI vector animation object and adds it to the global list of timepoints and positions. @@ -345,18 +312,6 @@ class RawMetadata(object): if six.b('uLoopPars') in raw_data: self._metadata_parsed['experiment']['loops'] = self._parse_loop_data(raw_data[six.b('uLoopPars')]) - @staticmethod - def _get_loops_from_data(loop_data): - loops = [loop_data] - if six.b('uiPeriodCount') in loop_data and loop_data[six.b('uiPeriodCount')] > 0: - # special ND experiment - if six.b('pPeriod') not in loop_data: - return [] - - # take the first dictionary element, it contains all loop data - loops = loop_data[six.b('pPeriod')][list(loop_data[six.b('pPeriod')].keys())[0]] - return loops - def _parse_loop_data(self, loop_data): """Parse the experimental loop data @@ -367,7 +322,7 @@ class RawMetadata(object): list: list of the parsed loops """ - loops = self._get_loops_from_data(loop_data) + loops = get_loops_from_data(loop_data) # take into account the absolute time in ms time_offset = 0 @@ -377,7 +332,7 @@ class RawMetadata(object): for loop in loops: # duration of this loop duration = get_from_dict_if_exists('dDuration', loop) or 0 - interval = self._determine_sampling_interval(duration, loop) + interval = determine_sampling_interval(duration, loop) # if duration is not saved, infer it duration = self.get_duration_from_interval_and_loops(duration, interval, loop) @@ -418,43 +373,6 @@ class RawMetadata(object): return duration - @staticmethod - def _determine_sampling_interval(duration, loop): - """Determines the loop sampling interval in milliseconds - - Args: - duration: loop duration in milliseconds - loop: loop dictionary - - Returns: - float: the sampling interval in milliseconds - - """ - interval = get_from_dict_if_exists('dPeriod', loop) - if interval is None or interval <= 0: - # Use a fallback if it is still not found - interval = get_from_dict_if_exists('dAvgPeriodDiff', loop) - if interval is None or interval <= 0: - # In some cases, both keys are not saved. Then try to calculate it. - interval = RawMetadata._guess_sampling_from_loops(duration, loop) - return interval - - @staticmethod - def _guess_sampling_from_loops(duration, loop): - """ In some cases, both keys are not saved. Then try to calculate it. - - Args: - duration: the total duration of the loop - loop: the raw loop data - - Returns: - float: the guessed sampling interval in milliseconds - - """ - number_of_loops = get_from_dict_if_exists('uiCount', loop) - number_of_loops = number_of_loops if number_of_loops is not None and number_of_loops > 0 else 1 - interval = duration / number_of_loops - return interval @property def image_text_info(self): diff --git a/nd2reader/reader.py b/nd2reader/reader.py index 4f3acf2..74d99b6 100644 --- a/nd2reader/reader.py +++ b/nd2reader/reader.py @@ -185,26 +185,9 @@ class ND2Reader(FramesSequenceND): np.ndarray: an array of times in milliseconds. """ - if self._timesteps is not None and len(timesteps) > 0: + if self._timesteps is not None and len(self._timesteps) > 0: return self._timesteps - timesteps = np.array([]) - current_time = 0.0 - - for loop in self.metadata['experiment']['loops']: - if loop['stimulation']: - continue - - if loop['sampling_interval'] == 0: - # This is a loop were no data is acquired - current_time += loop['duration'] - continue - - timesteps = np.concatenate( - (timesteps, np.arange(current_time, current_time + loop['duration'], loop['sampling_interval']))) - current_time += loop['duration'] - - # if experiment did not finish, number of timesteps is wrong. Take correct amount of leading timesteps. - self._timesteps = timesteps[:self.metadata['num_frames']] + self._timesteps = np.array(list(self._parser._raw_metadata.acquisition_times), dtype=np.float) * 1000.0 return self._timesteps diff --git a/setup.py b/setup.py index 5fb6487..c56a6ef 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -VERSION = '3.0.9' +VERSION = '3.1.0' if __name__ == '__main__': setup( diff --git a/sphinx/conf.py b/sphinx/conf.py index 89e9f3f..623711e 100644 --- a/sphinx/conf.py +++ b/sphinx/conf.py @@ -44,9 +44,9 @@ author = 'Ruben Verweij' # built documents. # # The short X.Y version. -version = '3.0.9' +version = '3.1.0' # The full version, including alpha/beta/rc tags. -release = '3.0.9' +release = '3.1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/tests/test_raw_metadata.py b/tests/test_raw_metadata.py index 2ff8f8d..914acd8 100644 --- a/tests/test_raw_metadata.py +++ b/tests/test_raw_metadata.py @@ -4,6 +4,7 @@ import six from nd2reader.artificial import ArtificialND2 from nd2reader.label_map import LabelMap from nd2reader.raw_metadata import RawMetadata +from nd2reader.common_raw_metadata import parse_roi_shape, parse_roi_type, parse_dimension_text_line class TestRawMetadata(unittest.TestCase): @@ -14,15 +15,20 @@ class TestRawMetadata(unittest.TestCase): self.metadata = RawMetadata(self.nd2.file_handle, self.label_map) def test_parse_roi_shape(self): - self.assertEqual(self.metadata._parse_roi_shape(3), 'rectangle') - self.assertEqual(self.metadata._parse_roi_shape(9), 'circle') - self.assertIsNone(self.metadata._parse_roi_shape(-1)) + self.assertEqual(parse_roi_shape(3), 'rectangle') + self.assertEqual(parse_roi_shape(9), 'circle') + self.assertIsNone(parse_roi_shape(-1)) def test_parse_roi_type(self): - self.assertEqual(self.metadata._parse_roi_type(3), 'reference') - self.assertEqual(self.metadata._parse_roi_type(2), 'background') - self.assertEqual(self.metadata._parse_roi_type(4), 'stimulation') - self.assertIsNone(self.metadata._parse_roi_type(-1)) + self.assertEqual(parse_roi_type(3), 'reference') + self.assertEqual(parse_roi_type(2), 'background') + self.assertEqual(parse_roi_type(4), 'stimulation') + self.assertIsNone(parse_roi_type(-1)) + + def test_parse_dimension_text(self): + line = six.b('Metadata:\r\nDimensions: T(443) x \xce\xbb(1)\r\nCamera Name: Andor Zyla VSC-01537') + self.assertEqual(parse_dimension_text_line(line), six.b('Dimensions: T(443) x \xce\xbb(1)')) + self.assertIsNone(parse_dimension_text_line(six.b('Dim: nothing'))) def test_dict(self): self.assertTrue(type(self.metadata.__dict__) is dict)