From ae00cc1223fe31ca6333715e8812eb64419c86f1 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 10 Sep 2019 17:53:25 -0400 Subject: [PATCH 1/8] fix to allow reading of ND2-files with missing frames, due to e.g. skipping z-positions in one channel of a multichannel image; based on most recent commit of master --- nd2reader/parser.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index de7e7c3..2bdcb58 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -283,7 +283,13 @@ class Parser(object): if np.any(image_data): return timestamp, Frame(image_data, metadata=self._get_frame_metadata()) - raise NoImageError + # If a blank "gap" image is encountered, generate an empty frame (black image) of corresponding height and width to avoid + # errors with ND2-files with missing frames. + else: + empty_frame = np.zeros([height, width]) + return timestamp, Frame(empty_frame, metadata=self._get_frame_metadata()) + + # raise NoImageError # <-- delete? def _get_frame_metadata(self): """Get the metadata for one frame From c92b78689d01f164b495b1511b0c86309320f131 Mon Sep 17 00:00:00 2001 From: Ruben Verweij Date: Thu, 19 Sep 2019 09:23:48 +0200 Subject: [PATCH 2/8] Fallback function for old get_frame_2D function --- nd2reader/reader.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nd2reader/reader.py b/nd2reader/reader.py index fe263d7..714dc0e 100644 --- a/nd2reader/reader.py +++ b/nd2reader/reader.py @@ -52,6 +52,11 @@ class ND2Reader(FramesSequenceND): except KeyError: return 0 + def get_frame_2D(self, c=0, t=0, z=0, x=0, y=0, v=0): + """Fallback function for backwards compatibility + """ + return get_frame_vczyx(v=v, c=c, t=t, z=z, x=x, y=y) + def get_frame_vczyx(self, v=None, c=None, t=None, z=None, x=None, y=None): x = self.metadata["width"] if x <= 0 else x y = self.metadata["height"] if y <= 0 else y From de1c2aff58db9370b79e6c018936bace5714014d Mon Sep 17 00:00:00 2001 From: Ruben Verweij Date: Thu, 19 Sep 2019 09:40:17 +0200 Subject: [PATCH 3/8] Fixes issue #16 --- nd2reader/raw_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nd2reader/raw_metadata.py b/nd2reader/raw_metadata.py index 66f3fb3..9adf88b 100644 --- a/nd2reader/raw_metadata.py +++ b/nd2reader/raw_metadata.py @@ -193,7 +193,7 @@ class RawMetadata(object): if not match: return [] count = int(match.group(1)) - return list(range(count)) + return range(count) def _parse_total_images_per_channel(self): """The total number of images per channel. From cd55b8ffddd5edb03fb429ecc8f85f9c2707403d Mon Sep 17 00:00:00 2001 From: Ruben Verweij Date: Thu, 19 Sep 2019 10:09:18 +0200 Subject: [PATCH 4/8] Fixes issue #11 --- nd2reader/raw_metadata.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/nd2reader/raw_metadata.py b/nd2reader/raw_metadata.py index 9adf88b..321136d 100644 --- a/nd2reader/raw_metadata.py +++ b/nd2reader/raw_metadata.py @@ -43,9 +43,10 @@ class RawMetadata(object): "fields_of_view": self._parse_fields_of_view(), "frames": self._parse_frames(), "z_levels": self._parse_z_levels(), + "z_coordinates": parse_if_not_none(self.z_data, self._parse_z_coordinates), "total_images_per_channel": frames_per_channel, "channels": self._parse_channels(), - "pixel_microns": parse_if_not_none(self.image_calibration, self._parse_calibration), + "pixel_microns": parse_if_not_none(self.image_calibration, self._parse_calibration) } self._set_default_if_not_empty('fields_of_view') @@ -161,6 +162,14 @@ class RawMetadata(object): """ return self._parse_dimension(r""".*?Z\((\d+)\).*?""") + def _parse_z_coordinates(self): + """The coordinate in micron for all z planes. + + Returns: + list: the z coordinates in micron + """ + return self.z_data.tolist() + def _parse_dimension_text(self): """While there are metadata values that represent a lot of what we want to capture, they seem to be unreliable. Sometimes certain elements don't exist, or change their data type randomly. However, the human-readable text @@ -492,7 +501,12 @@ class RawMetadata(object): Returns: dict: z_data """ - return read_array(self._fh, 'double', self._label_map.z_data) + try: + return read_array(self._fh, 'double', self._label_map.z_data) + except ValueError: + # Depending on the file format/exact settings, this value is + # sometimes saved as float instead of double + return read_array(self._fh, 'float', self._label_map.z_data) @property def roi_metadata(self): From 9e0358882c0a6da6e267a50bbeb75e569f403099 Mon Sep 17 00:00:00 2001 From: Ruben Verweij Date: Thu, 19 Sep 2019 11:48:31 +0200 Subject: [PATCH 5/8] Add acquisition times to parser --- nd2reader/parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index de7e7c3..5472535 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -140,6 +140,7 @@ class Parser(object): self._label_map = self._build_label_map() self._raw_metadata = RawMetadata(self._fh, self._label_map) self.metadata = self._raw_metadata.__dict__ + self.acquisition_times = self._raw_metadata.acquisition_times def _build_label_map(self): """ From e9cd68bd4402fe51f1dd7ff636371ad4e288271a Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 20 Sep 2019 14:03:30 -0400 Subject: [PATCH 6/8] removed NoImageError call, class def, and import references since now redundant --- nd2reader/exceptions.py | 11 ----------- nd2reader/parser.py | 4 +--- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/nd2reader/exceptions.py b/nd2reader/exceptions.py index 55c598b..cb1b1fb 100644 --- a/nd2reader/exceptions.py +++ b/nd2reader/exceptions.py @@ -6,17 +6,6 @@ class InvalidVersionError(Exception): """ pass - -class NoImageError(Exception): - """No image found. - - Some apparent images in ND2s are just completely blank placeholders. These are used when the number of images per - cycle are unequal (e.g. if you take fluorescent images every 2 minutes, and bright field images every minute). - - """ - pass - - class EmptyFileError(Exception): """This .nd2 file seems to be empty. diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 2bdcb58..746a79d 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -7,7 +7,7 @@ from pims.base_frames import Frame import numpy as np from nd2reader.common import get_version, read_chunk -from nd2reader.exceptions import InvalidVersionError, NoImageError +from nd2reader.exceptions import InvalidVersionError from nd2reader.label_map import LabelMap from nd2reader.raw_metadata import RawMetadata @@ -289,8 +289,6 @@ class Parser(object): empty_frame = np.zeros([height, width]) return timestamp, Frame(empty_frame, metadata=self._get_frame_metadata()) - # raise NoImageError # <-- delete? - def _get_frame_metadata(self): """Get the metadata for one frame From f07eab01601e1e4dee5075340e1c78883b58cf56 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Sep 2019 13:49:57 -0400 Subject: [PATCH 7/8] Handle gap frames with np.nan-filled array and issue warning about it. --- nd2reader/parser.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 746a79d..be22218 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -3,6 +3,7 @@ import struct import array import six +import warnings from pims.base_frames import Frame import numpy as np @@ -72,7 +73,7 @@ class Parser(object): try: timestamp, image = self._get_raw_image_data(image_group_number, channel_offset, self.metadata["height"], self.metadata["width"]) - except (TypeError, NoImageError): + except (TypeError): return Frame([], frame_no=frame_number, metadata=self._get_frame_metadata()) else: return image @@ -101,7 +102,7 @@ class Parser(object): try: timestamp, raw_image_data = self._get_raw_image_data(image_group_number, channel, height, width) - except (TypeError, NoImageError): + except (TypeError): return Frame([], frame_no=frame_number, metadata=self._get_frame_metadata()) else: return raw_image_data @@ -283,11 +284,12 @@ class Parser(object): if np.any(image_data): return timestamp, Frame(image_data, metadata=self._get_frame_metadata()) - # If a blank "gap" image is encountered, generate an empty frame (black image) of corresponding height and width to avoid - # errors with ND2-files with missing frames. - else: - empty_frame = np.zeros([height, width]) - return timestamp, Frame(empty_frame, metadata=self._get_frame_metadata()) + # If a blank "gap" image is encountered, generate an array of corresponding height and width to avoid + # errors with ND2-files with missing frames. Array is filled with nan to reflect that data is missing. + else: + empty_frame = np.full((height, width), np.nan) + warnings.warn('ND2 file contains gap frames which are represented by np.nan-filled arrays') + return timestamp, Frame(empty_frame, metadata=self._get_frame_metadata()) def _get_frame_metadata(self): """Get the metadata for one frame From d844144209dcfc81346e3ad0b155d9553339ec62 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Sep 2019 15:05:46 -0400 Subject: [PATCH 8/8] More informative warning if gap frames encountered and filled with np.nan --- nd2reader/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index be22218..4937c14 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -288,7 +288,7 @@ class Parser(object): # errors with ND2-files with missing frames. Array is filled with nan to reflect that data is missing. else: empty_frame = np.full((height, width), np.nan) - warnings.warn('ND2 file contains gap frames which are represented by np.nan-filled arrays') + warnings.warn('ND2 file contains gap frames which are represented by np.nan-filled arrays; to convert to zeros use e.g. np.nan_to_num(array)') return timestamp, Frame(empty_frame, metadata=self._get_frame_metadata()) def _get_frame_metadata(self):