diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 3c8c8a1..8534e79 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -8,9 +8,9 @@ from pims.base_frames import Frame import numpy as np from nd2reader.common import get_version, read_chunk -from nd2reader.exceptions import InvalidVersionError from nd2reader.label_map import LabelMap from nd2reader.raw_metadata import RawMetadata +from nd2reader import stitched class Parser(object): @@ -232,8 +232,7 @@ class Parser(object): Returns: """ - return (image_group_number - (field_of_view * len(self.metadata["z_levels"]) + z_level)) / ( - len(self.metadata["fields_of_view"]) * len(self.metadata["z_levels"])) + return (image_group_number - (field_of_view * len(self.metadata["z_levels"]) + z_level)) / (len(self.metadata["fields_of_view"]) * len(self.metadata["z_levels"])) @property def _channel_offset(self): @@ -268,6 +267,7 @@ class Parser(object): timestamp = struct.unpack("d", data[:8])[0] image_group_data = array.array("H", data) image_data_start = 4 + channel_offset + image_group_data = stitched.remove_parsed_unwanted_bytes(image_group_data, image_data_start, height, width) # The images for the various channels are interleaved within the same array. For example, the second image # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design @@ -276,7 +276,8 @@ class Parser(object): try: image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width)) except ValueError: - image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, int(round(len(image_group_data[image_data_start::number_of_true_channels])/height)))) + new_width = len(image_group_data[image_data_start::number_of_true_channels]) // height + image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, new_width)) # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you # don't have the same number of images each cycle. We discovered this because we only took GFP images every @@ -285,11 +286,12 @@ class Parser(object): if np.any(image_data): return timestamp, image_data - # If a blank "gap" image is encountered, generate an array of corresponding height and width to avoid - # errors with ND2-files with missing frames. Array is filled with nan to reflect that data is missing. + # If a blank "gap" image is encountered, generate an array of corresponding height and width to avoid + # errors with ND2-files with missing frames. Array is filled with nan to reflect that data is missing. else: empty_frame = np.full((height, width), np.nan) - warnings.warn('ND2 file contains gap frames which are represented by np.nan-filled arrays; to convert to zeros use e.g. np.nan_to_num(array)') + warnings.warn( + "ND2 file contains gap frames which are represented by np.nan-filled arrays; to convert to zeros use e.g. np.nan_to_num(array)") return timestamp, image_data def _get_frame_metadata(self): diff --git a/nd2reader/stitched.py b/nd2reader/stitched.py new file mode 100644 index 0000000..9721e21 --- /dev/null +++ b/nd2reader/stitched.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +import numpy as np # type: ignore +import warnings + + +def get_unwanted_bytes_ids(image_group_data, image_data_start, height, width): + # Check if the byte array size conforms to the image axes size. If not, check + # that the number of unexpected (unwanted) bytes is a multiple of the number of + # rows (height), as the same unmber of unwanted bytes is expected to be + # appended at the end of each row. Then, returns the indexes of the unwanted + # bytes. + number_of_true_channels = int(len(image_group_data[4:]) / (height * width)) + n_unwanted_bytes = (len(image_group_data[image_data_start:])) % (height * width) + if not n_unwanted_bytes: + return np.arange(0) + assert 0 == n_unwanted_bytes % height, ( + "An unexpected number of extra bytes was encountered based on the expected" + + " frame size, therefore the file could not be parsed." + ) + return np.arange( + image_data_start + height * number_of_true_channels, + len(image_group_data) - n_unwanted_bytes + 1, + height * number_of_true_channels, + ) + + +def remove_bytes_by_id(byte_ids, image_group_data, height): + # Remove bytes by ID. + bytes_per_row = len(byte_ids) // height + warnings.warn( + f"{len(byte_ids)} ({bytes_per_row}*{height}) unexpected zero " + + "bytes were found in the ND2 file and removed to allow further parsing." + ) + for i in range(len(byte_ids)): + del image_group_data[byte_ids[i] : (byte_ids[i] + bytes_per_row)] + + +def remove_parsed_unwanted_bytes(image_group_data, image_data_start, height, width): + # Stitched ND2 files have been reported to contain unexpected (according to + # image shape) zero bytes at the end of each image data row. This hinders + # proper reshaping of the data. Hence, here the unwanted zero bytes are + # identified and removed. + unwanted_byte_ids = get_unwanted_bytes_ids( + image_group_data, image_data_start, height, width + ) + if 0 != len(unwanted_byte_ids): + assert np.all( + image_group_data[unwanted_byte_ids + np.arange(len(unwanted_byte_ids))] == 0 + ), ( + f"{len(unwanted_byte_ids)} unexpected non-zero bytes were found" + + " in the ND2 file, the file could not be parsed." + ) + remove_bytes_by_id(unwanted_byte_ids, image_group_data, height) + return image_group_data diff --git a/tests/test_parser.py b/tests/test_parser.py index 11dba38..c0b4ddf 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2,8 +2,8 @@ import unittest from os import path from nd2reader.artificial import ArtificialND2 from nd2reader.common import check_or_make_dir -from nd2reader.exceptions import InvalidVersionError from nd2reader.parser import Parser +import urllib.request class TestParser(unittest.TestCase): @@ -13,15 +13,24 @@ class TestParser(unittest.TestCase): def setUp(self): dir_path = path.dirname(path.realpath(__file__)) - check_or_make_dir(path.join(dir_path, 'test_data/')) - self.test_file = path.join(dir_path, 'test_data/test.nd2') + check_or_make_dir(path.join(dir_path, "test_data/")) + self.test_file = path.join(dir_path, "test_data/test.nd2") self.create_test_nd2() def test_can_open_test_file(self): self.create_test_nd2() - with open(self.test_file, 'rb') as fh: + with open(self.test_file, "rb") as fh: parser = Parser(fh) self.assertTrue(parser.supported) - + def test_get_image(self): + stitched_path = "test_data/test_stitched.nd2" + if not path.isfile(stitched_path): + file_name, header = urllib.request.urlretrieve( + "https://downloads.openmicroscopy.org/images/ND2/karl/sample_image.nd2", + stitched_path, + ) + with open(stitched_path, "rb") as fh: + parser = Parser(fh) + parser.get_image(0)