Browse Source

Merge pull request #40 from ggirelli/ggirelli-patch-parser-for-stitched-images

Removed empty bytes at channel subchunk ends for issue #31
master
Ruben Verweij 4 years ago
committed by GitHub
parent
commit
18527bf9b1
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 77 additions and 12 deletions
  1. +9
    -7
      nd2reader/parser.py
  2. +54
    -0
      nd2reader/stitched.py
  3. +14
    -5
      tests/test_parser.py

+ 9
- 7
nd2reader/parser.py View File

@ -8,9 +8,9 @@ from pims.base_frames import Frame
import numpy as np import numpy as np
from nd2reader.common import get_version, read_chunk from nd2reader.common import get_version, read_chunk
from nd2reader.exceptions import InvalidVersionError
from nd2reader.label_map import LabelMap from nd2reader.label_map import LabelMap
from nd2reader.raw_metadata import RawMetadata from nd2reader.raw_metadata import RawMetadata
from nd2reader import stitched
class Parser(object): class Parser(object):
@ -232,8 +232,7 @@ class Parser(object):
Returns: Returns:
""" """
return (image_group_number - (field_of_view * len(self.metadata["z_levels"]) + z_level)) / (
len(self.metadata["fields_of_view"]) * len(self.metadata["z_levels"]))
return (image_group_number - (field_of_view * len(self.metadata["z_levels"]) + z_level)) / (len(self.metadata["fields_of_view"]) * len(self.metadata["z_levels"]))
@property @property
def _channel_offset(self): def _channel_offset(self):
@ -268,6 +267,7 @@ class Parser(object):
timestamp = struct.unpack("d", data[:8])[0] timestamp = struct.unpack("d", data[:8])[0]
image_group_data = array.array("H", data) image_group_data = array.array("H", data)
image_data_start = 4 + channel_offset image_data_start = 4 + channel_offset
image_group_data = stitched.remove_parsed_unwanted_bytes(image_group_data, image_data_start, height, width)
# The images for the various channels are interleaved within the same array. For example, the second image # The images for the various channels are interleaved within the same array. For example, the second image
# of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design
@ -276,7 +276,8 @@ class Parser(object):
try: try:
image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width)) image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, width))
except ValueError: except ValueError:
image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, int(round(len(image_group_data[image_data_start::number_of_true_channels])/height))))
new_width = len(image_group_data[image_data_start::number_of_true_channels]) // height
image_data = np.reshape(image_group_data[image_data_start::number_of_true_channels], (height, new_width))
# Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you
# don't have the same number of images each cycle. We discovered this because we only took GFP images every # don't have the same number of images each cycle. We discovered this because we only took GFP images every
@ -285,11 +286,12 @@ class Parser(object):
if np.any(image_data): if np.any(image_data):
return timestamp, image_data return timestamp, image_data
# If a blank "gap" image is encountered, generate an array of corresponding height and width to avoid
# errors with ND2-files with missing frames. Array is filled with nan to reflect that data is missing.
# If a blank "gap" image is encountered, generate an array of corresponding height and width to avoid
# errors with ND2-files with missing frames. Array is filled with nan to reflect that data is missing.
else: else:
empty_frame = np.full((height, width), np.nan) empty_frame = np.full((height, width), np.nan)
warnings.warn('ND2 file contains gap frames which are represented by np.nan-filled arrays; to convert to zeros use e.g. np.nan_to_num(array)')
warnings.warn(
"ND2 file contains gap frames which are represented by np.nan-filled arrays; to convert to zeros use e.g. np.nan_to_num(array)")
return timestamp, image_data return timestamp, image_data
def _get_frame_metadata(self): def _get_frame_metadata(self):


+ 54
- 0
nd2reader/stitched.py View File

@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
import numpy as np # type: ignore
import warnings
def get_unwanted_bytes_ids(image_group_data, image_data_start, height, width):
# Check if the byte array size conforms to the image axes size. If not, check
# that the number of unexpected (unwanted) bytes is a multiple of the number of
# rows (height), as the same unmber of unwanted bytes is expected to be
# appended at the end of each row. Then, returns the indexes of the unwanted
# bytes.
number_of_true_channels = int(len(image_group_data[4:]) / (height * width))
n_unwanted_bytes = (len(image_group_data[image_data_start:])) % (height * width)
if not n_unwanted_bytes:
return np.arange(0)
assert 0 == n_unwanted_bytes % height, (
"An unexpected number of extra bytes was encountered based on the expected"
+ " frame size, therefore the file could not be parsed."
)
return np.arange(
image_data_start + height * number_of_true_channels,
len(image_group_data) - n_unwanted_bytes + 1,
height * number_of_true_channels,
)
def remove_bytes_by_id(byte_ids, image_group_data, height):
# Remove bytes by ID.
bytes_per_row = len(byte_ids) // height
warnings.warn(
f"{len(byte_ids)} ({bytes_per_row}*{height}) unexpected zero "
+ "bytes were found in the ND2 file and removed to allow further parsing."
)
for i in range(len(byte_ids)):
del image_group_data[byte_ids[i] : (byte_ids[i] + bytes_per_row)]
def remove_parsed_unwanted_bytes(image_group_data, image_data_start, height, width):
# Stitched ND2 files have been reported to contain unexpected (according to
# image shape) zero bytes at the end of each image data row. This hinders
# proper reshaping of the data. Hence, here the unwanted zero bytes are
# identified and removed.
unwanted_byte_ids = get_unwanted_bytes_ids(
image_group_data, image_data_start, height, width
)
if 0 != len(unwanted_byte_ids):
assert np.all(
image_group_data[unwanted_byte_ids + np.arange(len(unwanted_byte_ids))] == 0
), (
f"{len(unwanted_byte_ids)} unexpected non-zero bytes were found"
+ " in the ND2 file, the file could not be parsed."
)
remove_bytes_by_id(unwanted_byte_ids, image_group_data, height)
return image_group_data

+ 14
- 5
tests/test_parser.py View File

@ -2,8 +2,8 @@ import unittest
from os import path from os import path
from nd2reader.artificial import ArtificialND2 from nd2reader.artificial import ArtificialND2
from nd2reader.common import check_or_make_dir from nd2reader.common import check_or_make_dir
from nd2reader.exceptions import InvalidVersionError
from nd2reader.parser import Parser from nd2reader.parser import Parser
import urllib.request
class TestParser(unittest.TestCase): class TestParser(unittest.TestCase):
@ -13,15 +13,24 @@ class TestParser(unittest.TestCase):
def setUp(self): def setUp(self):
dir_path = path.dirname(path.realpath(__file__)) dir_path = path.dirname(path.realpath(__file__))
check_or_make_dir(path.join(dir_path, 'test_data/'))
self.test_file = path.join(dir_path, 'test_data/test.nd2')
check_or_make_dir(path.join(dir_path, "test_data/"))
self.test_file = path.join(dir_path, "test_data/test.nd2")
self.create_test_nd2() self.create_test_nd2()
def test_can_open_test_file(self): def test_can_open_test_file(self):
self.create_test_nd2() self.create_test_nd2()
with open(self.test_file, 'rb') as fh:
with open(self.test_file, "rb") as fh:
parser = Parser(fh) parser = Parser(fh)
self.assertTrue(parser.supported) self.assertTrue(parser.supported)
def test_get_image(self):
stitched_path = "test_data/test_stitched.nd2"
if not path.isfile(stitched_path):
file_name, header = urllib.request.urlretrieve(
"https://downloads.openmicroscopy.org/images/ND2/karl/sample_image.nd2",
stitched_path,
)
with open(stitched_path, "rb") as fh:
parser = Parser(fh)
parser.get_image(0)

Loading…
Cancel
Save