From e141b0504cbb616b89ec516983a9e8339b5dd956 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Fri, 4 Dec 2015 00:15:43 -0600 Subject: [PATCH 1/2] #114: wrote fast filter that works, but we need way more testing for corner cases --- Makefile | 4 ++-- functional_tests/FYLM141111001.py | 14 ++++++++++++++ nd2reader/interface.py | 25 +++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3ddd695..b2301d6 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ build: docker build -t jimrybarski/nd2reader . shell: - xhost local:root; docker run --rm -v ~/nd2s:/var/nd2s -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -it jimrybarski/nd2reader bash + xhost local:root; docker run --rm -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -it jimrybarski/nd2reader bash py2: xhost local:root; docker run --rm -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -it jimrybarski/nd2reader python2.7 @@ -30,7 +30,7 @@ test: build docker run --rm -v $(CURDIR):/opt/nd2reader -it jimrybarski/nd2reader python2.7 test.py ftest: build - docker run --rm -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -it jimrybarski/nd2reader python3.4 /opt/nd2reader/ftest.py + xhost local:root; docker run --rm -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -it jimrybarski/nd2reader python3.4 /opt/nd2reader/ftest.py docker run --rm -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -it jimrybarski/nd2reader python2.7 /opt/nd2reader/ftest.py publish: diff --git a/functional_tests/FYLM141111001.py b/functional_tests/FYLM141111001.py index 9d1d6df..ad1ec59 100644 --- a/functional_tests/FYLM141111001.py +++ b/functional_tests/FYLM141111001.py @@ -4,6 +4,7 @@ run them unless you're Jim Rybarski. """ from nd2reader import Nd2 +import numpy as np from datetime import datetime import unittest @@ -109,3 +110,16 @@ class FunctionalTests(unittest.TestCase): def test_get_image_by_attribute_none(self): image = self.nd2.get_image(4, 0, "GFP", 0) self.assertIsNone(image) + + def test_fast_filter(self): + manual_images = [] + for _, image in zip(range(200), self.nd2): + if image is not None and image.channel == 'GFP': + manual_images.append(image) + filter_images = [] + for image in self.nd2.filter(channels=['GFP']): + filter_images.append(image) + if len(filter_images) == len(manual_images): + break + for a, b in zip(manual_images, filter_images): + self.assertTrue(np.array_equal(a, b)) diff --git a/nd2reader/interface.py b/nd2reader/interface.py index 32a4887..efda766 100644 --- a/nd2reader/interface.py +++ b/nd2reader/interface.py @@ -2,6 +2,7 @@ from nd2reader.parser import get_parser from nd2reader.version import get_version +import six class Nd2(object): @@ -181,6 +182,29 @@ class Nd2(object): self.height, self.width) + def filter(self, fields_of_view=None, channels=None, z_levels=None): + """ + Iterates over images matching the given criteria. This can be 2-10 times faster than manually iterating over + the Nd2 and checking the attributes of each image, as this method will not read from disk until a valid image is + found. + + """ + fields_of_view = self._to_list(fields_of_view, self.fields_of_view) + channels = self._to_list(channels, self.channels) + z_levels = self._to_list(z_levels, self.z_levels) + + for frame in self.frames: + for fov in fields_of_view: + for z in z_levels: + for c in channels: + image = self.get_image(frame, fov, c, z) + if image is not None: + yield image + + def _to_list(self, value, default): + value = default if value is None else value + return [value] if isinstance(value, int) or isinstance(value, six.string_types) else value + def close(self): """ Closes the file handle to the image. This actually sometimes will prevent problems so it's good to do this or @@ -188,3 +212,4 @@ class Nd2(object): """ self._fh.close() +1 \ No newline at end of file From f366f3800910bb72593b18843a66b918d1f5b094 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Fri, 4 Dec 2015 09:04:02 -0600 Subject: [PATCH 2/2] resolves #114. added index to Images to help with testing. Increased test coverage for filter() --- functional_tests/FYLM141111001.py | 46 +++++++++++++++++++++++++++++-- nd2reader/driver/v3.py | 4 +-- nd2reader/interface.py | 4 +-- nd2reader/model/image.py | 10 ++++++- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/functional_tests/FYLM141111001.py b/functional_tests/FYLM141111001.py index ad1ec59..06a0e3b 100644 --- a/functional_tests/FYLM141111001.py +++ b/functional_tests/FYLM141111001.py @@ -7,6 +7,7 @@ from nd2reader import Nd2 import numpy as np from datetime import datetime import unittest +import six class FunctionalTests(unittest.TestCase): @@ -108,18 +109,57 @@ class FunctionalTests(unittest.TestCase): self.assertTupleEqual((self.nd2[54].z_level, self.nd2[54].channel), (0, 'BF')) def test_get_image_by_attribute_none(self): + # Should handle missing images without an exception image = self.nd2.get_image(4, 0, "GFP", 0) self.assertIsNone(image) - def test_fast_filter(self): + def test_index(self): + # Do indexes get added to images properly? + for n, image in enumerate(self.nd2): + if image is not None: + self.assertEqual(n, image.index) + if n > 50: + break + + def test_filter(self): + # If we take the first 20 GFP images, they should be identical to the first 20 items iterated from filter() + # if we set our criteria to just "GFP" manual_images = [] - for _, image in zip(range(200), self.nd2): + for _, image in zip(range(20), self.nd2): if image is not None and image.channel == 'GFP': manual_images.append(image) filter_images = [] - for image in self.nd2.filter(channels=['GFP']): + + for image in self.nd2.filter(channels='GFP'): filter_images.append(image) if len(filter_images) == len(manual_images): break for a, b in zip(manual_images, filter_images): self.assertTrue(np.array_equal(a, b)) + + def test_filter_order_all(self): + # If we select every possible image using filter(), we should just get every image in order + n = 0 + for image in self.nd2.filter(channels=['BF', 'GFP'], z_levels=[0, 1, 2], fields_of_view=list(range(8))): + while True: + indexed_image = self.nd2[n] + if indexed_image is not None: + break + n += 1 + self.assertTrue(np.array_equal(image, indexed_image)) + n += 1 + if n > 100: + break + + def test_filter_order_subset(self): + # Test that images are always yielded in increasing order. This guarantees that no matter what subset of images + # we're filtering, we still get them in the chronological order they were acquired + n = -1 + for image in self.nd2.filter(channels='BF', z_levels=[0, 1], fields_of_view=[1, 2, 4]): + self.assertGreater(image.index, n) + self.assertEqual(image.channel, 'BF') + self.assertIn(image.field_of_view, (1, 2, 4)) + self.assertIn(image.z_level, (0, 1)) + n = image.index + if n > 100: + break diff --git a/nd2reader/driver/v3.py b/nd2reader/driver/v3.py index d231556..51c6df6 100644 --- a/nd2reader/driver/v3.py +++ b/nd2reader/driver/v3.py @@ -102,7 +102,7 @@ class V3Driver(object): except NoImageError: return None else: - image.add_params(timestamp, frame_number, field_of_view, channel, z_level) + image.add_params(index, timestamp, frame_number, field_of_view, channel, z_level) return image @property @@ -169,7 +169,7 @@ class V3Driver(object): height, width) image = Image(raw_image_data) - image.add_params(timestamp, frame_number, field_of_view, channel_name, z_level) + image.add_params(image_group_number, timestamp, frame_number, field_of_view, channel_name, z_level) except (TypeError, NoImageError): return None else: diff --git a/nd2reader/interface.py b/nd2reader/interface.py index efda766..dbf65dd 100644 --- a/nd2reader/interface.py +++ b/nd2reader/interface.py @@ -194,10 +194,10 @@ class Nd2(object): z_levels = self._to_list(z_levels, self.z_levels) for frame in self.frames: - for fov in fields_of_view: + for f in fields_of_view: for z in z_levels: for c in channels: - image = self.get_image(frame, fov, c, z) + image = self.get_image(frame, f, c, z) if image is not None: yield image diff --git a/nd2reader/model/image.py b/nd2reader/model/image.py index 75e6beb..4bbc0ad 100644 --- a/nd2reader/model/image.py +++ b/nd2reader/model/image.py @@ -12,14 +12,17 @@ class Image(np.ndarray): return np.asarray(array).view(cls) def __init__(self, array): + self._index = None self._timestamp = None self._frame_number = None self._field_of_view = None self._channel = None self._z_level = None - def add_params(self, timestamp, frame_number, field_of_view, channel, z_level): + def add_params(self, index, timestamp, frame_number, field_of_view, channel, z_level): """ + :param index: The integer that can be used to directly index this image + :type index: int :param timestamp: The number of milliseconds after the beginning of the acquisition that this image was taken. :type timestamp: float :param frame_number: The order in which this image was taken, with images of different channels/z-levels @@ -33,12 +36,17 @@ class Image(np.ndarray): :type z_level: int """ + self._index = index self._timestamp = timestamp self._frame_number = int(frame_number) self._field_of_view = field_of_view self._channel = channel self._z_level = z_level + @property + def index(self): + return self._index + @property def height(self): """