From 9bbbf7ba281e4ad5610fb095fb8a87e2059dd646 Mon Sep 17 00:00:00 2001 From: jim Date: Tue, 5 Jan 2016 21:06:41 -0600 Subject: [PATCH] resolves #140 --- ftest.py | 6 +- functional_tests/FYLM141111001.py | 15 ++-- functional_tests/monocycle.py | 123 ++++++++++++++++++++++++++++++ functional_tests/single.py | 2 +- nd2reader/driver/v3.py | 4 + nd2reader/parser/v3.py | 14 ++-- 6 files changed, 148 insertions(+), 16 deletions(-) create mode 100644 functional_tests/monocycle.py diff --git a/ftest.py b/ftest.py index d2ea0ab..4ab9c99 100644 --- a/ftest.py +++ b/ftest.py @@ -1,7 +1,7 @@ import unittest -from functional_tests.FYLM141111001 import FunctionalTests as FYLM141111Tests -from functional_tests.single import FunctionalTests as SingleTests - +from functional_tests.FYLM141111001 import FYLM141111Tests +from functional_tests.single import SingleTests +from functional_tests.monocycle import Monocycle1Tests, Monocycle2Tests if __name__ == '__main__': unittest.main() diff --git a/functional_tests/FYLM141111001.py b/functional_tests/FYLM141111001.py index 8e145a5..63e9196 100644 --- a/functional_tests/FYLM141111001.py +++ b/functional_tests/FYLM141111001.py @@ -9,7 +9,7 @@ from datetime import datetime import unittest -class FunctionalTests(unittest.TestCase): +class FYLM141111Tests(unittest.TestCase): def setUp(self): self.nd2 = Nd2("/var/nd2s/FYLM-141111-001.nd2") @@ -23,9 +23,9 @@ class FunctionalTests(unittest.TestCase): def test_date(self): self.assertEqual(self.nd2.date, datetime(2014, 11, 11, 15, 59, 19)) - # def test_length(self): - # # This will fail until we address issue #59 - # self.assertEqual(len(self.nd2), 17808) + @unittest.skip("This will fail until we address issue #59") + def test_length(self): + self.assertEqual(len(self.nd2), 17808) def test_frames(self): self.assertEqual(len(self.nd2.frames), 636) @@ -127,12 +127,13 @@ class FunctionalTests(unittest.TestCase): for _, image in zip(range(20), self.nd2): if image is not None and image.channel == 'GFP': manual_images.append(image) - filter_images = [] + filter_images = [] for image in self.nd2.select(channels='GFP'): filter_images.append(image) if len(filter_images) == len(manual_images): break + self.assertEqual(len(manual_images), len(filter_images)) self.assertGreater(len(manual_images), 0) for a, b in zip(manual_images, filter_images): @@ -141,7 +142,7 @@ class FunctionalTests(unittest.TestCase): self.assertEqual(a.field_of_view, b.field_of_view) self.assertEqual(a.channel, b.channel) - def test_filter_order_all(self): + def test_select_order_all(self): # If we select every possible image using select(), we should just get every image in order n = 0 for image in self.nd2.select(channels=['', 'GFP'], z_levels=[0, 1, 2], fields_of_view=list(range(8))): @@ -155,7 +156,7 @@ class FunctionalTests(unittest.TestCase): if n > 100: break - def test_filter_order_subset(self): + def test_select_order_subset(self): # Test that images are always yielded in increasing order. This guarantees that no matter what subset of images # we're filtering, we still get them in the chronological order they were acquired n = -1 diff --git a/functional_tests/monocycle.py b/functional_tests/monocycle.py new file mode 100644 index 0000000..08eca5d --- /dev/null +++ b/functional_tests/monocycle.py @@ -0,0 +1,123 @@ +""" +Tests on ND2s that have 1 or 2 cycles only. This is unlike the ND2s I work with typically, which are all done over very long periods of time. + +""" +from nd2reader import Nd2 +import numpy as np +import unittest + + +class Monocycle1Tests(unittest.TestCase): + def setUp(self): + self.nd2 = Nd2("/var/nd2s/simone1.nd2") + + def tearDown(self): + self.nd2.close() + + def test_select(self): + # If we take the first 20 GFP images, they should be identical to the first 20 items iterated from select() + # if we set our criteria to just "GFP" + manual_images = [] + for _, image in zip(range(20), self.nd2): + if image is not None and image.channel == 'FITC': + manual_images.append(image) + + filter_images = [] + for image in self.nd2.select(channels='FITC'): + filter_images.append(image) + if len(filter_images) == len(manual_images): + break + + self.assertEqual(len(manual_images), len(filter_images)) + self.assertGreater(len(manual_images), 0) + for a, b in zip(manual_images, filter_images): + self.assertTrue(np.array_equal(a, b)) + self.assertEqual(a.index, b.index) + self.assertEqual(a.field_of_view, b.field_of_view) + self.assertEqual(a.channel, b.channel) + + def test_select_order_all(self): + # If we select every possible image using select(), we should just get every image in order + n = 0 + for image in self.nd2.select(channels=['Cy3Narrow', 'DAPI', 'FITC', 'TxRed-modified'], z_levels=list(range(35)), fields_of_view=list(range(5))): + while True: + indexed_image = self.nd2[n] + if indexed_image is not None: + break + n += 1 + self.assertTrue(np.array_equal(image, indexed_image)) + n += 1 + if n > 100: + # Quit after the first hundred images just to save time. If there's a problem, we'll have seen it by now. + break + + def test_select_order_subset(self): + # Test that images are always yielded in increasing order. This guarantees that no matter what subset of images + # we're filtering, we still get them in the chronological order they were acquired + n = -1 + for image in self.nd2.select(channels='FITC', z_levels=[0, 1], fields_of_view=[1, 2, 4]): + self.assertGreater(image.index, n) + self.assertEqual(image.channel, 'FITC') + self.assertIn(image.field_of_view, (1, 2, 4)) + self.assertIn(image.z_level, (0, 1)) + n = image.index + if n > 100: + break + + +class Monocycle2Tests(unittest.TestCase): + def setUp(self): + self.nd2 = Nd2("/var/nd2s/hawkjo.nd2") + + def tearDown(self): + self.nd2.close() + + def test_select(self): + # If we take the first 20 HHQ 500 LP 1 images, they should be identical to the first 20 items iterated from select() + # if we set our criteria to just "HHQ 500 LP 1" + manual_images = [] + for _, image in zip(range(20), self.nd2): + if image is not None and image.channel == 'HHQ 500 LP 1': + manual_images.append(image) + + filter_images = [] + for image in self.nd2.select(channels='HHQ 500 LP 1'): + filter_images.append(image) + if len(filter_images) == len(manual_images): + break + + self.assertEqual(len(manual_images), len(filter_images)) + self.assertGreater(len(manual_images), 0) + for a, b in zip(manual_images, filter_images): + self.assertTrue(np.array_equal(a, b)) + self.assertEqual(a.index, b.index) + self.assertEqual(a.field_of_view, b.field_of_view) + self.assertEqual(a.channel, b.channel) + + def test_select_order_all(self): + # If we select every possible image using select(), we should just get every image in order + n = 0 + for image in self.nd2.select(channels=['HHQ 500 LP 1', 'HHQ 500 LP 2'], z_levels=[0], fields_of_view=list(range(100))): + while True: + indexed_image = self.nd2[n] + if indexed_image is not None: + break + n += 1 + self.assertTrue(np.array_equal(image, indexed_image)) + n += 1 + if n > 100: + # Quit after the first hundred images just to save time. If there's a problem, we'll have seen it by now. + break + + def test_select_order_subset(self): + # Test that images are always yielded in increasing order. This guarantees that no matter what subset of images + # we're filtering, we still get them in the chronological order they were acquired + n = -1 + for image in self.nd2.select(channels='HHQ 500 LP 2', z_levels=[0], fields_of_view=[1, 2, 4]): + self.assertGreater(image.index, n) + self.assertEqual(image.channel, 'HHQ 500 LP 2') + self.assertIn(image.field_of_view, (1, 2, 4)) + self.assertEqual(image.z_level, 0) + n = image.index + if n > 100: + break diff --git a/functional_tests/single.py b/functional_tests/single.py index d884248..d9c8b54 100644 --- a/functional_tests/single.py +++ b/functional_tests/single.py @@ -8,7 +8,7 @@ from datetime import datetime import unittest -class FunctionalTests(unittest.TestCase): +class SingleTests(unittest.TestCase): def setUp(self): self.nd2 = Nd2("/var/nd2s/single.nd2") diff --git a/nd2reader/driver/v3.py b/nd2reader/driver/v3.py index dbb22d3..7fcecda 100644 --- a/nd2reader/driver/v3.py +++ b/nd2reader/driver/v3.py @@ -161,16 +161,20 @@ class V3Driver(object): """ chunk = self._label_map.get_image_data_location(image_group_number) data = read_chunk(self._file_handle, chunk) + # print("data", data, "that was data") # All images in the same image group share the same timestamp! So if you have complicated image data, # your timestamps may not be entirely accurate. Practically speaking though, they'll only be off by a few # seconds unless you're doing something super weird. timestamp = struct.unpack("d", data[:8])[0] image_group_data = array.array("H", data) image_data_start = 4 + channel_offset + # The images for the various channels are interleaved within the same array. For example, the second image # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design # a data structure that way, please send the author of this library a message. + image_data = np.reshape(image_group_data[image_data_start::len(self._metadata.channels)], (height, width)) + # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you # don't have the same number of images each cycle. We discovered this because we only took GFP images every # other cycle to reduce phototoxicity, but NIS Elements still allocated memory as if we were going to take diff --git a/nd2reader/parser/v3.py b/nd2reader/parser/v3.py index 37ba676..f05a141 100644 --- a/nd2reader/parser/v3.py +++ b/nd2reader/parser/v3.py @@ -173,7 +173,7 @@ class V3Parser(BaseParser): frames = self._parse_frames(self.raw_metadata) z_levels = self._parse_z_levels(self.raw_metadata) total_images_per_channel = self._parse_total_images_per_channel(self.raw_metadata) - channels = sorted([key for key in self.camera_metadata.keys()]) + channels = self._parse_channels(self.raw_metadata) self.metadata = Metadata(height, width, channels, date, fields_of_view, frames, z_levels, total_images_per_channel) def _parse_camera_settings(self): @@ -181,20 +181,24 @@ class V3Parser(BaseParser): Looks up information in the raw metadata about the camera(s) and puts it into a CameraSettings object. Duplicate cameras can be returned if the same one was used for multiple channels. - :return: """ - for camera in self.raw_metadata.image_metadata_sequence[six.b('SLxPictureMetadata')][six.b('sPicturePlanes')][six.b('sSampleSetting')].values(): + for n, camera in enumerate(self.raw_metadata.image_metadata_sequence[six.b('SLxPictureMetadata')][six.b('sPicturePlanes')][six.b('sSampleSetting')].values()): name = camera[six.b('pCameraSetting')][six.b('CameraUserName')] id = camera[six.b('pCameraSetting')][six.b('CameraUniqueName')] exposure = camera[six.b('dExposureTime')] x_binning = camera[six.b('pCameraSetting')][six.b('FormatFast')][six.b('fmtDesc')][six.b('dBinningX')] y_binning = camera[six.b('pCameraSetting')][six.b('FormatFast')][six.b('fmtDesc')][six.b('dBinningY')] optical_configs = camera[six.b('sOpticalConfigs')] + + # This definitely is not working right. It seems to be totally inconsistent in each of the sample ND2s that I have. + # Fixing one breaks another. if six.b('') in optical_configs.keys(): channel_name = optical_configs[six.b('')][six.b('sOpticalConfigName')] + yield CameraSettings(name, id, exposure, x_binning, y_binning, channel_name) else: - channel_name = None - yield CameraSettings(name, id, exposure, x_binning, y_binning, channel_name) + channel_names = [channel[six.b('Name')] for key, channel in camera[six.b('pCameraSetting')][six.b('Metadata')][six.b('Channels')].items()] + for channel_name in channel_names: + yield CameraSettings(name, id, exposure, x_binning, y_binning, channel_name) def _parse_date(self, raw_metadata): """