From 1268b2768c4e0fb370b2cbd5a4c5590c6dcddef3 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Wed, 2 Dec 2015 22:03:03 -0600 Subject: [PATCH] #125: fixed bug by ordering channel names, which works but seems suspect. added py2 support to Dockerfile --- Dockerfile | 8 ++++--- Makefile | 21 +++++++++------- ftests.py => ftest.py | 0 functional_tests/FYLM141111001.py | 38 +++++++++++++++++++++++++---- nd2reader/interface.py | 40 +++++++++++++++++++++---------- nd2reader/model/metadata.py | 23 ++++++++++++++++++ nd2reader/parser/v3.py | 31 ++++++++++++++++++++---- tests.py => test.py | 0 woo.py | 14 +++++++++++ 9 files changed, 141 insertions(+), 34 deletions(-) rename ftests.py => ftest.py (100%) rename tests.py => test.py (100%) create mode 100644 woo.py diff --git a/Dockerfile b/Dockerfile index e860fd8..4002514 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,20 +19,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python-numpy \ python3-numpy \ libfreetype6-dev \ + python-matplotlib \ python3-matplotlib \ libfreetype6-dev \ libpng-dev \ libjpeg-dev \ pkg-config \ + python-skimage \ python3-skimage \ tk \ tk-dev \ + python-tk \ python3-tk \ + && pip install -U cython \ + scikit-image \ && pip3 install -U cython \ scikit-image \ && rm -rf /var/lib/apt/lists/* -COPY . /opt/nd2reader WORKDIR /opt/nd2reader -RUN python setup.py install -RUN python3 setup.py install diff --git a/Makefile b/Makefile index 8e1fc86..3ddd695 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,16 @@ -.PHONY: info build shell py2 py3 test +.PHONY: info build shell py2 py3 test ftest publish info: @echo "" @echo "Available Make Commands" @echo "" - @echo "build: builds the image" + @echo "build: builds the image" + @echo "shell: starts a bash shell in the container @echo "py2: maps ~/Documents/nd2s to /var/nd2s and runs a Python 2.7 interpreter" @echo "py3: maps ~/Documents/nd2s to /var/nd2s and runs a Python 3.4 interpreter" - @echo "test: runs all unit tests (in Python 3.4)" + @echo "test: runs all unit tests (in Python 3.4)" + @echo "ftest: runs all functional tests (requires specific ND2 files that are not publicly available" + @echo "publish: publishes the code base to PyPI (maintainers only)" @echo "" build: @@ -17,18 +20,18 @@ shell: xhost local:root; docker run --rm -v ~/nd2s:/var/nd2s -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -it jimrybarski/nd2reader bash py2: - xhost local:root; docker run --rm -v ~/nd2s:/var/nd2s -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -it jimrybarski/nd2reader python2.7 + xhost local:root; docker run --rm -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -it jimrybarski/nd2reader python2.7 py3: - xhost local:root; docker run --rm -v ~/nd2s:/var/nd2s -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -it jimrybarski/nd2reader python3.4 + xhost local:root; docker run --rm -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=unix$(DISPLAY) -it jimrybarski/nd2reader python3.4 test: build - docker run --rm -it jimrybarski/nd2reader python3.4 /opt/nd2reader/tests.py - docker run --rm -it jimrybarski/nd2reader python2.7 /opt/nd2reader/tests.py + docker run --rm -v $(CURDIR):/opt/nd2reader -it jimrybarski/nd2reader python3.4 test.py + docker run --rm -v $(CURDIR):/opt/nd2reader -it jimrybarski/nd2reader python2.7 test.py ftest: build - docker run --rm -v ~/nd2s:/var/nd2s -it jimrybarski/nd2reader python3.4 /opt/nd2reader/ftests.py - docker run --rm -v ~/nd2s:/var/nd2s -it jimrybarski/nd2reader python2.7 /opt/nd2reader/ftests.py + docker run --rm -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -it jimrybarski/nd2reader python3.4 /opt/nd2reader/ftest.py + docker run --rm -v $(CURDIR):/opt/nd2reader -v ~/nd2s:/var/nd2s -it jimrybarski/nd2reader python2.7 /opt/nd2reader/ftest.py publish: python setup.py sdist upload -r pypi diff --git a/ftests.py b/ftest.py similarity index 100% rename from ftests.py rename to ftest.py diff --git a/functional_tests/FYLM141111001.py b/functional_tests/FYLM141111001.py index 93f3f51..9d1d6df 100644 --- a/functional_tests/FYLM141111001.py +++ b/functional_tests/FYLM141111001.py @@ -33,7 +33,7 @@ class FunctionalTests(unittest.TestCase): self.assertEqual(len(self.nd2.fields_of_view), 8) def test_channels(self): - self.assertTupleEqual(tuple(sorted(self.nd2.channels)), ('', 'GFP')) + self.assertTupleEqual(tuple(sorted(self.nd2.channels)), ('BF', 'GFP')) def test_z_levels(self): self.assertTupleEqual(tuple(self.nd2.z_levels), (0, 1, 2)) @@ -43,7 +43,7 @@ class FunctionalTests(unittest.TestCase): self.assertEqual(image.field_of_view, 2) self.assertEqual(image.frame_number, 0) self.assertAlmostEqual(image.timestamp, 19.0340758) - self.assertEqual(image.channel, '') + self.assertEqual(image.channel, 'BF') self.assertEqual(image.z_level, 1) self.assertEqual(image.height, self.nd2.height) self.assertEqual(image.width, self.nd2.width) @@ -71,11 +71,41 @@ class FunctionalTests(unittest.TestCase): def test_get_image_by_attribute_ok(self): image = self.nd2.get_image(4, 0, "GFP", 1) self.assertIsNotNone(image) - image = self.nd2.get_image(4, 0, "", 0) + image = self.nd2.get_image(4, 0, "BF", 0) self.assertIsNotNone(image) - image = self.nd2.get_image(4, 0, "", 1) + image = self.nd2.get_image(4, 0, "BF", 1) self.assertIsNotNone(image) + def test_images(self): + self.assertTupleEqual((self.nd2[0].z_level, self.nd2[0].channel), (0, 'BF')) + self.assertIsNone(self.nd2[1]) + self.assertTupleEqual((self.nd2[2].z_level, self.nd2[2].channel), (1, 'BF')) + self.assertTupleEqual((self.nd2[3].z_level, self.nd2[3].channel), (1, 'GFP')) + self.assertTupleEqual((self.nd2[4].z_level, self.nd2[4].channel), (2, 'BF')) + self.assertIsNone(self.nd2[5]) + self.assertTupleEqual((self.nd2[6].z_level, self.nd2[6].channel), (0, 'BF')) + self.assertIsNone(self.nd2[7]) + self.assertTupleEqual((self.nd2[8].z_level, self.nd2[8].channel), (1, 'BF')) + self.assertTupleEqual((self.nd2[9].z_level, self.nd2[9].channel), (1, 'GFP')) + self.assertTupleEqual((self.nd2[10].z_level, self.nd2[10].channel), (2, 'BF')) + self.assertIsNone(self.nd2[11]) + self.assertTupleEqual((self.nd2[12].z_level, self.nd2[12].channel), (0, 'BF')) + self.assertIsNone(self.nd2[13]) + self.assertTupleEqual((self.nd2[14].z_level, self.nd2[14].channel), (1, 'BF')) + self.assertTupleEqual((self.nd2[15].z_level, self.nd2[15].channel), (1, 'GFP')) + self.assertTupleEqual((self.nd2[16].z_level, self.nd2[16].channel), (2, 'BF')) + self.assertIsNone(self.nd2[17]) + self.assertTupleEqual((self.nd2[18].z_level, self.nd2[18].channel), (0, 'BF')) + self.assertIsNone(self.nd2[19]) + self.assertIsNone(self.nd2[47]) + self.assertTupleEqual((self.nd2[48].z_level, self.nd2[48].channel), (0, 'BF')) + self.assertIsNone(self.nd2[49]) + self.assertTupleEqual((self.nd2[50].z_level, self.nd2[50].channel), (1, 'BF')) + self.assertIsNone(self.nd2[51]) + self.assertTupleEqual((self.nd2[52].z_level, self.nd2[52].channel), (2, 'BF')) + self.assertIsNone(self.nd2[53]) + self.assertTupleEqual((self.nd2[54].z_level, self.nd2[54].channel), (0, 'BF')) + def test_get_image_by_attribute_none(self): image = self.nd2.get_image(4, 0, "GFP", 0) self.assertIsNone(image) diff --git a/nd2reader/interface.py b/nd2reader/interface.py index f4305d5..32a4887 100644 --- a/nd2reader/interface.py +++ b/nd2reader/interface.py @@ -12,9 +12,8 @@ class Nd2(object): self._fh = open(filename, "rb") major_version, minor_version = get_version(self._fh) self._parser = get_parser(self._fh, major_version, minor_version) - self._driver = self._parser.driver self._metadata = self._parser.metadata - + def __enter__(self): return self @@ -27,7 +26,7 @@ class Nd2(object): "Created: %s" % (self.date if self.date is not None else "Unknown"), "Image size: %sx%s (HxW)" % (self.height, self.width), "Frames: %s" % len(self.frames), - "Channels: %s" % ", ".join(["'%s'" % str(channel) for channel in self.channels]), + "Channels: %s" % ", ".join(["%s" % str(channel) for channel in self.channels]), "Fields of View: %s" % len(self.fields_of_view), "Z-Levels: %s" % len(self.z_levels) ]) @@ -52,7 +51,7 @@ class Nd2(object): """ if isinstance(item, int): try: - image = self._driver.get_image(item) + image = self._parser.driver.get_image(item) except KeyError: raise IndexError else: @@ -78,6 +77,10 @@ class Nd2(object): for i in range(start, stop)[::step]: yield self[i] + @property + def camera_settings(self): + return self._parser.camera_metadata + @property def date(self): """ @@ -91,9 +94,11 @@ class Nd2(object): @property def z_levels(self): """ - A list of integers that represent the different levels on the Z-axis that images were taken. Currently this is just a list of numbers from 0 to N. - For example, an ND2 where images were taken at -3µm, 0µm, and +5µm from a set position would be represented by 0, 1 and 2, respectively. ND2s do store the actual - offset of each image in micrometers and in the future this will hopefully be available. For now, however, you will have to match up the order yourself. + A list of integers that represent the different levels on the Z-axis that images were taken. Currently this is + just a list of numbers from 0 to N. For example, an ND2 where images were taken at -3µm, 0µm, and +5µm from a + set position would be represented by 0, 1 and 2, respectively. ND2s do store the actual offset of each image + in micrometers and in the future this will hopefully be available. For now, however, you will have to match up + the order yourself. :return: list of int @@ -103,7 +108,8 @@ class Nd2(object): @property def fields_of_view(self): """ - A list of integers representing the various stage locations, in the order they were taken in the first round of acquisition. + A list of integers representing the various stage locations, in the order they were taken in the first round + of acquisition. :return: list of int @@ -123,8 +129,9 @@ class Nd2(object): @property def frames(self): """ - A list of integers representing groups of images. ND2s consider images to be part of the same frame if they are in the same field of view and don't have the same channel. - So if you take a bright field and GFP image at four different fields of view over and over again, you'll have 8 images and 4 frames per cycle. + A list of integers representing groups of images. ND2s consider images to be part of the same frame if they + are in the same field of view and don't have the same channel. So if you take a bright field and GFP image at + four different fields of view over and over again, you'll have 8 images and 4 frames per cycle. :return: list of int @@ -153,7 +160,8 @@ class Nd2(object): def get_image(self, frame_number, field_of_view, channel_name, z_level): """ - Attempts to return the image with the unique combination of given attributes. None will be returned if a match is not found. + Attempts to return the image with the unique combination of given attributes. None will be returned if a match + is not found. :type frame_number: int :param field_of_view: the label for the place in the XY-plane where this image was taken. @@ -166,11 +174,17 @@ class Nd2(object): :rtype: nd2reader.model.Image() or None """ - return self._driver.get_image_by_attributes(frame_number, field_of_view, channel_name, z_level, self.height, self.width) + return self._parser.driver.get_image_by_attributes(frame_number, + field_of_view, + channel_name, + z_level, + self.height, + self.width) def close(self): """ - Closes the file handle to the image. This actually sometimes will prevent problems so it's good to do this or use Nd2 as a context manager. + Closes the file handle to the image. This actually sometimes will prevent problems so it's good to do this or + use Nd2 as a context manager. """ self._fh.close() diff --git a/nd2reader/model/metadata.py b/nd2reader/model/metadata.py index 6e4f57a..3ee65fa 100644 --- a/nd2reader/model/metadata.py +++ b/nd2reader/model/metadata.py @@ -1,3 +1,6 @@ +import six + + class Metadata(object): """ A simple container for ND2 metadata. """ def __init__(self, height, width, channels, date, fields_of_view, frames, z_levels, total_images_per_channel): @@ -93,3 +96,23 @@ class Metadata(object): """ return self._total_images_per_channel + + +class CameraSettings(object): + def __init__(self, name, id, exposure, x_binning, y_binning, channel_name): + self.name = name.decode("utf8") + self.id = id.decode("utf8") + self.exposure = exposure + self.x_binning = int(x_binning) + self.y_binning = int(y_binning) + self.channel_name = channel_name + if six.PY3: + self.channel_name = self.channel_name.decode("utf8") if channel_name is not None else None + + def __repr__(self): + return "\n".join(["" % self.channel_name, + "Camera: %s" % self.name, + "Camera ID: %s" % self.id, + "Exposure Time (ms): %s" % self.exposure, + "Binning: %sx%s" % (self.x_binning, self.y_binning) + ]) diff --git a/nd2reader/parser/v3.py b/nd2reader/parser/v3.py index eb48641..1fbe0d7 100644 --- a/nd2reader/parser/v3.py +++ b/nd2reader/parser/v3.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from datetime import datetime -from nd2reader.model.metadata import Metadata +from nd2reader.model.metadata import Metadata, CameraSettings from nd2reader.model.label import LabelMap from nd2reader.parser.base import BaseParser from nd2reader.driver.v3 import V3Driver @@ -138,6 +138,8 @@ class V3Parser(BaseParser): self._metadata = None self._raw_metadata = None self._label_map = None + self._camera_metadata = {} + self._parse_metadata() @property def metadata(self): @@ -145,10 +147,12 @@ class V3Parser(BaseParser): :rtype: Metadata """ - if not self._metadata: - self._parse_metadata() return self._metadata + @property + def camera_metadata(self): + return self._camera_metadata + @property def driver(self): return V3Driver(self.metadata, self._label_map, self._fh) @@ -167,13 +171,30 @@ class V3Parser(BaseParser): """ height = self.raw_metadata.image_attributes[six.b('SLxImageAttributes')][six.b('uiHeight')] width = self.raw_metadata.image_attributes[six.b('SLxImageAttributes')][six.b('uiWidth')] - channels = self._parse_channels(self.raw_metadata) date = self._parse_date(self.raw_metadata) fields_of_view = self._parse_fields_of_view(self.raw_metadata) frames = self._parse_frames(self.raw_metadata) z_levels = self._parse_z_levels(self.raw_metadata) total_images_per_channel = self._parse_total_images_per_channel(self.raw_metadata) - self._metadata = Metadata(height, width, channels, date, fields_of_view, frames, z_levels, total_images_per_channel) + channels = [] + for camera_setting in self._parse_camera_settings(): + channels.append(camera_setting.channel_name) + self._camera_metadata[camera_setting.channel_name] = camera_setting + self._metadata = Metadata(height, width, sorted(list(channels)), date, fields_of_view, frames, z_levels, total_images_per_channel) + + def _parse_camera_settings(self): + for camera in self._raw_metadata.image_metadata_sequence[six.b('SLxPictureMetadata')][six.b('sPicturePlanes')][six.b('sSampleSetting')].values(): + name = camera[six.b('pCameraSetting')][six.b('CameraUserName')] + id = camera[six.b('pCameraSetting')][six.b('CameraUniqueName')] + exposure = camera[six.b('dExposureTime')] + x_binning = camera[six.b('pCameraSetting')][six.b('FormatFast')][six.b('fmtDesc')][six.b('dBinningX')] + y_binning = camera[six.b('pCameraSetting')][six.b('FormatFast')][six.b('fmtDesc')][six.b('dBinningY')] + optical_configs = camera[six.b('sOpticalConfigs')] + if six.b('') in optical_configs.keys(): + channel_name = optical_configs[six.b('')][six.b('sOpticalConfigName')] + else: + channel_name = None + yield CameraSettings(name, id, exposure, x_binning, y_binning, channel_name) def _parse_date(self, raw_metadata): """ diff --git a/tests.py b/test.py similarity index 100% rename from tests.py rename to test.py diff --git a/woo.py b/woo.py new file mode 100644 index 0000000..b51cc6d --- /dev/null +++ b/woo.py @@ -0,0 +1,14 @@ +from nd2reader import Nd2 +from pprint import pprint +import six + +n = Nd2("/home/jim/nd2s/FYLM-141111-001.nd2") + +# for k, v in n._parser.raw_metadata.image_metadata_sequence[b'SLxPictureMetadata'][b'sPicturePlanes'][b'sSampleSetting'][b'a1'].items(): +for camera in n._parser._raw_metadata.image_metadata_sequence[b'SLxPictureMetadata'][b'sPicturePlanes'][b'sSampleSetting'].values(): + name = camera[six.b('pCameraSetting')][six.b('CameraUserName')] + id = camera[six.b('pCameraSetting')][six.b('CameraUniqueName')] + channel_name = camera[six.b('sOpticalConfigs')][six.b('')][six.b('sOpticalConfigName')] + x_binning = camera[six.b('pCameraSetting')][six.b('FormatFast')][six.b('fmtDesc')][six.b('dBinningX')] + y_binning = camera[six.b('pCameraSetting')][six.b('FormatFast')][six.b('fmtDesc')][six.b('dBinningY')] + exposure = camera[six.b('dExposureTime')]