diff --git a/.idea/dictionaries/jim.xml b/.idea/dictionaries/jim.xml deleted file mode 100644 index a26e67a..0000000 --- a/.idea/dictionaries/jim.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - filemap - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 2b29e40..0000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 3b31283..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - \ No newline at end of file diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt new file mode 100644 index 0000000..1b0a856 --- /dev/null +++ b/CONTRIBUTORS.txt @@ -0,0 +1,4 @@ +Author: Jim Rybarski + +nd2reader is based on the read_nd2 module from the SLOTH library (http://pythonhosted.org/SLOTH/_modules/sloth/read_nd2.html). +Thanks to M.Kauer and B.Kauer for solving the hardest part of parsing ND2s. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..37f37ee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM ubuntu +MAINTAINER Jim Rybarski + +RUN mkdir -p /var/nds2 +RUN apt-get update && apt-get install -y \ + python-numpy + +COPY . /opt/nd2reader +WORKDIR /opt/nd2reader +RUN python setup.py install +WORKDIR /var/nd2s + +CMD /usr/bin/python2.7 diff --git a/LICENSE b/LICENSE.txt similarity index 100% rename from LICENSE rename to LICENSE.txt diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2438069 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY: build shell + +build: + docker build -t jimrybarski/nd2reader . + +shell: + docker run --rm -v ~/Documents/nd2s:/var/nd2s -it jimrybarski/nd2reader diff --git a/README.md b/README.md index 8d3f555..d001757 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,116 @@ -nd2reader -========= +# nd2reader -## Simple access to hierarchical .nd2 files +### About -# About +`nd2reader` is a pure-Python package that reads images produced by NIS Elements. -`nd2reader` is a pure-Python package that reads images produced by Nikon microscopes. Though it more or less works, it is currently under development and is not quite ready for use by the general public. Version 1.0 should be released in early 2015. +.nd2 files contain images and metadata, which can be split along multiple dimensions: time, fields of view (xy-plane), focus (z-plane), and filter channel. -.nd2 files contain images and metadata, which can be split along multiple dimensions: time, fields of view (xy-axis), focus (z-axis), and filter channel. `nd2reader` allows you to view any subset of images based on any or all of these dimensions. +`nd2reader` produces data in numpy arrays, which makes it trivial to use with the image analysis packages such as `scikit-image` and `OpenCV`. -`nd2reader` holds data in numpy arrays, which makes it trivial to use with the image analysis packages `scikit-image` and `OpenCV`. +### Installation -# Dependencies +Just use pip: -numpy +`pip install nd2reader` -# Installation +If you want to install via git, clone the repo and run: -I'll write this eventually. +`python setup.py install` + +### ND2s + +A quick summary of ND2 metadata can be obtained as shown below. +```python +>>> import nd2reader +>>> nd2 = nd2reader.Nd2("/path/to/my_images.nd2") +>>> nd2 + +Created: 2014-11-11 15:59:19 +Image size: 1280x800 (HxW) +Image cycles: 636 +Channels: '', 'GFP' +Fields of View: 8 +Z-Levels: 3 +``` + +### Simple Iteration + +For most cases, you'll just want to iterate over each image: + +```python +import nd2reader +nd2 = nd2reader.Nd2("/path/to/my_images.nd2") +for image in nd2: + do_something(image.data) +``` + +### Image Sets + +If you have complicated hierarchical data, it may be easier to use image sets, which groups images together if they +share the same time index and field of view: + +```python +import nd2reader +nd2 = nd2reader.Nd2("/path/to/my_complicated_images.nd2") +for image_set in nd2.image_sets: + # you can select images by channel + gfp_image = image_set.get("GFP") + do_something_gfp_related(gfp_image) + + # you can also specify the z-level. this defaults to 0 if not given + out_of_focus_image = image_set.get("Bright Field", z_level=1) + do_something_out_of_focus_related(out_of_focus_image) +``` + +### Direct Image Access + +There is a method, `get_image`, which allows random access to images. This might not always return an image, however, +if you acquired different numbers of images in each cycle of a program. For example, if you acquire GFP images every +other minute, but acquire bright field images every minute, `get_image` will return `None` at certain time indexes. + +### Images + +`Image` objects provide several pieces of useful data. + +```python +>>> import nd2reader +>>> nd2 = nd2reader.Nd2("/path/to/my_images.nd2") +>>> image = nd2.get_image(14, 2, "GFP", 1) +>>> image.data +array([[1809, 1783, 1830, ..., 1923, 1920, 1914], + [1687, 1855, 1792, ..., 1986, 1903, 1889], + [1758, 1901, 1849, ..., 1911, 2010, 1954], + ..., + [3363, 3370, 3570, ..., 3565, 3601, 3459], + [3480, 3428, 3328, ..., 3542, 3461, 3575], + [3497, 3666, 3635, ..., 3817, 3867, 3779]]) +>>> image.channel +'GFP' +>>> image.timestamp +1699.7947813408175 +>>> image.field_of_view +2 +>>> image.z_level +1 + +# You can also get a quick summary of image data: + +>>> image + +1280x800 (HxW) +Timestamp: 1699.79478134 +Field of View: 2 +Channel: GFP +Z-Level: 1 +``` + +### Bug Reports and Features + +If this fails to work exactly as expected, please open a Github issue. If you get an unhandled exception, please +paste the entire stack trace into the issue as well. + +### Contributing + +Please feel free to submit a pull request with any new features you think would be useful. You can also create an +issue if you'd just like to propose or discuss a potential idea. \ No newline at end of file diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 264eb55..26a6f1b 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -1,47 +1,104 @@ -import logging -from nd2reader.service import BaseNd2 +# -*- coding: utf-8 -*- + from nd2reader.model import Image, ImageSet +from nd2reader.parser import Nd2Parser -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) +class Nd2(Nd2Parser): + """ + Allows easy access to NIS Elements .nd2 image files. -class Nd2(BaseNd2): + """ def __init__(self, filename): super(Nd2, self).__init__(filename) + self._filename = filename + + def __repr__(self): + return "\n".join(["" % self._filename, + "Created: %s" % self._absolute_start.strftime("%Y-%m-%d %H:%M:%S"), + "Image size: %sx%s (HxW)" % (self.height, self.width), + "Image cycles: %s" % self._time_index_count, + "Channels: %s" % ", ".join(["'%s'" % channel for channel in self._channels]), + "Fields of View: %s" % self._field_of_view_count, + "Z-Levels: %s" % self._z_level_count + ]) - def get_image(self, time_index, fov, channel_name, z_level): - image_set_number = self._calculate_image_set_number(time_index, fov, z_level) - timestamp, raw_image_data = self._reader.get_raw_image_data(image_set_number, self.channel_offset[channel_name]) - return Image(timestamp, raw_image_data, fov, channel_name, z_level, self.height, self.width) + @property + def height(self): + """ + :return: height of each image, in pixels + :rtype: int + + """ + return self.metadata['ImageAttributes']['SLxImageAttributes']['uiHeight'] + + @property + def width(self): + """ + :return: width of each image, in pixels + :rtype: int + + """ + return self.metadata['ImageAttributes']['SLxImageAttributes']['uiWidth'] def __iter__(self): """ - Just return every image in order (might not be exactly the order that the images were physically taken, but it will - be within a few seconds). A better explanation is probably needed here. + Iterates over every image, in the order they were taken. + + :return: model.Image() """ for i in range(self._image_count): - for fov in range(self.field_of_view_count): - for z_level in range(self.z_level_count): - for channel in self.channels: - image = self.get_image(i, fov, channel.name, z_level) - if image.is_valid: + for fov in range(self._field_of_view_count): + for z_level in range(self._z_level_count): + for channel_name in self._channels: + image = self.get_image(i, fov, channel_name, z_level) + if image is not None: yield image - def image_sets(self, field_of_view, time_indices=None, channels=None, z_levels=None): - """ - Gets all the images for a given field of view and + @property + def image_sets(self): """ - timepoint_set = xrange(self.time_index_count) if time_indices is None else time_indices - channel_set = [channel.name for channel in self.channels] if channels is None else channels - z_level_set = xrange(self.z_level_count) if z_levels is None else z_levels + Iterates over groups of related images. This is useful if your ND2 contains multiple fields of view. + A typical use case might be that you have, say, four areas of interest that you're monitoring, and every + minute you take a bright field and GFP image of each one. For each cycle, this method would produce four + ImageSet objects, each containing one bright field and one GFP image. + + :return: model.ImageSet() - for timepoint in timepoint_set: + """ + for time_index in xrange(self._time_index_count): image_set = ImageSet() - for channel_name in channel_set: - for z_level in z_level_set: - image = self.get_image(timepoint, field_of_view, channel_name, z_level) - if image.is_valid: - image_set.add(image) - yield image_set \ No newline at end of file + for fov in range(self._field_of_view_count): + for channel_name in self._channels: + for z_level in xrange(self._z_level_count): + image = self.get_image(time_index, fov, channel_name, z_level) + if image is not None: + image_set.add(image) + yield image_set + + def get_image(self, time_index, field_of_view, channel_name, z_level): + """ + Returns an Image if data exists for the given parameters, otherwise returns None. In general, you should avoid + using this method unless you're very familiar with the structure of ND2 files. If you have a use case that + cannot be met by the `__iter__` or `image_sets` methods above, please create an issue on Github. + + :param time_index: the frame number + :type time_index: int + :param field_of_view: the label for the place in the XY-plane where this image was taken. + :type field_of_view: int + :param channel_name: the name of the color of this image + :type channel_name: str + :param z_level: the label for the location in the Z-plane where this image was taken. + :type z_level: int + :rtype: nd2reader.model.Image() or None + + """ + image_set_number = self._calculate_image_group_number(time_index, field_of_view, z_level) + try: + timestamp, raw_image_data = self._get_raw_image_data(image_set_number, self._channel_offset[channel_name]) + image = Image(timestamp, raw_image_data, field_of_view, channel_name, z_level, self.height, self.width) + except TypeError: + return None + else: + return image \ No newline at end of file diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 9c15ac2..bd547a1 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -1,56 +1,33 @@ +# -*- coding: utf-8 -*- + +import collections import numpy as np -import skimage.io import logging -from io import BytesIO -import array -import struct - log = logging.getLogger(__name__) -class Channel(object): - def __init__(self, name, camera, exposure_time): - self._name = name - self._camera = camera - self._exposure_time = exposure_time - - @property - def name(self): - return self._name - - @property - def camera(self): - return self._camera - - @property - def exposure_time(self): - return self._exposure_time - - -class ImageSet(object): - """ - A group of images that share the same timestamp. NIS Elements doesn't store a unique timestamp for every - image, rather, it stores one for each set of images that share the same field of view and z-axis level. - - """ - def __init__(self): - self._images = [] - - def add(self, image): +class Image(object): + def __init__(self, timestamp, raw_array, field_of_view, channel, z_level, height, width): """ - :type image: nd2reader.model.Image() + A wrapper around the raw pixel data of an image. + + :param timestamp: The number of milliseconds after the beginning of the acquisition that this image was taken. + :type timestamp: int + :param raw_array: The raw sequence of bytes that represents the image. + :type raw_array: array.array() + :param field_of_view: The label for the place in the XY-plane where this image was taken. + :type field_of_view: int + :param channel: The name of the color of this image + :type channel: str + :param z_level: The label for the location in the Z-plane where this image was taken. + :type z_level: int + :param height: The height of the image in pixels. + :type height: int + :param width: The width of the image in pixels. + :type width: int """ - self._images.append(image) - - def __iter__(self): - for image in self._images: - yield image - - -class Image(object): - def __init__(self, timestamp, raw_array, field_of_view, channel, z_level, height, width): self._timestamp = timestamp self._raw_data = raw_array self._field_of_view = field_of_view @@ -60,208 +37,111 @@ class Image(object): self._width = width self._data = None + def __repr__(self): + return "\n".join(["", + "%sx%s (HxW)" % (self._height, self._width), + "Timestamp: %s" % self.timestamp, + "Field of View: %s" % self.field_of_view, + "Channel: %s" % self.channel, + "Z-Level: %s" % self.z_level, + ]) + + @property + def data(self): + """ + The actual image data. + + :rtype np.array() + + """ + if self._data is None: + # The data is just a 1-dimensional array originally. + # We convert it to a 2D image here. + self._data = np.reshape(self._raw_data, (self._height, self._width)) + return self._data + @property def field_of_view(self): + """ + Which of the fixed locations this image was taken at. + + :rtype int: + + """ return self._field_of_view @property def timestamp(self): """ - The number of seconds after the beginning of the acquisition that the image was taken. Note that for a given field - of view and z-level offset, if you have images of multiple channels, they will all be given the same timestamp. - No, this doesn't make much sense. But that's how ND2s are structured, so if your experiment depends on millisecond - accuracy, you need to find an alternative imaging system. + The number of seconds after the beginning of the acquisition that the image was taken. Note that for a given + field of view and z-level offset, if you have images of multiple channels, they will all be given the same + timestamp. No, this doesn't make much sense. But that's how ND2s are structured, so if your experiment depends + on millisecond accuracy, you need to find an alternative imaging system. + + :rtype float: """ return self._timestamp / 1000.0 @property def channel(self): + """ + The name of the filter used to acquire this image. These are user-supplied in NIS Elements. + + :rtype str: + + """ return self._channel @property def z_level(self): - return self._z_level - - @property - def data(self): - if self._data is None: - # The data is just a flat, 1-dimensional array. We convert it to a 2D array and cast the data points as 16-bit integers - self._data = np.reshape(self._raw_data, (self._height, self._width)).astype(np.int64).astype(np.uint16) - return self._data + """ + The vertical offset of the image. These are simple integers starting from 0, where the 0 is the lowest + z-level and each subsequent level incremented by 1. - @property - def is_valid(self): - return np.any(self.data) + For example, if you acquired images at -3 µm, 0 µm, and +3 µm, your z-levels would be: - def show(self): - skimage.io.imshow(self.data) - skimage.io.show() + -3 µm: 0 + 0 µm: 1 + +3 µm: 2 + :rtype int: -class MetadataItem(object): - def __init__(self, start, data): - self._datatype = ord(data[start]) - self._label_length = 2 * ord(data[start + 1]) - self._data = data + """ + return self._z_level - @property - def is_valid(self): - return self._datatype > 0 - @property - def key(self): - return self._data[2:self._label_length].decode("utf16").encode("utf8") +class ImageSet(object): + """ + A group of images that were taken at roughly the same time. - @property - def length(self): - return self._length + """ + def __init__(self): + self._images = collections.defaultdict(dict) - @property - def data_start(self): - return self._label_length + 2 + def __len__(self): + """ The number of images in the image set. """ + return sum([len(channel) for channel in self._images.values()]) - @property - def _body(self): - """ - All data after the header. + def __repr__(self): + return "\n".join(["", + "Image count: %s" % len(self)]) + def get(self, channel, z_level=0): """ - return self._data[self.data_start:] + Retrieve an image with a given channel and z-level. For most users, z_level will always be 0. - def _get_bytes(self, count): - return self._data[self.data_start: self.data_start + count] + :type channel: str + :type z_level: int - @property - def value(self): - parser = {1: self._parse_unsigned_char, - 2: self._parse_unsigned_int, - 3: self._parse_unsigned_int, - 5: self._parse_unsigned_long, - 6: self._parse_double, - 8: self._parse_string, - 9: self._parse_char_array, - 11: self._parse_metadata_item - } - return parser[self._datatype]() - - def _parse_unsigned_char(self): - self._length = 1 - return self._unpack("B", self._get_bytes(self._length)) - - def _parse_unsigned_int(self): - self._length = 4 - return self._unpack("I", self._get_bytes(self._length)) - - def _parse_unsigned_long(self): - self._length = 8 - return self._unpack("Q", self._get_bytes(self._length)) - - def _parse_double(self): - self._length = 8 - return self._unpack("d", self._get_bytes(self._length)) - - def _parse_string(self): - # the string is of unknown length but ends at the first instance of \x00\x00 - stop = self._body.index("\x00\x00") - self._length = stop - return self._body[:stop - 1].decode("utf16").encode("utf8") - - def _parse_char_array(self): - array_length = self._unpack("Q", self._get_bytes(8)) - self._length = array_length + 8 - return array.array("B", self._body[8:array_length]) - - def _parse_metadata_item(self): - count, length = struct.unpack("=1.9.2 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..224a779 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py index b7acbd1..6ac6438 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,21 @@ -from setuptools import setup, find_packages +from distutils.core import setup setup( name="nd2reader", - packages=find_packages(), - version="0.9.7", - install_requires=[ - 'numpy', - ], + packages=['nd2reader', 'nd2reader.model'], + version="1.0.0", + description='A tool for reading ND2 files produced by NIS Elements', + author='Jim Rybarski', + author_email='jim@rybarski.com', + url='https://github.com/jimrybarski/nd2reader', + download_url='https://github.com/jimrybarski/nd2reader/tarball/1.0.0', + keywords=['nd2', 'nikon', 'microscopy', 'NIS Elements'], + classifiers=['Development Status :: 5 - Production/Stable', + 'Intended Audience :: Science/Research', + 'License :: Freely Distributable', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.7', + 'Topic :: Scientific/Engineering', + ] ) diff --git a/tests.py b/tests.py deleted file mode 100644 index 12dedb7..0000000 --- a/tests.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Auto-discovers all unittests in the tests directory and runs them - -""" -import unittest -loader = unittest.TestLoader() -tests = loader.discover('tests', pattern='*.py', top_level_dir='.') -testRunner = unittest.TextTestRunner() -testRunner.run(tests) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/model/__init__.py b/tests/model/__init__.py deleted file mode 100644 index 3f42d1b..0000000 --- a/tests/model/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -import unittest -from nd2reader.model import MetadataSet, MetadataItem - - -class MetadataItemTests(unittest.TestCase): - def test_is_valid(self): - data = b'\x0b\x13S\x00L\x00x\x00P\x00i\x00c\x00t\x00u\x00r\x00e\x00M\x00e\x00t\x00a\x00d\x00a\x00t\x00a\x00\x00\x00!\x00\x00\x00\xd5]\x00\x00\x00\x00\x00\x00\x06\nd\x00T\x00i\x00m\x00e\x00M\x00S\x00e\x00c\x00\x00\x00\x00\xc0T\x1c\x9b#\xbb@\x06\x0ed\x00T\x00i\x00m\x00e\x00A\x00b\x00s\x00o\x00l\x00u\x00t\x00e\x00\x00\x00Sf\xf5\xa6\xa7\xbeBA\x02\x0ce\x00T\x00i\x00m\x00e\x00S\x00o\x00u\x00r\x00c\x00e\x00\x00\x00\x00\x00\x00\x00\x06\x06d\x00X\x00P\x00o\x00s\x00\x00\x00\x00\x00\x00\x00\x00$\x9d\xc0\x06\x06d\x00Y\x00P\x00o\x00s\x00\x00\x00\x00\x00\x00\x00\xe0\r\xe5@\x03\x06u\x00i\x00R\x00o\x00w\x00\x00\x00\x00\x00\x00\x00\x03\nu\x00i\x00C\x00o\x00n\x002\x000\x00(\x00L\x00\x00\x00\x00\x00\x00\x00\x06\x06d\x00Z\x00P\x00o\x00s\x00\x00\x00\x9a\x99\x99\x99Y\x8d\xb8@\x01\x0eb\x00Z\x00P\x00o\x00s\x00A\x00b\x00s\x00o\x00l\x00u\x00t\x00e\x00\x00\x00\x01\x06\x07d\x00A\x00n\x00g' - item = MetadataItem(0, data) - self.assertTrue(item.is_valid) - - def test_key(self): - data = b'\x0b\x13S\x00L\x00x\x00P\x00i\x00c\x00t\x00u\x00r\x00e\x00M\x00e\x00t\x00a\x00d\x00a\x00t\x00a\x00\x00\x00!\x00\x00\x00\xd5]\x00\x00\x00\x00\x00\x00\x06\nd\x00T\x00i\x00m\x00e\x00M\x00S\x00e\x00c\x00\x00\x00\x00\xc0T\x1c\x9b#\xbb@\x06\x0ed\x00T\x00i\x00m\x00e\x00A\x00b\x00s\x00o\x00l\x00u\x00t\x00e\x00\x00\x00Sf\xf5\xa6\xa7\xbeBA\x02\x0ce\x00T\x00i\x00m\x00e\x00S\x00o\x00u\x00r\x00c\x00e\x00\x00\x00\x00\x00\x00\x00\x06\x06d\x00X\x00P\x00o\x00s\x00\x00\x00\x00\x00\x00\x00\x00$\x9d\xc0\x06\x06d\x00Y\x00P\x00o\x00s\x00\x00\x00\x00\x00\x00\x00\xe0\r\xe5@\x03\x06u\x00i\x00R\x00o\x00w\x00\x00\x00\x00\x00\x00\x00\x03\nu\x00i\x00C\x00o\x00n\x002\x000\x00(\x00L\x00\x00\x00\x00\x00\x00\x00\x06\x06d\x00Z\x00P\x00o\x00s\x00\x00\x00\x9a\x99\x99\x99Y\x8d\xb8@\x01\x0eb\x00Z\x00P\x00o\x00s\x00A\x00b\x00s\x00o\x00l\x00u\x00t\x00e\x00\x00\x00\x01\x06\x07d\x00A\x00n\x00g' - item = MetadataItem(0, data) - self.assertEqual(item.key, "SLxPictureMetadata") - - def test_parse_double(self): - data = b'\x06\nd\x00T\x00i\x00m\x00e\x00M\x00S\x00e\x00c\x00\x00\x00\x00\xc0T\x1c\x9b#\xbb@\x06\x0e' - item = MetadataItem(0, data) - self.assertEqual(item.value, 6947.605901047587) - diff --git a/tests/service/__init__.py b/tests/service/__init__.py deleted file mode 100644 index e69de29..0000000