From 4533a9241122389e760d0d2cfa9a3df1e397bc46 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Mon, 20 Apr 2015 21:37:30 -0500 Subject: [PATCH 01/39] created dev branch --- .idea/inspectionProfiles/profiles_settings.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml index 3b31283..c60c33b 100644 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -1,7 +1,7 @@ - \ No newline at end of file From 9b0910fd5be25d04f07e050ae9fb9edd1d0a76d2 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Mon, 20 Apr 2015 21:39:53 -0500 Subject: [PATCH 02/39] resolves #7 - deleted pycharm metadata --- .idea/dictionaries/jim.xml | 7 ------- .idea/inspectionProfiles/Project_Default.xml | 16 ---------------- .idea/inspectionProfiles/profiles_settings.xml | 7 ------- 3 files changed, 30 deletions(-) delete mode 100644 .idea/dictionaries/jim.xml delete mode 100644 .idea/inspectionProfiles/Project_Default.xml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml diff --git a/.idea/dictionaries/jim.xml b/.idea/dictionaries/jim.xml deleted file mode 100644 index a26e67a..0000000 --- a/.idea/dictionaries/jim.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - filemap - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 2b29e40..0000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index c60c33b..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - \ No newline at end of file From b5a84b805987914b8de80e69ace981881d58eb4f Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 26 Apr 2015 18:16:11 -0500 Subject: [PATCH 03/39] resolves #5: removed two unnecessary array typecasting operations --- nd2reader/model/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 9c15ac2..1ae84e4 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -86,8 +86,8 @@ class Image(object): @property def data(self): if self._data is None: - # The data is just a flat, 1-dimensional array. We convert it to a 2D array and cast the data points as 16-bit integers - self._data = np.reshape(self._raw_data, (self._height, self._width)).astype(np.int64).astype(np.uint16) + # The data is just a flat, 1-dimensional array. We convert it to a 2D image here. + self._data = np.reshape(self._raw_data, (self._height, self._width)) return self._data @property From 7fcf2a5740ece7589b65a7fe96ba22fff1fa8195 Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 26 Apr 2015 18:51:41 -0500 Subject: [PATCH 04/39] #11 - partial refactor --- nd2reader/__init__.py | 84 ++++++++- nd2reader/model/__init__.py | 174 +----------------- nd2reader/{service/__init__.py => reader.py} | 182 ++----------------- 3 files changed, 94 insertions(+), 346 deletions(-) rename nd2reader/{service/__init__.py => reader.py} (60%) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 264eb55..92281c7 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -1,18 +1,25 @@ +# -*- coding: utf-8 -*- + +from collections import namedtuple +from nd2reader.model import Channel import logging -from nd2reader.service import BaseNd2 from nd2reader.model import Image, ImageSet +from nd2reader.reader import Nd2FileReader + +chunk = namedtuple('Chunk', ['location', 'length']) +field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) -class Nd2(BaseNd2): +class Nd2(Nd2FileReader): def __init__(self, filename): super(Nd2, self).__init__(filename) def get_image(self, time_index, fov, channel_name, z_level): image_set_number = self._calculate_image_set_number(time_index, fov, z_level) - timestamp, raw_image_data = self._reader.get_raw_image_data(image_set_number, self.channel_offset[channel_name]) + timestamp, raw_image_data = self.get_raw_image_data(image_set_number, self.channel_offset[channel_name]) return Image(timestamp, raw_image_data, fov, channel_name, z_level, self.height, self.width) def __iter__(self): @@ -44,4 +51,73 @@ class Nd2(BaseNd2): image = self.get_image(timepoint, field_of_view, channel_name, z_level) if image.is_valid: image_set.add(image) - yield image_set \ No newline at end of file + yield image_set + + self._channel_offset = None + + @property + def height(self): + """ + :return: height of each image, in pixels + + """ + return self._metadata['ImageAttributes']['SLxImageAttributes']['uiHeight'] + + @property + def width(self): + """ + :return: width of each image, in pixels + + """ + return self._metadata['ImageAttributes']['SLxImageAttributes']['uiWidth'] + + @property + def channels(self): + metadata = self._metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] + try: + validity = self._metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] + except KeyError: + # If none of the channels have been deleted, there is no validity list, so we just make one + validity = [True for i in metadata] + # Channel information is contained in dictionaries with the keys a0, a1...an where the number + # indicates the order in which the channel is stored. So by sorting the dicts alphabetically + # we get the correct order. + for (label, chan), valid in zip(sorted(metadata['sPlaneNew'].items()), validity): + if not valid: + continue + name = chan['sDescription'] + exposure_time = metadata['sSampleSetting'][label]['dExposureTime'] + camera = metadata['sSampleSetting'][label]['pCameraSetting']['CameraUserName'] + yield Channel(name, camera, exposure_time) + + @property + def channel_names(self): + """ + A convenience method for getting an alphabetized list of channel names. + + :return: list[str] + + """ + for channel in sorted(self.channels, key=lambda x: x.name): + yield channel.name + + @property + def _image_count(self): + return self._metadata['ImageAttributes']['SLxImageAttributes']['uiSequenceCount'] + + @property + def _sequence_count(self): + return self._metadata['ImageEvents']['RLxExperimentRecord']['uiCount'] + + @property + def channel_offset(self): + if self._channel_offset is None: + self._channel_offset = {} + for n, channel in enumerate(self.channels): + self._channel_offset[channel.name] = n + return self._channel_offset + + def _calculate_image_set_number(self, time_index, fov, z_level): + return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) + + diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 1ae84e4..39227a1 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -1,10 +1,6 @@ import numpy as np import skimage.io import logging -from io import BytesIO -import array -import struct - log = logging.getLogger(__name__) @@ -96,172 +92,4 @@ class Image(object): def show(self): skimage.io.imshow(self.data) - skimage.io.show() - - -class MetadataItem(object): - def __init__(self, start, data): - self._datatype = ord(data[start]) - self._label_length = 2 * ord(data[start + 1]) - self._data = data - - @property - def is_valid(self): - return self._datatype > 0 - - @property - def key(self): - return self._data[2:self._label_length].decode("utf16").encode("utf8") - - @property - def length(self): - return self._length - - @property - def data_start(self): - return self._label_length + 2 - - @property - def _body(self): - """ - All data after the header. - - """ - return self._data[self.data_start:] - - def _get_bytes(self, count): - return self._data[self.data_start: self.data_start + count] - - @property - def value(self): - parser = {1: self._parse_unsigned_char, - 2: self._parse_unsigned_int, - 3: self._parse_unsigned_int, - 5: self._parse_unsigned_long, - 6: self._parse_double, - 8: self._parse_string, - 9: self._parse_char_array, - 11: self._parse_metadata_item - } - return parser[self._datatype]() - - def _parse_unsigned_char(self): - self._length = 1 - return self._unpack("B", self._get_bytes(self._length)) - - def _parse_unsigned_int(self): - self._length = 4 - return self._unpack("I", self._get_bytes(self._length)) - - def _parse_unsigned_long(self): - self._length = 8 - return self._unpack("Q", self._get_bytes(self._length)) - - def _parse_double(self): - self._length = 8 - return self._unpack("d", self._get_bytes(self._length)) - - def _parse_string(self): - # the string is of unknown length but ends at the first instance of \x00\x00 - stop = self._body.index("\x00\x00") - self._length = stop - return self._body[:stop - 1].decode("utf16").encode("utf8") - - def _parse_char_array(self): - array_length = self._unpack("Q", self._get_bytes(8)) - self._length = array_length + 8 - return array.array("B", self._body[8:array_length]) - - def _parse_metadata_item(self): - count, length = struct.unpack(" Date: Fri, 8 May 2015 02:57:08 +0000 Subject: [PATCH 05/39] added requirements file --- nd2reader/__init__.py | 3 ++- requirements.txt | 1 + setup.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 requirements.txt diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 92281c7..a01b3b6 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -9,8 +9,9 @@ from nd2reader.reader import Nd2FileReader chunk = namedtuple('Chunk', ['location', 'length']) field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) +print(__name__) log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) +log.setLevel(logging.WARN) class Nd2(Nd2FileReader): diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..296d654 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +numpy \ No newline at end of file diff --git a/setup.py b/setup.py index b7acbd1..93db35b 100644 --- a/setup.py +++ b/setup.py @@ -7,4 +7,4 @@ setup( install_requires=[ 'numpy', ], -) +) \ No newline at end of file From 26ff38d039cf16f09c7c86cfae77b816358ff164 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Thu, 7 May 2015 23:08:42 -0500 Subject: [PATCH 06/39] created dockerfile --- Dockerfile | 24 ++++++++++++++++++++++++ nd2reader/reader.py | 4 +++- setup.py | 5 +---- 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..95602c0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu +MAINTAINER Jim Rybarski + +RUN apt-get update && apt-get install -y \ + gcc \ + gfortran \ + libblas-dev \ + liblapack-dev \ + libatlas-dev \ + tk \ + tk-dev \ + libpng12-dev \ + python \ + python-dev \ + python-pip \ + libfreetype6-dev \ + python-skimage + +RUN pip install numpy +RUN pip install --upgrade scikit-image + +COPY . /opt/nd2reader +WORKDIR /opt/nd2reader +RUN python setup.py install diff --git a/nd2reader/reader.py b/nd2reader/reader.py index 3b3732a..d269bf3 100644 --- a/nd2reader/reader.py +++ b/nd2reader/reader.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import array import numpy as np import struct @@ -280,4 +282,4 @@ class Nd2FileReader(object): res[name].append(value) x = data.read() assert not x, "skip %d %s" % (len(x), repr(x[:30])) - return res \ No newline at end of file + return res diff --git a/setup.py b/setup.py index 93db35b..5107f45 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,5 @@ from setuptools import setup, find_packages setup( name="nd2reader", packages=find_packages(), - version="0.9.7", - install_requires=[ - 'numpy', - ], + version="0.9.7" ) \ No newline at end of file From de8915fd6b16475c1773d31156d540567fe12cf6 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 10 May 2015 18:33:18 +0000 Subject: [PATCH 07/39] simplified even more --- nd2reader/__init__.py | 64 ++++++----------- nd2reader/reader.py | 162 ++++++++++++++++++++---------------------- 2 files changed, 98 insertions(+), 128 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index a01b3b6..00648d9 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -1,16 +1,14 @@ # -*- coding: utf-8 -*- -from collections import namedtuple from nd2reader.model import Channel +from datetime import datetime import logging from nd2reader.model import Image, ImageSet from nd2reader.reader import Nd2FileReader -chunk = namedtuple('Chunk', ['location', 'length']) -field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) -print(__name__) log = logging.getLogger(__name__) +log.addHandler(logging.StreamHandler()) log.setLevel(logging.WARN) @@ -18,11 +16,6 @@ class Nd2(Nd2FileReader): def __init__(self, filename): super(Nd2, self).__init__(filename) - def get_image(self, time_index, fov, channel_name, z_level): - image_set_number = self._calculate_image_set_number(time_index, fov, z_level) - timestamp, raw_image_data = self.get_raw_image_data(image_set_number, self.channel_offset[channel_name]) - return Image(timestamp, raw_image_data, fov, channel_name, z_level, self.height, self.width) - def __iter__(self): """ Just return every image in order (might not be exactly the order that the images were physically taken, but it will @@ -57,20 +50,8 @@ class Nd2(Nd2FileReader): self._channel_offset = None @property - def height(self): - """ - :return: height of each image, in pixels - - """ - return self._metadata['ImageAttributes']['SLxImageAttributes']['uiHeight'] - - @property - def width(self): - """ - :return: width of each image, in pixels - - """ - return self._metadata['ImageAttributes']['SLxImageAttributes']['uiWidth'] + def metadata(self): + return self._metadata @property def channels(self): @@ -103,22 +84,21 @@ class Nd2(Nd2FileReader): yield channel.name @property - def _image_count(self): - return self._metadata['ImageAttributes']['SLxImageAttributes']['uiSequenceCount'] - - @property - def _sequence_count(self): - return self._metadata['ImageEvents']['RLxExperimentRecord']['uiCount'] - - @property - def channel_offset(self): - if self._channel_offset is None: - self._channel_offset = {} - for n, channel in enumerate(self.channels): - self._channel_offset[channel.name] = n - return self._channel_offset - - def _calculate_image_set_number(self, time_index, fov, z_level): - return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) - - + def absolute_start(self): + if self._absolute_start is None: + for line in self._metadata['ImageTextInfo']['SLxImageTextInfo'].values(): + absolute_start_12 = None + absolute_start_24 = None + # ND2s seem to randomly switch between 12- and 24-hour representations. + try: + absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") + except ValueError: + pass + try: + absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") + except ValueError: + pass + if not absolute_start_12 and not absolute_start_24: + continue + self._absolute_start = absolute_start_12 if absolute_start_12 else absolute_start_24 + return self._absolute_start \ No newline at end of file diff --git a/nd2reader/reader.py b/nd2reader/reader.py index d269bf3..20ab5d2 100644 --- a/nd2reader/reader.py +++ b/nd2reader/reader.py @@ -1,11 +1,15 @@ # -*- coding: utf-8 -*- +from abc import abstractproperty import array +from collections import namedtuple import numpy as np import struct import re from StringIO import StringIO -from datetime import datetime +from nd2reader.model import Image + +field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) class Nd2FileReader(object): @@ -17,6 +21,7 @@ class Nd2FileReader(object): self._absolute_start = None self._filename = filename self._file_handler = None + self._channel_offset = None self._chunk_map_start_location = None self._label_map = {} self._metadata = {} @@ -24,6 +29,31 @@ class Nd2FileReader(object): self._parse_dict_data() self.__dimensions = None + def get_image(self, time_index, fov, channel_name, z_level): + image_set_number = self._calculate_image_set_number(time_index, fov, z_level) + timestamp, raw_image_data = self.get_raw_image_data(image_set_number, self.channel_offset[channel_name]) + return Image(timestamp, raw_image_data, fov, channel_name, z_level, self.height, self.width) + + @abstractproperty + def channels(self): + raise NotImplemented + + @property + def height(self): + """ + :return: height of each image, in pixels + + """ + return self._metadata['ImageAttributes']['SLxImageAttributes']['uiHeight'] + + @property + def width(self): + """ + :return: width of each image, in pixels + + """ + return self._metadata['ImageAttributes']['SLxImageAttributes']['uiWidth'] + @property def _dimensions(self): if self.__dimensions is None: @@ -40,30 +70,6 @@ class Nd2FileReader(object): break return self.__dimensions - @property - def absolute_start(self): - if self._absolute_start is None: - for line in self._metadata['ImageTextInfo']['SLxImageTextInfo'].values(): - absolute_start_12 = None - absolute_start_24 = None - - # ND2s seem to randomly switch between 12- and 24-hour representations. - try: - absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") - except ValueError: - pass - - try: - absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") - except ValueError: - pass - - if not absolute_start_12 and not absolute_start_24: - continue - - self._absolute_start = absolute_start_12 if absolute_start_12 else absolute_start_24 - - return self._absolute_start @property def fh(self): @@ -74,8 +80,11 @@ class Nd2FileReader(object): @property def time_index_count(self): """ - The number of images for a given field of view, channel, and z_level combination. - Effectively the number of frames. + The number of image sets. If images were acquired using some kind of cycle, all images at each step in the + program will have the same timestamp (even though they may have varied by a few seconds in reality). For example, + if you have four fields of view that you're constantly monitoring, and you take a bright field and GFP image of + each, and you repeat that process 100 times, you'll have 800 individual images. But there will only be 400 + time indexes. :rtype: int @@ -125,35 +134,49 @@ class Nd2FileReader(object): else: return count + @property + def _image_count(self): + return self._metadata['ImageAttributes']['SLxImageAttributes']['uiSequenceCount'] + + @property + def _sequence_count(self): + return self._metadata['ImageEvents']['RLxExperimentRecord']['uiCount'] + + @property + def channel_offset(self): + """ + Image data is interleaved for each image set. That is, if there are four images in a set, the first image + will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth. Why this would be the + case is beyond me, but that's how it works. + + """ + if self._channel_offset is None: + self._channel_offset = {} + for n, channel in enumerate(self.channels): + self._channel_offset[channel.name] = n + return self._channel_offset + + def _calculate_image_set_number(self, time_index, fov, z_level): + return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) + def get_raw_image_data(self, image_set_number, channel_offset): chunk = self._label_map["ImageDataSeq|%d!" % image_set_number] - data = self._read_chunk(chunk.location) + data = self._read_chunk(chunk) timestamp = struct.unpack("d", data[:8])[0] - # The images for the various channels are interleaved within each other. Yes, this is an incredibly unintuitive and nonsensical way - # to store data. + # The images for the various channels are interleaved within each other. image_data = array.array("H", data) image_data_start = 4 + channel_offset return timestamp, image_data[image_data_start::self.channel_count] def _parse_dict_data(self): # TODO: Don't like this name - for label in self._top_level_dict_labels: - chunk_location = self._label_map[label].location - data = self._read_chunk(chunk_location) + for label in self._label_map.keys(): + if not label.endswith("LV!") or "LV|" in label: + continue + data = self._read_chunk(self._label_map[label]) stop = label.index("LV") self._metadata[label[:stop]] = self.read_lv_encoding(data, 1) - @property - def metadata(self): - return self._metadata - - @property - def _top_level_dict_labels(self): - # TODO: I don't like this name either - for label in self._label_map.keys(): - if label.endswith("LV!") or "LV|" in label: - yield label - def _read_map(self): """ Every label ends with an exclamation point, however, we can't directly search for those to find all the labels @@ -171,13 +194,10 @@ class Nd2FileReader(object): data_start = raw_text.index("!", label_start) + 1 key = raw_text[label_start: data_start] location, length = struct.unpack("QQ", raw_text[data_start: data_start + 16]) - label, value = key, chunk(location=location, length=length) - - if label == "ND2 CHUNK MAP SIGNATURE 0000001!": + if key == "ND2 CHUNK MAP SIGNATURE 0000001!": # We've reached the end of the chunk map break - - self._label_map[label] = value + self._label_map[key] = location label_start = data_start + 16 def _read_chunk(self, chunk_location): @@ -186,53 +206,23 @@ class Nd2FileReader(object): """ self.fh.seek(chunk_location) - chunk_data = self._read_chunk_metadata() - header, relative_offset, data_length = self._parse_chunk_metadata(chunk_data) - return self._read_chunk_data(chunk_location, relative_offset, data_length) - - def _read_chunk_metadata(self): - """ - Gets the chunks metadata, which is always 16 bytes - - """ - return self.fh.read(16) - - def _read_chunk_data(self, chunk_location, relative_offset, data_length): - """ - Reads the actual data for a given chunk - - """ + # The chunk metadata is always 16 bytes long + chunk_metadata = self.fh.read(16) + header, relative_offset, data_length = struct.unpack("IIQ", chunk_metadata) + if header != 0xabeceda: + raise ValueError("The ND2 file seems to be corrupted.") # We start at the location of the chunk metadata, skip over the metadata, and then proceed to the # start of the actual data field, which is at some arbitrary place after the metadata. self.fh.seek(chunk_location + 16 + relative_offset) return self.fh.read(data_length) - @staticmethod - def _parse_chunk_metadata(chunk_data): - """ - Finds out everything about a given chunk. Every chunk begins with the same value, so if that's ever - different we can assume the file has suffered some kind of damage. - - """ - header, relative_offset, data_length = struct.unpack("IIQ", chunk_data) - if header != 0xabeceda: - raise ValueError("The ND2 file seems to be corrupted.") - return header, relative_offset, data_length - - def _get_raw_chunk_map_text(self): - """ - Reads the entire chunk map and returns it as a string. - - """ - - @staticmethod def as_numpy_array(arr): return np.frombuffer(arr) def _z_level_count(self): name = "CustomData|Z!" - st = self._read_chunk(self._label_map[name].location) + st = self._read_chunk(self._label_map[name]) res = array.array("d", st) return len(res) @@ -282,4 +272,4 @@ class Nd2FileReader(object): res[name].append(value) x = data.read() assert not x, "skip %d %s" % (len(x), repr(x[:30])) - return res + return res \ No newline at end of file From 5930c63e511e7d96869395a5a6eea8600ae41e6c Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 10 May 2015 15:08:30 -0500 Subject: [PATCH 08/39] #11 made parser section more readable --- nd2reader/reader.py | 61 +++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/nd2reader/reader.py b/nd2reader/reader.py index 20ab5d2..fecf17f 100644 --- a/nd2reader/reader.py +++ b/nd2reader/reader.py @@ -228,48 +228,49 @@ class Nd2FileReader(object): def read_lv_encoding(self, data, count): data = StringIO(data) - res = {} + metadata = {} total_count = 0 - for c in range(count): - lastpos = data.tell() + for _ in xrange(count): + cursor_position = data.tell() total_count += 1 - hdr = data.read(2) - if not hdr: + header = data.read(2) + if not header: break - typ = ord(hdr[0]) - bname = data.read(2*ord(hdr[1])) - name = bname.decode("utf16")[:-1].encode("utf8") - if typ == 1: + data_type, name_length = map(ord, header) + name = data.read(name_length * 2).decode("utf16")[:-1].encode("utf8") + if data_type == 1: value, = struct.unpack("B", data.read(1)) - elif typ in [2, 3]: + elif data_type in [2, 3]: value, = struct.unpack("I", data.read(4)) - elif typ == 5: + elif data_type == 5: value, = struct.unpack("Q", data.read(8)) - elif typ == 6: + elif data_type == 6: value, = struct.unpack("d", data.read(8)) - elif typ == 8: + elif data_type == 8: value = data.read(2) while value[-2:] != "\x00\x00": value += data.read(2) value = value.decode("utf16")[:-1].encode("utf8") - elif typ == 9: + elif data_type == 9: cnt, = struct.unpack("Q", data.read(8)) value = array.array("B", data.read(cnt)) - elif typ == 11: - newcount, length = struct.unpack(" Date: Sun, 10 May 2015 16:09:21 -0500 Subject: [PATCH 09/39] #11 eliminated the messy parts of the parser, mostly --- nd2reader/reader.py | 107 ++++++++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 43 deletions(-) diff --git a/nd2reader/reader.py b/nd2reader/reader.py index fecf17f..1b1401d 100644 --- a/nd2reader/reader.py +++ b/nd2reader/reader.py @@ -13,6 +13,10 @@ field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) class Nd2FileReader(object): + CHUNK_HEADER = 0xabeceda + CHUNK_MAP_START = "ND2 FILEMAP SIGNATURE NAME 0001!" + CHUNK_MAP_END = "ND2 CHUNK MAP SIGNATURE 0000001!" + """ Reads .nd2 files, provides an interface to the metadata, and generates numpy arrays from the image data. @@ -23,12 +27,17 @@ class Nd2FileReader(object): self._file_handler = None self._channel_offset = None self._chunk_map_start_location = None + self._cursor_position = None self._label_map = {} self._metadata = {} self._read_map() - self._parse_dict_data() + self._parse_metadata() self.__dimensions = None + @staticmethod + def as_numpy_array(arr): + return np.frombuffer(arr) + def get_image(self, time_index, fov, channel_name, z_level): image_set_number = self._calculate_image_set_number(time_index, fov, z_level) timestamp, raw_image_data = self.get_raw_image_data(image_set_number, self.channel_offset[channel_name]) @@ -168,14 +177,13 @@ class Nd2FileReader(object): image_data_start = 4 + channel_offset return timestamp, image_data[image_data_start::self.channel_count] - def _parse_dict_data(self): - # TODO: Don't like this name + def _parse_metadata(self): for label in self._label_map.keys(): if not label.endswith("LV!") or "LV|" in label: continue data = self._read_chunk(self._label_map[label]) stop = label.index("LV") - self._metadata[label[:stop]] = self.read_lv_encoding(data, 1) + self._metadata[label[:stop]] = self._read_file(data, 1) def _read_map(self): """ @@ -188,13 +196,13 @@ class Nd2FileReader(object): chunk_map_start_location = struct.unpack("Q", self.fh.read(8))[0] self.fh.seek(chunk_map_start_location) raw_text = self.fh.read(-1) - label_start = raw_text.index("ND2 FILEMAP SIGNATURE NAME 0001!") + 32 + label_start = raw_text.index(Nd2FileReader.CHUNK_MAP_START) + 32 while True: data_start = raw_text.index("!", label_start) + 1 key = raw_text[label_start: data_start] location, length = struct.unpack("QQ", raw_text[data_start: data_start + 16]) - if key == "ND2 CHUNK MAP SIGNATURE 0000001!": + if key == Nd2FileReader.CHUNK_MAP_END: # We've reached the end of the chunk map break self._label_map[key] = location @@ -209,58 +217,73 @@ class Nd2FileReader(object): # The chunk metadata is always 16 bytes long chunk_metadata = self.fh.read(16) header, relative_offset, data_length = struct.unpack("IIQ", chunk_metadata) - if header != 0xabeceda: + if header != Nd2FileReader.CHUNK_HEADER: raise ValueError("The ND2 file seems to be corrupted.") # We start at the location of the chunk metadata, skip over the metadata, and then proceed to the # start of the actual data field, which is at some arbitrary place after the metadata. self.fh.seek(chunk_location + 16 + relative_offset) return self.fh.read(data_length) - @staticmethod - def as_numpy_array(arr): - return np.frombuffer(arr) - def _z_level_count(self): name = "CustomData|Z!" st = self._read_chunk(self._label_map[name]) - res = array.array("d", st) - return len(res) - - def read_lv_encoding(self, data, count): + return len(array.array("d", st)) + + def _parse_unsigned_char(self, data): + return struct.unpack("B", data.read(1))[0] + + def _parse_unsigned_int(self, data): + return struct.unpack("I", data.read(4))[0] + + def _parse_unsigned_long(self, data): + return struct.unpack("Q", data.read(8))[0] + + def _parse_double(self, data): + return struct.unpack("d", data.read(8))[0] + + def _parse_string(self, data): + value = data.read(2) + while not value.endswith("\x00\x00"): + # the string ends at the first instance of \x00\x00 + value += data.read(2) + return value.decode("utf16")[:-1].encode("utf8") + + def _parse_char_array(self, data): + array_length = struct.unpack("Q", data.read(8))[0] + return array.array("B", data.read(array_length)) + + def _parse_metadata_item(self, args): + data, cursor_position = args + new_count, length = struct.unpack(" Date: Mon, 11 May 2015 12:50:03 +0000 Subject: [PATCH 10/39] much refactor. very change --- nd2reader/__init__.py | 167 +++++++++++++++----- nd2reader/model/__init__.py | 19 --- nd2reader/parser.py | 179 ++++++++++++++++++++++ nd2reader/reader.py | 297 ------------------------------------ 4 files changed, 305 insertions(+), 357 deletions(-) create mode 100644 nd2reader/parser.py delete mode 100644 nd2reader/reader.py diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 00648d9..e5d4839 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -1,27 +1,31 @@ # -*- coding: utf-8 -*- -from nd2reader.model import Channel +import array from datetime import datetime import logging from nd2reader.model import Image, ImageSet -from nd2reader.reader import Nd2FileReader - +from nd2reader.parser import Nd2Parser +import numpy as np +import re +import struct log = logging.getLogger(__name__) log.addHandler(logging.StreamHandler()) log.setLevel(logging.WARN) -class Nd2(Nd2FileReader): - def __init__(self, filename): +class Nd2(Nd2Parser): + def __init__(self, filename, image_sets=False): super(Nd2, self).__init__(filename) + self._use_image_sets = image_sets def __iter__(self): - """ - Just return every image in order (might not be exactly the order that the images were physically taken, but it will - be within a few seconds). A better explanation is probably needed here. + if self._use_image_sets: + return self.image_sets() + else: + return self.images() - """ + def images(self): for i in range(self._image_count): for fov in range(self.field_of_view_count): for z_level in range(self.z_level_count): @@ -30,34 +34,27 @@ class Nd2(Nd2FileReader): if image.is_valid: yield image - def image_sets(self, field_of_view, time_indices=None, channels=None, z_levels=None): - """ - Gets all the images for a given field of view and - """ - timepoint_set = xrange(self.time_index_count) if time_indices is None else time_indices - channel_set = [channel.name for channel in self.channels] if channels is None else channels - z_level_set = xrange(self.z_level_count) if z_levels is None else z_levels - - for timepoint in timepoint_set: + def image_sets(self): + for time_index in xrange(self.time_index_count): image_set = ImageSet() - for channel_name in channel_set: - for z_level in z_level_set: - image = self.get_image(timepoint, field_of_view, channel_name, z_level) - if image.is_valid: - image_set.add(image) + for fov in range(self.field_of_view_count): + for channel_name in self.channels: + for z_level in xrange(self.z_level_count): + image = self.get_image(time_index, fov, channel_name, z_level) + if image.is_valid: + image_set.add(image) yield image_set - self._channel_offset = None - - @property - def metadata(self): - return self._metadata + def get_image(self, time_index, fov, channel_name, z_level): + image_set_number = self._calculate_image_set_number(time_index, fov, z_level) + timestamp, raw_image_data = self._get_raw_image_data(image_set_number, self._channel_offset[channel_name]) + return Image(timestamp, raw_image_data, fov, channel_name, z_level, self.height, self.width) @property def channels(self): - metadata = self._metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] + metadata = self.metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] try: - validity = self._metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] + validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] except KeyError: # If none of the channels have been deleted, there is no validity list, so we just make one validity = [True for i in metadata] @@ -67,26 +64,28 @@ class Nd2(Nd2FileReader): for (label, chan), valid in zip(sorted(metadata['sPlaneNew'].items()), validity): if not valid: continue - name = chan['sDescription'] - exposure_time = metadata['sSampleSetting'][label]['dExposureTime'] - camera = metadata['sSampleSetting'][label]['pCameraSetting']['CameraUserName'] - yield Channel(name, camera, exposure_time) + yield chan['sDescription'] @property - def channel_names(self): + def height(self): """ - A convenience method for getting an alphabetized list of channel names. + :return: height of each image, in pixels - :return: list[str] + """ + return self.metadata['ImageAttributes']['SLxImageAttributes']['uiHeight'] + @property + def width(self): """ - for channel in sorted(self.channels, key=lambda x: x.name): - yield channel.name + :return: width of each image, in pixels + + """ + return self.metadata['ImageAttributes']['SLxImageAttributes']['uiWidth'] @property def absolute_start(self): if self._absolute_start is None: - for line in self._metadata['ImageTextInfo']['SLxImageTextInfo'].values(): + for line in self.metadata['ImageTextInfo']['SLxImageTextInfo'].values(): absolute_start_12 = None absolute_start_24 = None # ND2s seem to randomly switch between 12- and 24-hour representations. @@ -101,4 +100,90 @@ class Nd2(Nd2FileReader): if not absolute_start_12 and not absolute_start_24: continue self._absolute_start = absolute_start_12 if absolute_start_12 else absolute_start_24 - return self._absolute_start \ No newline at end of file + return self._absolute_start + + @property + def channel_count(self): + pattern = r""".*?λ\((\d+)\).*?""" + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return 1 + else: + return count + + @property + def field_of_view_count(self): + """ + The metadata contains information about fields of view, but it contains it even if some fields + of view were cropped. We can't find anything that states which fields of view are actually + in the image data, so we have to calculate it. There probably is something somewhere, since + NIS Elements can figure it out, but we haven't found it yet. + + """ + pattern = r""".*?XY\((\d+)\).*?""" + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return 1 + else: + return count + + @property + def time_index_count(self): + """ + The number of image sets. If images were acquired using some kind of cycle, all images at each step in the + program will have the same timestamp (even though they may have varied by a few seconds in reality). For example, + if you have four fields of view that you're constantly monitoring, and you take a bright field and GFP image of + each, and you repeat that process 100 times, you'll have 800 individual images. But there will only be 400 + time indexes. + + :rtype: int + + """ + pattern = r""".*?T'\((\d+)\).*?""" + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return 1 + else: + return count + + @property + def z_level_count(self): + pattern = r""".*?Z\((\d+)\).*?""" + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return 1 + else: + return count + + @staticmethod + def as_numpy_array(arr): + return np.frombuffer(arr) + + @property + def _channel_offset(self): + """ + Image data is interleaved for each image set. That is, if there are four images in a set, the first image + will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth. Why this would be the + case is beyond me, but that's how it works. + + """ + channel_offset = {} + for n, channel in enumerate(self.channels): + self._channel_offset[channel.name] = n + return channel_offset + + def _get_raw_image_data(self, image_set_number, channel_offset): + chunk = self._label_map["ImageDataSeq|%d!" % image_set_number] + data = self._read_chunk(chunk) + timestamp = struct.unpack("d", data[:8])[0] + # The images for the various channels are interleaved within each other. + image_data = array.array("H", data) + image_data_start = 4 + channel_offset + return timestamp, image_data[image_data_start::self.channel_count] + + def _calculate_image_set_number(self, time_index, fov, z_level): + return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 39227a1..800cb7b 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -5,25 +5,6 @@ import logging log = logging.getLogger(__name__) -class Channel(object): - def __init__(self, name, camera, exposure_time): - self._name = name - self._camera = camera - self._exposure_time = exposure_time - - @property - def name(self): - return self._name - - @property - def camera(self): - return self._camera - - @property - def exposure_time(self): - return self._exposure_time - - class ImageSet(object): """ A group of images that share the same timestamp. NIS Elements doesn't store a unique timestamp for every diff --git a/nd2reader/parser.py b/nd2reader/parser.py new file mode 100644 index 0000000..e8e3d96 --- /dev/null +++ b/nd2reader/parser.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- + +import array +from collections import namedtuple +import struct +from StringIO import StringIO + +field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) + + +class Nd2Parser(object): + CHUNK_HEADER = 0xabeceda + CHUNK_MAP_START = "ND2 FILEMAP SIGNATURE NAME 0001!" + CHUNK_MAP_END = "ND2 CHUNK MAP SIGNATURE 0000001!" + + """ + Reads .nd2 files, provides an interface to the metadata, and generates numpy arrays from the image data. + + """ + def __init__(self, filename): + self._absolute_start = None + self._filename = filename + self._fh = None + self._chunk_map_start_location = None + self._cursor_position = None + self._dimension_text = None + self._label_map = {} + self.metadata = {} + self._read_map() + self._parse_metadata() + + @property + def _file_handle(self): + if self._fh is None: + self._fh = open(self._filename, "rb") + return self._fh + + @property + def _dimensions(self): + if self._dimension_text is None: + for line in self.metadata['ImageTextInfo']['SLxImageTextInfo'].values(): + if "Dimensions:" in line: + metadata = line + break + else: + raise ValueError("Could not parse metadata dimensions!") + for line in metadata.split("\r\n"): + if line.startswith("Dimensions:"): + self._dimension_text = line + break + else: + raise ValueError("Could not parse metadata dimensions!") + return self._dimension_text + + @property + def _image_count(self): + return self.metadata['ImageAttributes']['SLxImageAttributes']['uiSequenceCount'] + + @property + def _sequence_count(self): + return self.metadata['ImageEvents']['RLxExperimentRecord']['uiCount'] + + def _parse_metadata(self): + for label in self._label_map.keys(): + if not label.endswith("LV!") or "LV|" in label: + continue + data = self._read_chunk(self._label_map[label]) + stop = label.index("LV") + self.metadata[label[:stop]] = self._read_metadata(data, 1) + + def _read_map(self): + """ + Every label ends with an exclamation point, however, we can't directly search for those to find all the labels + as some of the bytes contain the value 33, which is the ASCII code for "!". So we iteratively find each label, + grab the subsequent data (always 16 bytes long), advance to the next label and repeat. + + """ + self._file_handle.seek(-8, 2) + chunk_map_start_location = struct.unpack("Q", self._file_handle.read(8))[0] + self._file_handle.seek(chunk_map_start_location) + raw_text = self._file_handle.read(-1) + label_start = raw_text.index(Nd2Parser.CHUNK_MAP_START) + 32 + + while True: + data_start = raw_text.index("!", label_start) + 1 + key = raw_text[label_start: data_start] + location, length = struct.unpack("QQ", raw_text[data_start: data_start + 16]) + if key == Nd2Parser.CHUNK_MAP_END: + # We've reached the end of the chunk map + break + self._label_map[key] = location + label_start = data_start + 16 + + def _read_chunk(self, chunk_location): + """ + Gets the data for a given chunk pointer + + """ + self._file_handle.seek(chunk_location) + # The chunk metadata is always 16 bytes long + chunk_metadata = self._file_handle.read(16) + header, relative_offset, data_length = struct.unpack("IIQ", chunk_metadata) + if header != Nd2Parser.CHUNK_HEADER: + raise ValueError("The ND2 file seems to be corrupted.") + # We start at the location of the chunk metadata, skip over the metadata, and then proceed to the + # start of the actual data field, which is at some arbitrary place after the metadata. + self._file_handle.seek(chunk_location + 16 + relative_offset) + return self._file_handle.read(data_length) + + def _z_level_count(self): + st = self._read_chunk(self._label_map["CustomData|Z!"]) + return len(array.array("d", st)) + + def _parse_unsigned_char(self, data): + return struct.unpack("B", data.read(1))[0] + + def _parse_unsigned_int(self, data): + return struct.unpack("I", data.read(4))[0] + + def _parse_unsigned_long(self, data): + return struct.unpack("Q", data.read(8))[0] + + def _parse_double(self, data): + return struct.unpack("d", data.read(8))[0] + + def _parse_string(self, data): + value = data.read(2) + while not value.endswith("\x00\x00"): + # the string ends at the first instance of \x00\x00 + value += data.read(2) + return value.decode("utf16")[:-1].encode("utf8") + + def _parse_char_array(self, data): + array_length = struct.unpack("Q", data.read(8))[0] + return array.array("B", data.read(array_length)) + + def _parse_metadata_item(self, args): + data, cursor_position = args + new_count, length = struct.unpack(" Date: Mon, 11 May 2015 12:54:43 +0000 Subject: [PATCH 11/39] deleted unused method --- nd2reader/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index e5d4839..4f47559 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -5,7 +5,6 @@ from datetime import datetime import logging from nd2reader.model import Image, ImageSet from nd2reader.parser import Nd2Parser -import numpy as np import re import struct @@ -57,7 +56,7 @@ class Nd2(Nd2Parser): validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] except KeyError: # If none of the channels have been deleted, there is no validity list, so we just make one - validity = [True for i in metadata] + validity = [True for _ in metadata] # Channel information is contained in dictionaries with the keys a0, a1...an where the number # indicates the order in which the channel is stored. So by sorting the dicts alphabetically # we get the correct order. @@ -159,10 +158,6 @@ class Nd2(Nd2Parser): else: return count - @staticmethod - def as_numpy_array(arr): - return np.frombuffer(arr) - @property def _channel_offset(self): """ From e995f4b6f4eeec583ba263521d2c0c72577b07bc Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 11 May 2015 09:06:06 -0500 Subject: [PATCH 12/39] resolves #16 removed scikit image reference, deleted obsolete files --- nd2reader/model/__init__.py | 7 +------ tests.py | 9 --------- tests/__init__.py | 0 tests/model/__init__.py | 20 -------------------- tests/service/__init__.py | 0 5 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 tests.py delete mode 100644 tests/__init__.py delete mode 100644 tests/model/__init__.py delete mode 100644 tests/service/__init__.py diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 800cb7b..883972b 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -1,5 +1,4 @@ import numpy as np -import skimage.io import logging log = logging.getLogger(__name__) @@ -69,8 +68,4 @@ class Image(object): @property def is_valid(self): - return np.any(self.data) - - def show(self): - skimage.io.imshow(self.data) - skimage.io.show() \ No newline at end of file + return np.any(self.data) \ No newline at end of file diff --git a/tests.py b/tests.py deleted file mode 100644 index 12dedb7..0000000 --- a/tests.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Auto-discovers all unittests in the tests directory and runs them - -""" -import unittest -loader = unittest.TestLoader() -tests = loader.discover('tests', pattern='*.py', top_level_dir='.') -testRunner = unittest.TextTestRunner() -testRunner.run(tests) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/model/__init__.py b/tests/model/__init__.py deleted file mode 100644 index 3f42d1b..0000000 --- a/tests/model/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -import unittest -from nd2reader.model import MetadataSet, MetadataItem - - -class MetadataItemTests(unittest.TestCase): - def test_is_valid(self): - data = b'\x0b\x13S\x00L\x00x\x00P\x00i\x00c\x00t\x00u\x00r\x00e\x00M\x00e\x00t\x00a\x00d\x00a\x00t\x00a\x00\x00\x00!\x00\x00\x00\xd5]\x00\x00\x00\x00\x00\x00\x06\nd\x00T\x00i\x00m\x00e\x00M\x00S\x00e\x00c\x00\x00\x00\x00\xc0T\x1c\x9b#\xbb@\x06\x0ed\x00T\x00i\x00m\x00e\x00A\x00b\x00s\x00o\x00l\x00u\x00t\x00e\x00\x00\x00Sf\xf5\xa6\xa7\xbeBA\x02\x0ce\x00T\x00i\x00m\x00e\x00S\x00o\x00u\x00r\x00c\x00e\x00\x00\x00\x00\x00\x00\x00\x06\x06d\x00X\x00P\x00o\x00s\x00\x00\x00\x00\x00\x00\x00\x00$\x9d\xc0\x06\x06d\x00Y\x00P\x00o\x00s\x00\x00\x00\x00\x00\x00\x00\xe0\r\xe5@\x03\x06u\x00i\x00R\x00o\x00w\x00\x00\x00\x00\x00\x00\x00\x03\nu\x00i\x00C\x00o\x00n\x002\x000\x00(\x00L\x00\x00\x00\x00\x00\x00\x00\x06\x06d\x00Z\x00P\x00o\x00s\x00\x00\x00\x9a\x99\x99\x99Y\x8d\xb8@\x01\x0eb\x00Z\x00P\x00o\x00s\x00A\x00b\x00s\x00o\x00l\x00u\x00t\x00e\x00\x00\x00\x01\x06\x07d\x00A\x00n\x00g' - item = MetadataItem(0, data) - self.assertTrue(item.is_valid) - - def test_key(self): - data = b'\x0b\x13S\x00L\x00x\x00P\x00i\x00c\x00t\x00u\x00r\x00e\x00M\x00e\x00t\x00a\x00d\x00a\x00t\x00a\x00\x00\x00!\x00\x00\x00\xd5]\x00\x00\x00\x00\x00\x00\x06\nd\x00T\x00i\x00m\x00e\x00M\x00S\x00e\x00c\x00\x00\x00\x00\xc0T\x1c\x9b#\xbb@\x06\x0ed\x00T\x00i\x00m\x00e\x00A\x00b\x00s\x00o\x00l\x00u\x00t\x00e\x00\x00\x00Sf\xf5\xa6\xa7\xbeBA\x02\x0ce\x00T\x00i\x00m\x00e\x00S\x00o\x00u\x00r\x00c\x00e\x00\x00\x00\x00\x00\x00\x00\x06\x06d\x00X\x00P\x00o\x00s\x00\x00\x00\x00\x00\x00\x00\x00$\x9d\xc0\x06\x06d\x00Y\x00P\x00o\x00s\x00\x00\x00\x00\x00\x00\x00\xe0\r\xe5@\x03\x06u\x00i\x00R\x00o\x00w\x00\x00\x00\x00\x00\x00\x00\x03\nu\x00i\x00C\x00o\x00n\x002\x000\x00(\x00L\x00\x00\x00\x00\x00\x00\x00\x06\x06d\x00Z\x00P\x00o\x00s\x00\x00\x00\x9a\x99\x99\x99Y\x8d\xb8@\x01\x0eb\x00Z\x00P\x00o\x00s\x00A\x00b\x00s\x00o\x00l\x00u\x00t\x00e\x00\x00\x00\x01\x06\x07d\x00A\x00n\x00g' - item = MetadataItem(0, data) - self.assertEqual(item.key, "SLxPictureMetadata") - - def test_parse_double(self): - data = b'\x06\nd\x00T\x00i\x00m\x00e\x00M\x00S\x00e\x00c\x00\x00\x00\x00\xc0T\x1c\x9b#\xbb@\x06\x0e' - item = MetadataItem(0, data) - self.assertEqual(item.value, 6947.605901047587) - diff --git a/tests/service/__init__.py b/tests/service/__init__.py deleted file mode 100644 index e69de29..0000000 From 3c0e4a45f1d495906da32b892a3a81c435299c93 Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 11 May 2015 09:16:58 -0500 Subject: [PATCH 13/39] resolves #19 - more efficient checking of spacer images --- nd2reader/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 883972b..4d59b4d 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -68,4 +68,4 @@ class Image(object): @property def is_valid(self): - return np.any(self.data) \ No newline at end of file + return np.any(self._raw_data) \ No newline at end of file From bc8afedcf92470eff6e0f470137fd8655cd60590 Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 11 May 2015 11:47:40 -0500 Subject: [PATCH 14/39] resolves #23 - fixed bug due to improperly refactored logic condition --- nd2reader/parser.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index e8e3d96..0aaa705 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -22,7 +22,7 @@ class Nd2Parser(object): self._filename = filename self._fh = None self._chunk_map_start_location = None - self._cursor_position = None + self._cursor_position = 0 self._dimension_text = None self._label_map = {} self.metadata = {} @@ -62,11 +62,10 @@ class Nd2Parser(object): def _parse_metadata(self): for label in self._label_map.keys(): - if not label.endswith("LV!") or "LV|" in label: - continue - data = self._read_chunk(self._label_map[label]) - stop = label.index("LV") - self.metadata[label[:stop]] = self._read_metadata(data, 1) + if label.endswith("LV!") or "LV|" in label: + data = self._read_chunk(self._label_map[label]) + stop = label.index("LV") + self.metadata[label[:stop]] = self._read_metadata(data, 1) def _read_map(self): """ @@ -134,10 +133,9 @@ class Nd2Parser(object): array_length = struct.unpack("Q", data.read(8))[0] return array.array("B", data.read(array_length)) - def _parse_metadata_item(self, args): - data, cursor_position = args + def _parse_metadata_item(self, data): new_count, length = struct.unpack(" Date: Mon, 11 May 2015 12:18:55 -0500 Subject: [PATCH 15/39] fixed typos --- nd2reader/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 4f47559..4e7409b 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -28,8 +28,8 @@ class Nd2(Nd2Parser): for i in range(self._image_count): for fov in range(self.field_of_view_count): for z_level in range(self.z_level_count): - for channel in self.channels: - image = self.get_image(i, fov, channel.name, z_level) + for channel_name in self.channels: + image = self.get_image(i, fov, channel_name, z_level) if image.is_valid: yield image @@ -168,7 +168,7 @@ class Nd2(Nd2Parser): """ channel_offset = {} for n, channel in enumerate(self.channels): - self._channel_offset[channel.name] = n + channel_offset[channel] = n return channel_offset def _get_raw_image_data(self, image_set_number, channel_offset): From c9e53c4ecc3b582cf459eae3fb3e3d960a15724d Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 11 May 2015 14:42:52 -0500 Subject: [PATCH 16/39] attribution to original authors --- LICENSE | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index beceabe..131008d 100644 --- a/LICENSE +++ b/LICENSE @@ -9,4 +9,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program. If not, see . \ No newline at end of file +along with this program. If not, see . + +Author: Jim Rybarski + +Thanks to M.Kauer and B.Kauer who wrote the [SLOTH library](http://pythonhosted.org/SLOTH/_modules/sloth/read_nd2.html), upon which nd2reader is based. \ No newline at end of file From 5286abce4276b122a505a4844818fe1a547c8148 Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 11 May 2015 15:04:09 -0500 Subject: [PATCH 17/39] license and attribution --- CONTRIBUTORS.txt | 4 ++++ LICENSE => LICENSE.txt | 6 +----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 CONTRIBUTORS.txt rename LICENSE => LICENSE.txt (75%) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt new file mode 100644 index 0000000..0db601c --- /dev/null +++ b/CONTRIBUTORS.txt @@ -0,0 +1,4 @@ +Author: Jim Rybarski + +nd2reader is based on the read_nd2 module from the SLOTH library (http://pythonhosted.org/SLOTH/_modules/sloth/read_nd2.html). +Thanks to M.Kauer and B.Kauer for solving the hardest part of this puzzle. \ No newline at end of file diff --git a/LICENSE b/LICENSE.txt similarity index 75% rename from LICENSE rename to LICENSE.txt index 131008d..beceabe 100644 --- a/LICENSE +++ b/LICENSE.txt @@ -9,8 +9,4 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Author: Jim Rybarski - -Thanks to M.Kauer and B.Kauer who wrote the [SLOTH library](http://pythonhosted.org/SLOTH/_modules/sloth/read_nd2.html), upon which nd2reader is based. \ No newline at end of file +along with this program. If not, see . \ No newline at end of file From 99c26b474f88be6580b241dfc98f0a97e3e67c38 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 16 May 2015 23:42:59 +0000 Subject: [PATCH 18/39] added makefile, simplified the Dockerfile --- Dockerfile | 22 +++++-------------- Makefile | 7 ++++++ nd2reader/model/__init__.py | 44 ++++++++++++++++++------------------- nd2reader/parser.py | 8 +++---- 4 files changed, 39 insertions(+), 42 deletions(-) create mode 100644 Makefile diff --git a/Dockerfile b/Dockerfile index 95602c0..51e6ff0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,14 @@ FROM ubuntu MAINTAINER Jim Rybarski +RUN mkdir -p /var/nds2 RUN apt-get update && apt-get install -y \ - gcc \ - gfortran \ - libblas-dev \ - liblapack-dev \ - libatlas-dev \ - tk \ - tk-dev \ - libpng12-dev \ - python \ - python-dev \ - python-pip \ - libfreetype6-dev \ - python-skimage - -RUN pip install numpy -RUN pip install --upgrade scikit-image + python-numpy \ + python-setuptools COPY . /opt/nd2reader WORKDIR /opt/nd2reader RUN python setup.py install +WORKDIR /var/nd2s + +CMD /usr/bin/python2.7 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..30e5e42 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY: build test shell + +build: + docker build -t jimrybarski/nd2reader . + +shell: + docker run --rm -v ~/Documents/nd2s:/var/nd2s -it jimrybarski/nd2reader diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 4d59b4d..64bf398 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -4,27 +4,6 @@ import logging log = logging.getLogger(__name__) -class ImageSet(object): - """ - A group of images that share the same timestamp. NIS Elements doesn't store a unique timestamp for every - image, rather, it stores one for each set of images that share the same field of view and z-axis level. - - """ - def __init__(self): - self._images = [] - - def add(self, image): - """ - :type image: nd2reader.model.Image() - - """ - self._images.append(image) - - def __iter__(self): - for image in self._images: - yield image - - class Image(object): def __init__(self, timestamp, raw_array, field_of_view, channel, z_level, height, width): self._timestamp = timestamp @@ -68,4 +47,25 @@ class Image(object): @property def is_valid(self): - return np.any(self._raw_data) \ No newline at end of file + return np.any(self._raw_data) + + +class ImageSet(object): + """ + A group of images that share the same timestamp. NIS Elements doesn't store a unique timestamp for every + image, rather, it stores one for each set of images that share the same field of view and z-axis level. + + """ + def __init__(self): + self._images = [] + + def add(self, image): + """ + :type image: nd2reader.model.Image() + + """ + self._images.append(image) + + def __iter__(self): + for image in self._images: + yield image \ No newline at end of file diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 0aaa705..ee49d53 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -9,14 +9,14 @@ field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) class Nd2Parser(object): - CHUNK_HEADER = 0xabeceda - CHUNK_MAP_START = "ND2 FILEMAP SIGNATURE NAME 0001!" - CHUNK_MAP_END = "ND2 CHUNK MAP SIGNATURE 0000001!" - """ Reads .nd2 files, provides an interface to the metadata, and generates numpy arrays from the image data. """ + CHUNK_HEADER = 0xabeceda + CHUNK_MAP_START = "ND2 FILEMAP SIGNATURE NAME 0001!" + CHUNK_MAP_END = "ND2 CHUNK MAP SIGNATURE 0000001!" + def __init__(self, filename): self._absolute_start = None self._filename = filename From 3d387a23c1b635995665f870726802f069b60e71 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 17 May 2015 04:59:41 +0000 Subject: [PATCH 19/39] resolves #4 --- nd2reader/__init__.py | 49 ++++++++++++++++++------------------- nd2reader/model/__init__.py | 43 ++++++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 35 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 4e7409b..c99a1b6 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -10,7 +10,7 @@ import struct log = logging.getLogger(__name__) log.addHandler(logging.StreamHandler()) -log.setLevel(logging.WARN) +log.setLevel(logging.DEBUG) class Nd2(Nd2Parser): @@ -19,12 +19,6 @@ class Nd2(Nd2Parser): self._use_image_sets = image_sets def __iter__(self): - if self._use_image_sets: - return self.image_sets() - else: - return self.images() - - def images(self): for i in range(self._image_count): for fov in range(self.field_of_view_count): for z_level in range(self.z_level_count): @@ -33,6 +27,7 @@ class Nd2(Nd2Parser): if image.is_valid: yield image + @property def image_sets(self): for time_index in xrange(self.time_index_count): image_set = ImageSet() @@ -42,7 +37,7 @@ class Nd2(Nd2Parser): image = self.get_image(time_index, fov, channel_name, z_level) if image.is_valid: image_set.add(image) - yield image_set + yield image_set def get_image(self, time_index, fov, channel_name, z_level): image_set_number = self._calculate_image_set_number(time_index, fov, z_level) @@ -83,23 +78,22 @@ class Nd2(Nd2Parser): @property def absolute_start(self): - if self._absolute_start is None: - for line in self.metadata['ImageTextInfo']['SLxImageTextInfo'].values(): - absolute_start_12 = None - absolute_start_24 = None - # ND2s seem to randomly switch between 12- and 24-hour representations. - try: - absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") - except ValueError: - pass - try: - absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") - except ValueError: - pass - if not absolute_start_12 and not absolute_start_24: - continue - self._absolute_start = absolute_start_12 if absolute_start_12 else absolute_start_24 - return self._absolute_start + for line in self.metadata['ImageTextInfo']['SLxImageTextInfo'].values(): + absolute_start_12 = None + absolute_start_24 = None + # ND2s seem to randomly switch between 12- and 24-hour representations. + try: + absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") + except ValueError: + pass + try: + absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") + except ValueError: + pass + if not absolute_start_12 and not absolute_start_24: + continue + return absolute_start_12 if absolute_start_12 else absolute_start_24 + raise ValueError("This ND2 has no recorded start time. This is probably a bug.") @property def channel_count(self): @@ -182,3 +176,8 @@ class Nd2(Nd2Parser): def _calculate_image_set_number(self, time_index, fov, z_level): return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) + + + +for image_set in Nd2("FYLM-141111-001.nd2").image_sets: + print(image_set.get("", 1).data) diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 64bf398..d660b3c 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -1,3 +1,4 @@ +import collections import numpy as np import logging @@ -22,10 +23,10 @@ class Image(object): @property def timestamp(self): """ - The number of seconds after the beginning of the acquisition that the image was taken. Note that for a given field - of view and z-level offset, if you have images of multiple channels, they will all be given the same timestamp. - No, this doesn't make much sense. But that's how ND2s are structured, so if your experiment depends on millisecond - accuracy, you need to find an alternative imaging system. + The number of seconds after the beginning of the acquisition that the image was taken. Note that for a given + field of view and z-level offset, if you have images of multiple channels, they will all be given the same + timestamp. No, this doesn't make much sense. But that's how ND2s are structured, so if your experiment depends + on millisecond accuracy, you need to find an alternative imaging system. """ return self._timestamp / 1000.0 @@ -47,6 +48,16 @@ class Image(object): @property def is_valid(self): + """ + Not every image stored in an ND2 is a real image! If you take 4 images at one field of view and 2 at another + in a repeating cycle, there will be 4 images at BOTH field of view. The 2 non-images are the same size as all + the other images, only pure black (i.e. every pixel has a value of zero). + + This is probably an artifact of some algorithm in NIS Elements determining the maximum number of possible + images and pre-allocating the space with zeros. Regardless of why they exit, we can't tell that they're + not actual images until we examine the data. If every pixel value is exactly 0, it's a gap image. + + """ return np.any(self._raw_data) @@ -57,15 +68,27 @@ class ImageSet(object): """ def __init__(self): - self._images = [] + self._images = collections.defaultdict(dict) + + def get(self, channel="", z_level=0): + """ + Retrieve an image with a given channel and z-level. For most users, z_level will always be 0. + + """ + try: + image = self._images[channel][z_level] + except KeyError: + return None + else: + return image + + def __len__(self): + """ The number of images in the image set. """ + return sum([len(channel) for channel in self._images.values()]) def add(self, image): """ :type image: nd2reader.model.Image() """ - self._images.append(image) - - def __iter__(self): - for image in self._images: - yield image \ No newline at end of file + self._images[image.channel][image.z_level] = image \ No newline at end of file From 50a6de765bd83f5a40a4d2a6b7d95126366ed076 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 17 May 2015 05:01:52 +0000 Subject: [PATCH 20/39] removed test lines --- nd2reader/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index c99a1b6..8191c6e 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -175,9 +175,4 @@ class Nd2(Nd2Parser): return timestamp, image_data[image_data_start::self.channel_count] def _calculate_image_set_number(self, time_index, fov, z_level): - return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) - - - -for image_set in Nd2("FYLM-141111-001.nd2").image_sets: - print(image_set.get("", 1).data) + return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) \ No newline at end of file From a8a0d2a0837b996a69c0659aa61cb5b138ebc255 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 17 May 2015 05:28:16 +0000 Subject: [PATCH 21/39] added repr --- nd2reader/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 8191c6e..7406142 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -14,9 +14,19 @@ log.setLevel(logging.DEBUG) class Nd2(Nd2Parser): - def __init__(self, filename, image_sets=False): + def __init__(self, filename): super(Nd2, self).__init__(filename) - self._use_image_sets = image_sets + self._filename = filename + + def __repr__(self): + return "\n".join(["ND2: %s" % self._filename, + "Created: %s" % self.absolute_start.strftime("%Y-%m-%d %H:%M:%S"), + "Image size: %sx%s (HxW)" % (self.height, self.width), + "Image cycles: %s" % self.time_index_count, + "Channels: %s" % ", ".join(["'%s'" % channel for channel in self.channels]), + "Fields of View: %s" % self.field_of_view_count, + "Z-Levels: %s" % self.z_level_count + ]) def __iter__(self): for i in range(self._image_count): @@ -125,11 +135,7 @@ class Nd2(Nd2Parser): @property def time_index_count(self): """ - The number of image sets. If images were acquired using some kind of cycle, all images at each step in the - program will have the same timestamp (even though they may have varied by a few seconds in reality). For example, - if you have four fields of view that you're constantly monitoring, and you take a bright field and GFP image of - each, and you repeat that process 100 times, you'll have 800 individual images. But there will only be 400 - time indexes. + The number of cycles. :rtype: int From 0dcb18ce0045dee0e225bafa0683d132e88a1836 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 17 May 2015 05:31:38 +0000 Subject: [PATCH 22/39] changed log level --- nd2reader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 7406142..0d202fe 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -10,7 +10,7 @@ import struct log = logging.getLogger(__name__) log.addHandler(logging.StreamHandler()) -log.setLevel(logging.DEBUG) +log.setLevel(logging.WARNING) class Nd2(Nd2Parser): From 98391dc3c012f3efed3d2fddd01895194e3104f4 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 17 May 2015 05:51:23 +0000 Subject: [PATCH 23/39] refactored the way images are validated --- nd2reader/__init__.py | 17 ++++++++++++----- nd2reader/model/__init__.py | 14 -------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 0d202fe..06e8709 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -34,7 +34,7 @@ class Nd2(Nd2Parser): for z_level in range(self.z_level_count): for channel_name in self.channels: image = self.get_image(i, fov, channel_name, z_level) - if image.is_valid: + if image is not None: yield image @property @@ -45,14 +45,19 @@ class Nd2(Nd2Parser): for channel_name in self.channels: for z_level in xrange(self.z_level_count): image = self.get_image(time_index, fov, channel_name, z_level) - if image.is_valid: + if image is not None: image_set.add(image) yield image_set def get_image(self, time_index, fov, channel_name, z_level): image_set_number = self._calculate_image_set_number(time_index, fov, z_level) - timestamp, raw_image_data = self._get_raw_image_data(image_set_number, self._channel_offset[channel_name]) - return Image(timestamp, raw_image_data, fov, channel_name, z_level, self.height, self.width) + try: + timestamp, raw_image_data = self._get_raw_image_data(image_set_number, self._channel_offset[channel_name]) + image = Image(timestamp, raw_image_data, fov, channel_name, z_level, self.height, self.width) + except TypeError: + return None + else: + return image @property def channels(self): @@ -178,7 +183,9 @@ class Nd2(Nd2Parser): # The images for the various channels are interleaved within each other. image_data = array.array("H", data) image_data_start = 4 + channel_offset - return timestamp, image_data[image_data_start::self.channel_count] + if any(image_data): + return timestamp, image_data[image_data_start::self.channel_count] + return None def _calculate_image_set_number(self, time_index, fov, z_level): return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) \ No newline at end of file diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index d660b3c..796b96e 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -46,20 +46,6 @@ class Image(object): self._data = np.reshape(self._raw_data, (self._height, self._width)) return self._data - @property - def is_valid(self): - """ - Not every image stored in an ND2 is a real image! If you take 4 images at one field of view and 2 at another - in a repeating cycle, there will be 4 images at BOTH field of view. The 2 non-images are the same size as all - the other images, only pure black (i.e. every pixel has a value of zero). - - This is probably an artifact of some algorithm in NIS Elements determining the maximum number of possible - images and pre-allocating the space with zeros. Regardless of why they exit, we can't tell that they're - not actual images until we examine the data. If every pixel value is exactly 0, it's a gap image. - - """ - return np.any(self._raw_data) - class ImageSet(object): """ From 2861d551cbdeade26432524a37d3f0d6a93482f3 Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 17 May 2015 19:34:13 -0500 Subject: [PATCH 24/39] #2 moved count methods to parser and fixed subtle bug in method that filtered out gap images --- nd2reader/__init__.py | 144 ++++-------------------------------------- nd2reader/parser.py | 124 ++++++++++++++++++++++++++++++++++-- 2 files changed, 132 insertions(+), 136 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 06e8709..0532299 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -1,12 +1,9 @@ # -*- coding: utf-8 -*- -import array -from datetime import datetime import logging from nd2reader.model import Image, ImageSet from nd2reader.parser import Nd2Parser -import re -import struct + log = logging.getLogger(__name__) log.addHandler(logging.StreamHandler()) @@ -20,30 +17,30 @@ class Nd2(Nd2Parser): def __repr__(self): return "\n".join(["ND2: %s" % self._filename, - "Created: %s" % self.absolute_start.strftime("%Y-%m-%d %H:%M:%S"), + "Created: %s" % self._absolute_start.strftime("%Y-%m-%d %H:%M:%S"), "Image size: %sx%s (HxW)" % (self.height, self.width), - "Image cycles: %s" % self.time_index_count, - "Channels: %s" % ", ".join(["'%s'" % channel for channel in self.channels]), - "Fields of View: %s" % self.field_of_view_count, - "Z-Levels: %s" % self.z_level_count + "Image cycles: %s" % self._time_index_count, + "Channels: %s" % ", ".join(["'%s'" % channel for channel in self._channels]), + "Fields of View: %s" % self._field_of_view_count, + "Z-Levels: %s" % self._z_level_count ]) def __iter__(self): for i in range(self._image_count): - for fov in range(self.field_of_view_count): - for z_level in range(self.z_level_count): - for channel_name in self.channels: + for fov in range(self._field_of_view_count): + for z_level in range(self._z_level_count): + for channel_name in self._channels: image = self.get_image(i, fov, channel_name, z_level) if image is not None: yield image @property def image_sets(self): - for time_index in xrange(self.time_index_count): + for time_index in xrange(self._time_index_count): image_set = ImageSet() - for fov in range(self.field_of_view_count): - for channel_name in self.channels: - for z_level in xrange(self.z_level_count): + for fov in range(self._field_of_view_count): + for channel_name in self._channels: + for z_level in xrange(self._z_level_count): image = self.get_image(time_index, fov, channel_name, z_level) if image is not None: image_set.add(image) @@ -59,22 +56,6 @@ class Nd2(Nd2Parser): else: return image - @property - def channels(self): - metadata = self.metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] - try: - validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] - except KeyError: - # If none of the channels have been deleted, there is no validity list, so we just make one - validity = [True for _ in metadata] - # Channel information is contained in dictionaries with the keys a0, a1...an where the number - # indicates the order in which the channel is stored. So by sorting the dicts alphabetically - # we get the correct order. - for (label, chan), valid in zip(sorted(metadata['sPlaneNew'].items()), validity): - if not valid: - continue - yield chan['sDescription'] - @property def height(self): """ @@ -90,102 +71,3 @@ class Nd2(Nd2Parser): """ return self.metadata['ImageAttributes']['SLxImageAttributes']['uiWidth'] - - @property - def absolute_start(self): - for line in self.metadata['ImageTextInfo']['SLxImageTextInfo'].values(): - absolute_start_12 = None - absolute_start_24 = None - # ND2s seem to randomly switch between 12- and 24-hour representations. - try: - absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") - except ValueError: - pass - try: - absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") - except ValueError: - pass - if not absolute_start_12 and not absolute_start_24: - continue - return absolute_start_12 if absolute_start_12 else absolute_start_24 - raise ValueError("This ND2 has no recorded start time. This is probably a bug.") - - @property - def channel_count(self): - pattern = r""".*?λ\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count - - @property - def field_of_view_count(self): - """ - The metadata contains information about fields of view, but it contains it even if some fields - of view were cropped. We can't find anything that states which fields of view are actually - in the image data, so we have to calculate it. There probably is something somewhere, since - NIS Elements can figure it out, but we haven't found it yet. - - """ - pattern = r""".*?XY\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count - - @property - def time_index_count(self): - """ - The number of cycles. - - :rtype: int - - """ - pattern = r""".*?T'\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count - - @property - def z_level_count(self): - pattern = r""".*?Z\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count - - @property - def _channel_offset(self): - """ - Image data is interleaved for each image set. That is, if there are four images in a set, the first image - will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth. Why this would be the - case is beyond me, but that's how it works. - - """ - channel_offset = {} - for n, channel in enumerate(self.channels): - channel_offset[channel] = n - return channel_offset - - def _get_raw_image_data(self, image_set_number, channel_offset): - chunk = self._label_map["ImageDataSeq|%d!" % image_set_number] - data = self._read_chunk(chunk) - timestamp = struct.unpack("d", data[:8])[0] - # The images for the various channels are interleaved within each other. - image_data = array.array("H", data) - image_data_start = 4 + channel_offset - if any(image_data): - return timestamp, image_data[image_data_start::self.channel_count] - return None - - def _calculate_image_set_number(self, time_index, fov, z_level): - return time_index * self.field_of_view_count * self.z_level_count + (fov * self.z_level_count + z_level) \ No newline at end of file diff --git a/nd2reader/parser.py b/nd2reader/parser.py index ee49d53..e6c3917 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -2,6 +2,9 @@ import array from collections import namedtuple +from datetime import datetime +import numpy as np +import re import struct from StringIO import StringIO @@ -18,7 +21,6 @@ class Nd2Parser(object): CHUNK_MAP_END = "ND2 CHUNK MAP SIGNATURE 0000001!" def __init__(self, filename): - self._absolute_start = None self._filename = filename self._fh = None self._chunk_map_start_location = None @@ -35,6 +37,18 @@ class Nd2Parser(object): self._fh = open(self._filename, "rb") return self._fh + def _get_raw_image_data(self, image_set_number, channel_offset): + chunk = self._label_map["ImageDataSeq|%d!" % image_set_number] + data = self._read_chunk(chunk) + timestamp = struct.unpack("d", data[:8])[0] + # The images for the various channels are interleaved within each other. + image_set_data = array.array("H", data) + image_data_start = 4 + channel_offset + image_data = image_set_data[image_data_start::self._channel_count] + if np.any(image_data): + return timestamp, image_data + return None + @property def _dimensions(self): if self._dimension_text is None: @@ -52,6 +66,110 @@ class Nd2Parser(object): raise ValueError("Could not parse metadata dimensions!") return self._dimension_text + @property + def _channels(self): + metadata = self.metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] + try: + validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] + except KeyError: + # If none of the channels have been deleted, there is no validity list, so we just make one + validity = [True for _ in metadata] + # Channel information is contained in dictionaries with the keys a0, a1...an where the number + # indicates the order in which the channel is stored. So by sorting the dicts alphabetically + # we get the correct order. + for (label, chan), valid in zip(sorted(metadata['sPlaneNew'].items()), validity): + if not valid: + continue + yield chan['sDescription'] + + def _calculate_image_set_number(self, time_index, fov, z_level): + return time_index * self._field_of_view_count * self._z_level_count + (fov * self._z_level_count + z_level) + + @property + def _channel_offset(self): + """ + Image data is interleaved for each image set. That is, if there are four images in a set, the first image + will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth. Why this would be the + case is beyond me, but that's how it works. + + """ + channel_offset = {} + for n, channel in enumerate(self._channels): + channel_offset[channel] = n + return channel_offset + + @property + def _absolute_start(self): + for line in self.metadata['ImageTextInfo']['SLxImageTextInfo'].values(): + absolute_start_12 = None + absolute_start_24 = None + # ND2s seem to randomly switch between 12- and 24-hour representations. + try: + absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") + except ValueError: + pass + try: + absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") + except ValueError: + pass + if not absolute_start_12 and not absolute_start_24: + continue + return absolute_start_12 if absolute_start_12 else absolute_start_24 + raise ValueError("This ND2 has no recorded start time. This is probably a bug.") + + @property + def _channel_count(self): + pattern = r""".*?λ\((\d+)\).*?""" + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return 1 + else: + return count + + @property + def _field_of_view_count(self): + """ + The metadata contains information about fields of view, but it contains it even if some fields + of view were cropped. We can't find anything that states which fields of view are actually + in the image data, so we have to calculate it. There probably is something somewhere, since + NIS Elements can figure it out, but we haven't found it yet. + + """ + pattern = r""".*?XY\((\d+)\).*?""" + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return 1 + else: + return count + + @property + def _time_index_count(self): + """ + The number of cycles. + + :rtype: int + + """ + pattern = r""".*?T'\((\d+)\).*?""" + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return 1 + else: + return count + + @property + def _z_level_count(self): + pattern = r""".*?Z\((\d+)\).*?""" + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return 1 + else: + return count + @property def _image_count(self): return self.metadata['ImageAttributes']['SLxImageAttributes']['uiSequenceCount'] @@ -106,10 +224,6 @@ class Nd2Parser(object): self._file_handle.seek(chunk_location + 16 + relative_offset) return self._file_handle.read(data_length) - def _z_level_count(self): - st = self._read_chunk(self._label_map["CustomData|Z!"]) - return len(array.array("d", st)) - def _parse_unsigned_char(self, data): return struct.unpack("B", data.read(1))[0] From ffb3f43f7f88d8c2c6ffce3af9452699b797fb54 Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 17 May 2015 19:41:37 -0500 Subject: [PATCH 25/39] #2 added reprs --- nd2reader/__init__.py | 2 +- nd2reader/model/__init__.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 0532299..c0d16d9 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -16,7 +16,7 @@ class Nd2(Nd2Parser): self._filename = filename def __repr__(self): - return "\n".join(["ND2: %s" % self._filename, + return "\n".join(["" % self._filename, "Created: %s" % self._absolute_start.strftime("%Y-%m-%d %H:%M:%S"), "Image size: %sx%s (HxW)" % (self.height, self.width), "Image cycles: %s" % self._time_index_count, diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 796b96e..55d57c3 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -16,6 +16,15 @@ class Image(object): self._width = width self._data = None + def __repr__(self): + return "\n".join(["", + "%sx%s (HxW)" % (self._height, self._width), + "Timestamp: %s" % self.timestamp, + "Field of View: %s" % self.field_of_view, + "Channel: %s" % self.channel, + "Z-Level: %s" % self.z_level, + ]) + @property def field_of_view(self): return self._field_of_view @@ -56,6 +65,10 @@ class ImageSet(object): def __init__(self): self._images = collections.defaultdict(dict) + def __repr__(self): + return "\n".join(["", + "Image count: %s" % len(self)]) + def get(self, channel="", z_level=0): """ Retrieve an image with a given channel and z-level. For most users, z_level will always be 0. From e6471f5a1fdc65a088e676f8dacb0af6850690ba Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 16:25:19 +0000 Subject: [PATCH 26/39] #2 finished comments and cleanup of image and image set --- nd2reader/model/__init__.py | 94 +++++++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 20 deletions(-) diff --git a/nd2reader/model/__init__.py b/nd2reader/model/__init__.py index 55d57c3..bd547a1 100644 --- a/nd2reader/model/__init__.py +++ b/nd2reader/model/__init__.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import collections import numpy as np import logging @@ -7,6 +9,25 @@ log = logging.getLogger(__name__) class Image(object): def __init__(self, timestamp, raw_array, field_of_view, channel, z_level, height, width): + """ + A wrapper around the raw pixel data of an image. + + :param timestamp: The number of milliseconds after the beginning of the acquisition that this image was taken. + :type timestamp: int + :param raw_array: The raw sequence of bytes that represents the image. + :type raw_array: array.array() + :param field_of_view: The label for the place in the XY-plane where this image was taken. + :type field_of_view: int + :param channel: The name of the color of this image + :type channel: str + :param z_level: The label for the location in the Z-plane where this image was taken. + :type z_level: int + :param height: The height of the image in pixels. + :type height: int + :param width: The width of the image in pixels. + :type width: int + + """ self._timestamp = timestamp self._raw_data = raw_array self._field_of_view = field_of_view @@ -25,8 +46,28 @@ class Image(object): "Z-Level: %s" % self.z_level, ]) + @property + def data(self): + """ + The actual image data. + + :rtype np.array() + + """ + if self._data is None: + # The data is just a 1-dimensional array originally. + # We convert it to a 2D image here. + self._data = np.reshape(self._raw_data, (self._height, self._width)) + return self._data + @property def field_of_view(self): + """ + Which of the fixed locations this image was taken at. + + :rtype int: + + """ return self._field_of_view @property @@ -37,56 +78,69 @@ class Image(object): timestamp. No, this doesn't make much sense. But that's how ND2s are structured, so if your experiment depends on millisecond accuracy, you need to find an alternative imaging system. + :rtype float: + """ return self._timestamp / 1000.0 @property def channel(self): + """ + The name of the filter used to acquire this image. These are user-supplied in NIS Elements. + + :rtype str: + + """ return self._channel @property def z_level(self): - return self._z_level + """ + The vertical offset of the image. These are simple integers starting from 0, where the 0 is the lowest + z-level and each subsequent level incremented by 1. - @property - def data(self): - if self._data is None: - # The data is just a flat, 1-dimensional array. We convert it to a 2D image here. - self._data = np.reshape(self._raw_data, (self._height, self._width)) - return self._data + For example, if you acquired images at -3 µm, 0 µm, and +3 µm, your z-levels would be: + + -3 µm: 0 + 0 µm: 1 + +3 µm: 2 + + :rtype int: + + """ + return self._z_level class ImageSet(object): """ - A group of images that share the same timestamp. NIS Elements doesn't store a unique timestamp for every - image, rather, it stores one for each set of images that share the same field of view and z-axis level. + A group of images that were taken at roughly the same time. """ def __init__(self): self._images = collections.defaultdict(dict) + def __len__(self): + """ The number of images in the image set. """ + return sum([len(channel) for channel in self._images.values()]) + def __repr__(self): return "\n".join(["", "Image count: %s" % len(self)]) - def get(self, channel="", z_level=0): + def get(self, channel, z_level=0): """ Retrieve an image with a given channel and z-level. For most users, z_level will always be 0. - """ - try: - image = self._images[channel][z_level] - except KeyError: - return None - else: - return image + :type channel: str + :type z_level: int - def __len__(self): - """ The number of images in the image set. """ - return sum([len(channel) for channel in self._images.values()]) + """ + return self._images.get(channel).get(z_level) def add(self, image): """ + Stores an image. + :type image: nd2reader.model.Image() """ From d5c8eb5cd1f4143a3db8a22da1ddb66162b1918f Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 16:55:04 +0000 Subject: [PATCH 27/39] #2 finished comments and cleanup of public interface --- nd2reader/__init__.py | 83 +++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index c0d16d9..4ef37c4 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -1,16 +1,14 @@ # -*- coding: utf-8 -*- -import logging from nd2reader.model import Image, ImageSet from nd2reader.parser import Nd2Parser -log = logging.getLogger(__name__) -log.addHandler(logging.StreamHandler()) -log.setLevel(logging.WARNING) - - class Nd2(Nd2Parser): + """ + Allows easy access to NIS Elements .nd2 image files. + + """ def __init__(self, filename): super(Nd2, self).__init__(filename) self._filename = filename @@ -25,7 +23,31 @@ class Nd2(Nd2Parser): "Z-Levels: %s" % self._z_level_count ]) + @property + def height(self): + """ + :return: height of each image, in pixels + :rtype: int + + """ + return self.metadata['ImageAttributes']['SLxImageAttributes']['uiHeight'] + + @property + def width(self): + """ + :return: width of each image, in pixels + :rtype: int + + """ + return self.metadata['ImageAttributes']['SLxImageAttributes']['uiWidth'] + def __iter__(self): + """ + Iterates over every image, in the order they were taken. + + :return: model.Image() + + """ for i in range(self._image_count): for fov in range(self._field_of_view_count): for z_level in range(self._z_level_count): @@ -36,6 +58,15 @@ class Nd2(Nd2Parser): @property def image_sets(self): + """ + Iterates over groups of related images. This is useful if your ND2 contains multiple fields of view. + A typical use case might be that you have, say, four areas of interest that you're monitoring, and every + minute you take a bright field and GFP image of each one. For each cycle, this method would produce four + ImageSet objects, each containing one bright field and one GFP image. + + :return: model.ImageSet() + + """ for time_index in xrange(self._time_index_count): image_set = ImageSet() for fov in range(self._field_of_view_count): @@ -46,28 +77,28 @@ class Nd2(Nd2Parser): image_set.add(image) yield image_set - def get_image(self, time_index, fov, channel_name, z_level): - image_set_number = self._calculate_image_set_number(time_index, fov, z_level) + def get_image(self, time_index, field_of_view, channel_name, z_level): + """ + Returns an Image if data exists for the given parameters, otherwise returns None. In general, you should avoid + using this method unless you're very familiar with the structure of ND2 files. If you have a use case that + cannot be met by the `__iter__` or `image_sets` methods above, please create an issue on Github. + + :param time_index: the frame number + :type time_index: int + :param field_of_view: the label for the place in the XY-plane where this image was taken. + :type field_of_view: int + :param channel_name: the name of the color of this image + :type channel_name: str + :param z_level: the label for the location in the Z-plane where this image was taken. + :type z_level: int + :rtype: nd2reader.model.Image() or None + + """ + image_set_number = self._calculate_image_set_number(time_index, field_of_view, z_level) try: timestamp, raw_image_data = self._get_raw_image_data(image_set_number, self._channel_offset[channel_name]) - image = Image(timestamp, raw_image_data, fov, channel_name, z_level, self.height, self.width) + image = Image(timestamp, raw_image_data, field_of_view, channel_name, z_level, self.height, self.width) except TypeError: return None else: - return image - - @property - def height(self): - """ - :return: height of each image, in pixels - - """ - return self.metadata['ImageAttributes']['SLxImageAttributes']['uiHeight'] - - @property - def width(self): - """ - :return: width of each image, in pixels - - """ - return self.metadata['ImageAttributes']['SLxImageAttributes']['uiWidth'] + return image \ No newline at end of file From fe32c6286414b6abe798b725b474913f2ef08877 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 17:28:35 +0000 Subject: [PATCH 28/39] #2 finished cleaning up parser --- nd2reader/__init__.py | 2 +- nd2reader/parser.py | 110 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 4ef37c4..26a6f1b 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -94,7 +94,7 @@ class Nd2(Nd2Parser): :rtype: nd2reader.model.Image() or None """ - image_set_number = self._calculate_image_set_number(time_index, field_of_view, z_level) + image_set_number = self._calculate_image_group_number(time_index, field_of_view, z_level) try: timestamp, raw_image_data = self._get_raw_image_data(image_set_number, self._channel_offset[channel_name]) image = Image(timestamp, raw_image_data, field_of_view, channel_name, z_level, self.height, self.width) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index e6c3917..3c37df7 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -14,6 +14,7 @@ field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) class Nd2Parser(object): """ Reads .nd2 files, provides an interface to the metadata, and generates numpy arrays from the image data. + You should not ever need to instantiate this class manually unless you're a developer. """ CHUNK_HEADER = 0xabeceda @@ -37,20 +38,48 @@ class Nd2Parser(object): self._fh = open(self._filename, "rb") return self._fh - def _get_raw_image_data(self, image_set_number, channel_offset): - chunk = self._label_map["ImageDataSeq|%d!" % image_set_number] + def _get_raw_image_data(self, image_group_number, channel_offset): + """ + Reads the raw bytes and the timestamp of an image. + + :param image_group_number: groups are made of images with the same time index, field of view and z-level. + :type image_group_number: int + :param channel_offset: the offset in the array where the bytes for this image are found. + :type channel_offset: int + + :return: (int, array.array()) or None + + """ + chunk = self._label_map["ImageDataSeq|%d!" % image_group_number] data = self._read_chunk(chunk) + # All images in the same image group share the same timestamp! So if you have complicated image data, + # your timestamps may not be entirely accurate. Practically speaking though, they'll only be off by a few + # seconds unless you're doing something super weird. timestamp = struct.unpack("d", data[:8])[0] - # The images for the various channels are interleaved within each other. - image_set_data = array.array("H", data) + image_group_data = array.array("H", data) image_data_start = 4 + channel_offset - image_data = image_set_data[image_data_start::self._channel_count] + # The images for the various channels are interleaved within the same array. For example, the second image + # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design + # a data structure that way, please send the author of this library a message. + image_data = image_group_data[image_data_start::self._channel_count] + # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you + # don't have the same number of images each cycle. We discovered this because we only took GFP images every + # other cycle to reduce phototoxicity, but NIS Elements still allocated memory as if we were going to take them + # every cyle. if np.any(image_data): return timestamp, image_data return None @property def _dimensions(self): + """ + While there are metadata values that represent a lot of what we want to capture, they seem to be unreliable. + Sometimes certain elements don't exist, or change their data type randomly. However, the human-readable text + is always there and in the same exact format, so we just parse that instead. + + :rtype: str + + """ if self._dimension_text is None: for line in self.metadata['ImageTextInfo']['SLxImageTextInfo'].values(): if "Dimensions:" in line: @@ -68,6 +97,13 @@ class Nd2Parser(object): @property def _channels(self): + """ + These are labels created by the NIS Elements user. Typically they may a short description of the filter cube + used (e.g. "bright field", "GFP", etc.) + + :rtype: str + + """ metadata = self.metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] try: validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] @@ -82,15 +118,26 @@ class Nd2Parser(object): continue yield chan['sDescription'] - def _calculate_image_set_number(self, time_index, fov, z_level): + def _calculate_image_group_number(self, time_index, fov, z_level): + """ + Images are grouped together if they share the same time index, field of view, and z-level. + + :type time_index: int + :type fov: int + :type z_level: int + + :rtype: int + + """ return time_index * self._field_of_view_count * self._z_level_count + (fov * self._z_level_count + z_level) @property def _channel_offset(self): """ Image data is interleaved for each image set. That is, if there are four images in a set, the first image - will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth. Why this would be the - case is beyond me, but that's how it works. + will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth. + + :rtype: int """ channel_offset = {} @@ -100,6 +147,12 @@ class Nd2Parser(object): @property def _absolute_start(self): + """ + The date and time when acquisition began. + + :rtype: datetime.datetime() + + """ for line in self.metadata['ImageTextInfo']['SLxImageTextInfo'].values(): absolute_start_12 = None absolute_start_24 = None @@ -119,6 +172,12 @@ class Nd2Parser(object): @property def _channel_count(self): + """ + The number of different channels used, including bright field. + + :rtype: int + + """ pattern = r""".*?λ\((\d+)\).*?""" try: count = int(re.match(pattern, self._dimensions).group(1)) @@ -135,6 +194,8 @@ class Nd2Parser(object): in the image data, so we have to calculate it. There probably is something somewhere, since NIS Elements can figure it out, but we haven't found it yet. + :rtype: int + """ pattern = r""".*?XY\((\d+)\).*?""" try: @@ -162,6 +223,12 @@ class Nd2Parser(object): @property def _z_level_count(self): + """ + The number of different levels in the Z-plane. + + :rtype: int + + """ pattern = r""".*?Z\((\d+)\).*?""" try: count = int(re.match(pattern, self._dimensions).group(1)) @@ -172,13 +239,19 @@ class Nd2Parser(object): @property def _image_count(self): - return self.metadata['ImageAttributes']['SLxImageAttributes']['uiSequenceCount'] + """ + The total number of images in the ND2. Warning: this may be inaccurate as it includes "gap" images. - @property - def _sequence_count(self): - return self.metadata['ImageEvents']['RLxExperimentRecord']['uiCount'] + :rtype: int + + """ + return self.metadata['ImageAttributes']['SLxImageAttributes']['uiSequenceCount'] def _parse_metadata(self): + """ + Reads all metadata. + + """ for label in self._label_map.keys(): if label.endswith("LV!") or "LV|" in label: data = self._read_chunk(self._label_map[label]) @@ -248,6 +321,10 @@ class Nd2Parser(object): return array.array("B", data.read(array_length)) def _parse_metadata_item(self, data): + """ + Reads hierarchical data, analogous to a Python dict. + + """ new_count, length = struct.unpack(" Date: Sat, 23 May 2015 17:38:49 +0000 Subject: [PATCH 29/39] resolves #2 --- nd2reader/parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 3c37df7..650e751 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -64,8 +64,8 @@ class Nd2Parser(object): image_data = image_group_data[image_data_start::self._channel_count] # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you # don't have the same number of images each cycle. We discovered this because we only took GFP images every - # other cycle to reduce phototoxicity, but NIS Elements still allocated memory as if we were going to take them - # every cyle. + # other cycle to reduce phototoxicity, but NIS Elements still allocated memory as if we were going to take + # them every cycle. if np.any(image_data): return timestamp, image_data return None @@ -137,7 +137,7 @@ class Nd2Parser(object): Image data is interleaved for each image set. That is, if there are four images in a set, the first image will consist of pixels 1, 5, 9, etc, the second will be pixels 2, 6, 10, and so forth. - :rtype: int + :rtype: dict """ channel_offset = {} From 44e5217e766812294e9dce5b0f1adbfe57a47337 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 17:47:51 +0000 Subject: [PATCH 30/39] realized unit testing isn't even worth it here. yeah, I said that. quote me on it. --- CONTRIBUTORS.txt | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 0db601c..1b0a856 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -1,4 +1,4 @@ Author: Jim Rybarski nd2reader is based on the read_nd2 module from the SLOTH library (http://pythonhosted.org/SLOTH/_modules/sloth/read_nd2.html). -Thanks to M.Kauer and B.Kauer for solving the hardest part of this puzzle. \ No newline at end of file +Thanks to M.Kauer and B.Kauer for solving the hardest part of parsing ND2s. \ No newline at end of file diff --git a/Makefile b/Makefile index 30e5e42..2438069 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test shell +.PHONY: build shell build: docker build -t jimrybarski/nd2reader . From b4c01b7c8a70844ffed42b2358ffe6ea1e2786cc Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 18:20:22 +0000 Subject: [PATCH 31/39] #27 began writing tutorial --- README.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 8d3f555..c7c8213 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,84 @@ -nd2reader -========= +# nd2reader -## Simple access to hierarchical .nd2 files +## Simple access to .nd2 files -# About +### About -`nd2reader` is a pure-Python package that reads images produced by Nikon microscopes. Though it more or less works, it is currently under development and is not quite ready for use by the general public. Version 1.0 should be released in early 2015. +`nd2reader` is a pure-Python package that reads images produced by NIS Elements. -.nd2 files contain images and metadata, which can be split along multiple dimensions: time, fields of view (xy-axis), focus (z-axis), and filter channel. `nd2reader` allows you to view any subset of images based on any or all of these dimensions. +.nd2 files contain images and metadata, which can be split along multiple dimensions: time, fields of view (xy-plane), focus (z-plane), and filter channel. -`nd2reader` holds data in numpy arrays, which makes it trivial to use with the image analysis packages `scikit-image` and `OpenCV`. +`nd2reader` produces data in numpy arrays, which makes it trivial to use with the image analysis packages `scikit-image` and `OpenCV`. -# Dependencies +### Installation -numpy +Just use pip: -# Installation +`pip install nd2reader` -I'll write this eventually. +If you want to install via git, clone the repo and run: + +`python setup.py install` + +### Usage + +nd2reader provides two main ways to view image data. For most cases, you'll just want to iterate over each image: + +``` +import nd2reader +nd2 = nd2reader.Nd2("/path/to/my_images.nd2") +for image in nd2: + do_something(image.data) +``` + +If you have complicated hierarchical data, it may be easier to use image sets, which groups images together if they +share the same time index and field of view: + +``` +import nd2reader +nd2 = nd2reader.Nd2("/path/to/my_complicated_images.nd2") +for image_set in nd2.image_sets: + # you can select images by channel + gfp_image = image_set.get("GFP") + do_something_gfp_related(gfp_image) + + # you can also specify the z-level. this defaults to 0 if not given + out_of_focus_image = image_set.get("Bright Field", z_level=1) + do_something_out_of_focus_related(out_of_focus_image) +``` + +`Image` objects provide several pieces of useful data. + +``` +>>> import nd2reader +>>> nd2 = nd2reader.Nd2("/path/to/my_images.nd2") +>>> image = nd2.get_image(14, 2, "GFP", 1) +>>> image.data +array([[1809, 1783, 1830, ..., 1923, 1920, 1914], + [1687, 1855, 1792, ..., 1986, 1903, 1889], + [1758, 1901, 1849, ..., 1911, 2010, 1954], + ..., + [3363, 3370, 3570, ..., 3565, 3601, 3459], + [3480, 3428, 3328, ..., 3542, 3461, 3575], + [3497, 3666, 3635, ..., 3817, 3867, 3779]]) +>>> image.channel +'GFP' +>>> image.timestamp +1699.7947813408175 +>>> image.field_of_view +2 +>>> image.z_level +1 +``` + +You can also get a quick summary of image data. + +``` +>>> image + +1280x800 (HxW) +Timestamp: 1699.79478134 +Field of View: 2 +Channel: GFP +Z-Level: 1 +``` \ No newline at end of file From ae06dca96c5614278fe8ed017d63a071775178d5 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 18:32:27 +0000 Subject: [PATCH 32/39] #27 better instructions --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c7c8213..709db26 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,12 @@ # nd2reader -## Simple access to .nd2 files - ### About `nd2reader` is a pure-Python package that reads images produced by NIS Elements. .nd2 files contain images and metadata, which can be split along multiple dimensions: time, fields of view (xy-plane), focus (z-plane), and filter channel. -`nd2reader` produces data in numpy arrays, which makes it trivial to use with the image analysis packages `scikit-image` and `OpenCV`. +`nd2reader` produces data in numpy arrays, which makes it trivial to use with the image analysis packages such as `scikit-image` and `OpenCV`. ### Installation @@ -20,9 +18,9 @@ If you want to install via git, clone the repo and run: `python setup.py install` -### Usage +### Simple Iteration -nd2reader provides two main ways to view image data. For most cases, you'll just want to iterate over each image: +For most cases, you'll just want to iterate over each image: ``` import nd2reader @@ -31,10 +29,12 @@ for image in nd2: do_something(image.data) ``` +### Image Sets + If you have complicated hierarchical data, it may be easier to use image sets, which groups images together if they share the same time index and field of view: -``` +```python import nd2reader nd2 = nd2reader.Nd2("/path/to/my_complicated_images.nd2") for image_set in nd2.image_sets: @@ -47,6 +47,14 @@ for image_set in nd2.image_sets: do_something_out_of_focus_related(out_of_focus_image) ``` +### Direct Image Access + +There is a method, `get_image`, which allows random access to images. This might not always return an image, however, +if you acquired different numbers of images in each cycle of a program. For example, if you acquire GFP images every +other minute, but acquire bright field images every minute, `get_image` will return `None` at certain time indexes. + +### Images + `Image` objects provide several pieces of useful data. ``` From 1daac4a2d9a7f535f8eb7bf914ec5931fcc2ce87 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 18:33:53 +0000 Subject: [PATCH 33/39] #27 syntax highlighting --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 709db26..9f7532c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ If you want to install via git, clone the repo and run: For most cases, you'll just want to iterate over each image: -``` +```python import nd2reader nd2 = nd2reader.Nd2("/path/to/my_images.nd2") for image in nd2: @@ -57,7 +57,7 @@ other minute, but acquire bright field images every minute, `get_image` will ret `Image` objects provide several pieces of useful data. -``` +```python >>> import nd2reader >>> nd2 = nd2reader.Nd2("/path/to/my_images.nd2") >>> image = nd2.get_image(14, 2, "GFP", 1) @@ -81,7 +81,7 @@ array([[1809, 1783, 1830, ..., 1923, 1920, 1914], You can also get a quick summary of image data. -``` +```python >>> image 1280x800 (HxW) From b182e4a02318c3d7041a0881dd7644587f240970 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 18:37:31 +0000 Subject: [PATCH 34/39] #27 more sensible example --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 9f7532c..afad4e8 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,9 @@ array([[1809, 1783, 1830, ..., 1923, 1920, 1914], 2 >>> image.z_level 1 -``` -You can also get a quick summary of image data. +# You can also get a quick summary of image data: -```python >>> image 1280x800 (HxW) From c3d659cfd3d2c9f2883da31ee609cb326091c51f Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 18:51:08 +0000 Subject: [PATCH 35/39] #27 added contribution instructions --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index afad4e8..1db47a8 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,21 @@ If you want to install via git, clone the repo and run: `python setup.py install` +### ND2s + +A quick summary of ND2 metadata can be obtained as shown below. +```python +>>> import nd2reader +>>> nd2 = nd2reader.Nd2("/path/to/my_images.nd2") +>>> nd2 + +Created: 2014-11-11 15:59:19 +Image size: 1280x800 (HxW) +Image cycles: 636 +Channels: '', 'GFP' +Fields of View: 8 +Z-Levels: 3 + ### Simple Iteration For most cases, you'll just want to iterate over each image: @@ -87,4 +102,14 @@ Timestamp: 1699.79478134 Field of View: 2 Channel: GFP Z-Level: 1 -``` \ No newline at end of file +``` + +### Bug Reports and Features + +If this fails to work exactly as expected, please open a Github issue. If you get an unhandled exception, please +paste the entire stack trace into the issue as well. + +### Contributing + +Please feel free to submit a pull request with any new features you think would be useful. You can also create an +issue if you'd just like to propose or discuss a potential idea. \ No newline at end of file From 5791d5fc808a8664a9b9c4a321296bfea8d59032 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 18:52:07 +0000 Subject: [PATCH 36/39] #27 fixed formatting error --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1db47a8..d001757 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Image cycles: 636 Channels: '', 'GFP' Fields of View: 8 Z-Levels: 3 +``` ### Simple Iteration From f3ebf77ddc7d290b475aa8f44babc93cc54d46b5 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 19:14:16 +0000 Subject: [PATCH 37/39] resolves #26 --- setup.cfg | 2 ++ setup.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..224a779 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py index 5107f45..daac0b0 100644 --- a/setup.py +++ b/setup.py @@ -3,5 +3,12 @@ from setuptools import setup, find_packages setup( name="nd2reader", packages=find_packages(), - version="0.9.7" -) \ No newline at end of file + version="1.0.0", + description='A tool for reading ND2 files produced by NIS Elements', + author='Jim Rybarski', + author_email='jim@rybarski.com', + url='https://github.com/jimrybarski/nd2reader', + download_url='https://github.com/jimrybarski/nd2reader/tarball/1.0.0', + keywords=['nd2', 'nikon', 'microscopy', 'NIS Elements'], + classifiers=[], +) From fde3f028a293c1e56b2b4f0ab17fe27148219e51 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 19:19:01 +0000 Subject: [PATCH 38/39] #26 fix installation --- requirements.txt | 2 +- setup.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 296d654..ccee7d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -numpy \ No newline at end of file +numpy>=1.9.2 \ No newline at end of file diff --git a/setup.py b/setup.py index daac0b0..49e06ac 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ -from setuptools import setup, find_packages +from distutils.core import setup setup( name="nd2reader", - packages=find_packages(), + packages=['nd2reader'], version="1.0.0", description='A tool for reading ND2 files produced by NIS Elements', author='Jim Rybarski', @@ -10,5 +10,5 @@ setup( url='https://github.com/jimrybarski/nd2reader', download_url='https://github.com/jimrybarski/nd2reader/tarball/1.0.0', keywords=['nd2', 'nikon', 'microscopy', 'NIS Elements'], - classifiers=[], + classifiers=[] ) From 1a27e175c253ac88b77b5ed99064887aa978770d Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 23 May 2015 19:50:23 +0000 Subject: [PATCH 39/39] minor fixes regarding installation --- Dockerfile | 3 +-- nd2reader/parser.py | 1 + setup.py | 11 +++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51e6ff0..37f37ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,7 @@ MAINTAINER Jim Rybarski RUN mkdir -p /var/nds2 RUN apt-get update && apt-get install -y \ - python-numpy \ - python-setuptools + python-numpy COPY . /opt/nd2reader WORKDIR /opt/nd2reader diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 650e751..a794627 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -8,6 +8,7 @@ import re import struct from StringIO import StringIO + field_of_view = namedtuple('FOV', ['number', 'x', 'y', 'z', 'pfs_offset']) diff --git a/setup.py b/setup.py index 49e06ac..6ac6438 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup setup( name="nd2reader", - packages=['nd2reader'], + packages=['nd2reader', 'nd2reader.model'], version="1.0.0", description='A tool for reading ND2 files produced by NIS Elements', author='Jim Rybarski', @@ -10,5 +10,12 @@ setup( url='https://github.com/jimrybarski/nd2reader', download_url='https://github.com/jimrybarski/nd2reader/tarball/1.0.0', keywords=['nd2', 'nikon', 'microscopy', 'NIS Elements'], - classifiers=[] + classifiers=['Development Status :: 5 - Production/Stable', + 'Intended Audience :: Science/Research', + 'License :: Freely Distributable', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.7', + 'Topic :: Scientific/Engineering', + ] )