From 095814450c7d25b6a1a0fb3b9ddb599fe3725eec Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Tue, 26 May 2015 06:05:41 +0000 Subject: [PATCH 01/20] #39 halfway implemented getitem. can you get non-cyclical field of view orders? if so that would invalidate this approach --- nd2reader/__init__.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 26a6f1b..75c0741 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -56,6 +56,31 @@ class Nd2(Nd2Parser): if image is not None: yield image + def __getitem__(self, item): + if isinstance(item, int): + try: + channel_offset = item % self._channel_count + fov = self._calculate_field_of_view(item) + channel = self._calculate_channel(item) + z_level = self._calculate_z_level(item) + item -= channel_offset + item /= self._channel_count + timestamp, raw_image_data = self._get_raw_image_data(item, channel_offset) + image = Image(timestamp, raw_image_data, fov, channel, z_level, self.height, self.width) + except TypeError: + return None + else: + return image + + def _calculate_field_of_view(self, frame_number): + return (frame_number - (frame_number % (self._channel_count + self._z_level_count))) % self._field_of_view_count + + def _calculate_channel(self, frame_number): + pass + + def _calculate_z_level(self, frame_number): + pass + @property def image_sets(self): """ From 8733b366d8ec482d8d5a52aae4193b0036abd81f Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 30 May 2015 22:35:09 +0000 Subject: [PATCH 02/20] #39 --- nd2reader/__init__.py | 10 ++--- nd2reader/parser.py | 95 +++++++++++++++++++++---------------------- 2 files changed, 52 insertions(+), 53 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 75c0741..efad3a9 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -73,13 +73,13 @@ class Nd2(Nd2Parser): return image def _calculate_field_of_view(self, frame_number): - return (frame_number - (frame_number % (self._channel_count + self._z_level_count))) % self._field_of_view_count + return (frame_number - (frame_number % (len(self.channels) + len(self.z_levels))) % len(self.fields_of_view) def _calculate_channel(self, frame_number): - pass + return self._channels[frame_number % self._channel_count] def _calculate_z_level(self, frame_number): - pass + return self._z_levels[] @property def image_sets(self): @@ -119,9 +119,9 @@ class Nd2(Nd2Parser): :rtype: nd2reader.model.Image() or None """ - image_set_number = self._calculate_image_group_number(time_index, field_of_view, z_level) + image_group_number = self._calculate_image_group_number(time_index, field_of_view, z_level) try: - timestamp, raw_image_data = self._get_raw_image_data(image_set_number, self._channel_offset[channel_name]) + timestamp, raw_image_data = self._get_raw_image_data(image_group_number, self._channel_offset[channel_name]) image = Image(timestamp, raw_image_data, field_of_view, channel_name, z_level, self.height, self.width) except TypeError: return None diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 58914a3..2758879 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -21,13 +21,18 @@ class Nd2Parser(object): def __init__(self, filename): self._filename = filename self._fh = None + self._channels = None + self._channel_count = None self._chunk_map_start_location = None self._cursor_position = 0 self._dimension_text = None + self._fields_of_view = None self._label_map = {} self.metadata = {} self._read_map() + self._time_indexes = None self._parse_metadata() + self._z_levels = None @property def _file_handle(self): @@ -93,7 +98,7 @@ class Nd2Parser(object): return self._dimension_text @property - def _channels(self): + def channels(self): """ These are labels created by the NIS Elements user. Typically they may a short description of the filter cube used (e.g. "bright field", "GFP", etc.) @@ -101,19 +106,21 @@ class Nd2Parser(object): :rtype: str """ - metadata = self.metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] - try: - validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] - except KeyError: - # If none of the channels have been deleted, there is no validity list, so we just make one - validity = [True for _ in metadata] - # Channel information is contained in dictionaries with the keys a0, a1...an where the number - # indicates the order in which the channel is stored. So by sorting the dicts alphabetically - # we get the correct order. - for (label, chan), valid in zip(sorted(metadata['sPlaneNew'].items()), validity): - if not valid: - continue - yield chan['sDescription'] + if not self._channels: + metadata = self.metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] + try: + validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] + except KeyError: + # If none of the channels have been deleted, there is no validity list, so we just make one + validity = [True for _ in metadata] + # Channel information is contained in dictionaries with the keys a0, a1...an where the number + # indicates the order in which the channel is stored. So by sorting the dicts alphabetically + # we get the correct order. + for (label, chan), valid in zip(sorted(metadata['sPlaneNew'].items()), validity): + if not valid: + continue + self._channels.append(chan['sDescription']) + return self._channels def _calculate_image_group_number(self, time_index, fov, z_level): """ @@ -126,7 +133,7 @@ class Nd2Parser(object): :rtype: int """ - return time_index * self._field_of_view_count * self._z_level_count + (fov * self._z_level_count + z_level) + return time_index * len(self.fields_of_view) * len(self.z_levels) + (fov * len(self.z_levels) + z_level) @property def _channel_offset(self): @@ -167,24 +174,28 @@ class Nd2Parser(object): return absolute_start_12 if absolute_start_12 else absolute_start_24 raise ValueError("This ND2 has no recorded start time. This is probably a bug.") + def _parse_dimension_text(self, pattern): + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return [0] + else: + return range(count) + @property - def _channel_count(self): + def channel_count(self): """ The number of different channels used, including bright field. :rtype: int """ - pattern = r""".*?λ\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count + if self._channel_count is None: + self._channel_count = self._parse_dimension_text(r""".*?λ\((\d+)\).*?""") + return self._channel_count @property - def _field_of_view_count(self): + def fields_of_view(self): """ The metadata contains information about fields of view, but it contains it even if some fields of view were cropped. We can't find anything that states which fields of view are actually @@ -194,45 +205,33 @@ class Nd2Parser(object): :rtype: int """ - pattern = r""".*?XY\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count + if self._fields_of_view is None: + self._fields_of_view = self._parse_dimension_text(r""".*?XY\((\d+)\).*?""") + return self._fields_of_view @property - def _time_index_count(self): + def time_indexes(self): """ The number of cycles. :rtype: int """ - pattern = r""".*?T'\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count + if self._time_indexes is None: + self._time_indexes = self._parse_dimension_text(r""".*?T'\((\d+)\).*?""") + return self._time_indexes @property - def _z_level_count(self): + def z_levels(self): """ - The number of different levels in the Z-plane. + The different levels in the Z-plane. Just a sequence from 0 to n. :rtype: int """ - pattern = r""".*?Z\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count + if self._z_levels is None: + self._z_levels = self._parse_dimension_text(r""".*?Z\((\d+)\).*?""") + return self._z_levels @property def _image_count(self): From c9d3a82930e2a3c9a8ae2f76dddf6facf470ed28 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 02:31:11 +0000 Subject: [PATCH 03/20] #39: getting individual images works, however, z_level and field of view are wrong --- nd2reader/__init__.py | 28 ++++++++++++++-------------- nd2reader/parser.py | 19 ++++--------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index babff0c..6905c98 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -18,10 +18,10 @@ class Nd2(Nd2Parser): return "\n".join(["" % self._filename, "Created: %s" % self._absolute_start.strftime("%Y-%m-%d %H:%M:%S"), "Image size: %sx%s (HxW)" % (self.height, self.width), - "Image cycles: %s" % self._time_index_count, - "Channels: %s" % ", ".join(["'%s'" % str(channel) for channel in self._channels]), - "Fields of View: %s" % self._field_of_view_count, - "Z-Levels: %s" % self._z_level_count + "Image cycles: %s" % len(self.time_indexes), + "Channels: %s" % ", ".join(["'%s'" % str(channel) for channel in self.channels]), + "Fields of View: %s" % len(self.fields_of_view), + "Z-Levels: %s" % len(self.z_levels) ]) def __len__(self): @@ -60,8 +60,8 @@ class Nd2(Nd2Parser): """ for i in range(self._image_count): - for fov in range(self._field_of_view_count): - for z_level in range(self._z_level_count): + for fov in self._fields_of_view: + for z_level in self._z_levels: for channel_name in self._channels: image = self.get_image(i, fov, channel_name, z_level) if image is not None: @@ -70,12 +70,12 @@ class Nd2(Nd2Parser): def __getitem__(self, item): if isinstance(item, int): try: - channel_offset = item % self._channel_count + channel_offset = item % len(self.channels) fov = self._calculate_field_of_view(item) channel = self._calculate_channel(item) z_level = self._calculate_z_level(item) item -= channel_offset - item /= self._channel_count + item /= len(self.channels) timestamp, raw_image_data = self._get_raw_image_data(item, channel_offset) image = Image(timestamp, raw_image_data, fov, channel, z_level, self.height, self.width) except TypeError: @@ -84,13 +84,13 @@ class Nd2(Nd2Parser): return image def _calculate_field_of_view(self, frame_number): - return (frame_number - (frame_number % (len(self.channels) + len(self.z_levels))) % len(self.fields_of_view) + return (frame_number - (frame_number % (len(self.channels) + len(self.z_levels)))) % len(self.fields_of_view) def _calculate_channel(self, frame_number): - return self._channels[frame_number % self._channel_count] + return self._channels[frame_number % len(self.channels)] def _calculate_z_level(self, frame_number): - return self._z_levels[] + return self.z_levels[frame_number % len(self.channels) % len(self.fields_of_view)] @property def image_sets(self): @@ -103,11 +103,11 @@ class Nd2(Nd2Parser): :return: model.ImageSet() """ - for time_index in range(self._time_index_count): + for time_index in self._time_indexes: image_set = ImageSet() - for fov in range(self._field_of_view_count): + for fov in self._fields_of_view: for channel_name in self._channels: - for z_level in range(self._z_level_count): + for z_level in self._z_levels: image = self.get_image(time_index, fov, channel_name, z_level) if image is not None: image_set.add(image) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 9ec7277..9566f21 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -95,8 +95,6 @@ class Nd2Parser(object): break else: raise ValueError("Could not parse metadata dimensions!") - if six.PY3: - return self._dimension_text.decode("utf8") return self._dimension_text @property @@ -183,20 +181,11 @@ class Nd2Parser(object): count = int(re.match(pattern, self._dimensions).group(1)) except AttributeError: return [0] + except TypeError: + count = int(re.match(pattern, self._dimensions.decode("utf8")).group(1)) + return list(range(count)) else: - return range(count) - - @property - def channel_count(self): - """ - The number of different channels used, including bright field. - - :rtype: int - - """ - if self._channel_count is None: - self._channel_count = self._parse_dimension_text(r""".*?λ\((\d+)\).*?""") - return self._channel_count + return list(range(count)) @property def fields_of_view(self): From e8cb08f4ccec268a8a9c5212905986db881d6a07 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 03:14:05 +0000 Subject: [PATCH 04/20] #39: calculating channel and z_level works right --- nd2reader/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 6905c98..47ccdaa 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -84,13 +84,13 @@ class Nd2(Nd2Parser): return image def _calculate_field_of_view(self, frame_number): - return (frame_number - (frame_number % (len(self.channels) + len(self.z_levels)))) % len(self.fields_of_view) + return frame_number - (frame_number % (len(self.z_levels) + len(self.channels))) def _calculate_channel(self, frame_number): return self._channels[frame_number % len(self.channels)] def _calculate_z_level(self, frame_number): - return self.z_levels[frame_number % len(self.channels) % len(self.fields_of_view)] + return self.z_levels[(frame_number * len(self.channels)) % len(self.z_levels) - frame_number % len(self.channels)] @property def image_sets(self): From 0f9c6128e3af7cfbb738c27e745d049bea15db0f Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 04:00:11 +0000 Subject: [PATCH 05/20] #39: field of view works --- nd2reader/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 47ccdaa..0f40628 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -84,13 +84,13 @@ class Nd2(Nd2Parser): return image def _calculate_field_of_view(self, frame_number): - return frame_number - (frame_number % (len(self.z_levels) + len(self.channels))) + return int((frame_number - (frame_number % (len(self.z_levels) * len(self.channels)))) / (len(self.z_levels) * len(self.channels))) def _calculate_channel(self, frame_number): return self._channels[frame_number % len(self.channels)] def _calculate_z_level(self, frame_number): - return self.z_levels[(frame_number * len(self.channels)) % len(self.z_levels) - frame_number % len(self.channels)] + return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / 2) % len(self.z_levels))] @property def image_sets(self): From 89ddce3a7bf3acc049eed2155f91723d7a4309a4 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 04:43:06 +0000 Subject: [PATCH 06/20] resolves #39 --- nd2reader/__init__.py | 90 ++++++++++++++++++------------------------- nd2reader/parser.py | 2 +- setup.py | 2 +- 3 files changed, 39 insertions(+), 55 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 0f40628..5166932 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -34,64 +34,20 @@ class Nd2(Nd2Parser): """ return self._image_count * self._channel_count - @property - def height(self): - """ - :return: height of each image, in pixels - :rtype: int - - """ - return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiHeight')] - - @property - def width(self): - """ - :return: width of each image, in pixels - :rtype: int - - """ - return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiWidth')] - - def __iter__(self): - """ - Iterates over every image, in the order they were taken. - - :return: model.Image() - - """ - for i in range(self._image_count): - for fov in self._fields_of_view: - for z_level in self._z_levels: - for channel_name in self._channels: - image = self.get_image(i, fov, channel_name, z_level) - if image is not None: - yield image - - def __getitem__(self, item): - if isinstance(item, int): + def __getitem__(self, frame_number): + if isinstance(frame_number, int): try: - channel_offset = item % len(self.channels) - fov = self._calculate_field_of_view(item) - channel = self._calculate_channel(item) - z_level = self._calculate_z_level(item) - item -= channel_offset - item /= len(self.channels) - timestamp, raw_image_data = self._get_raw_image_data(item, channel_offset) + channel_offset = frame_number % len(self.channels) + fov = self._calculate_field_of_view(frame_number) + channel = self._calculate_channel(frame_number) + z_level = self._calculate_z_level(frame_number) + timestamp, raw_image_data = self._get_raw_image_data(frame_number, channel_offset) image = Image(timestamp, raw_image_data, fov, channel, z_level, self.height, self.width) - except TypeError: + except (TypeError, ValueError): return None else: return image - def _calculate_field_of_view(self, frame_number): - return int((frame_number - (frame_number % (len(self.z_levels) * len(self.channels)))) / (len(self.z_levels) * len(self.channels))) - - def _calculate_channel(self, frame_number): - return self._channels[frame_number % len(self.channels)] - - def _calculate_z_level(self, frame_number): - return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / 2) % len(self.z_levels))] - @property def image_sets(self): """ @@ -103,7 +59,7 @@ class Nd2(Nd2Parser): :return: model.ImageSet() """ - for time_index in self._time_indexes: + for time_index in self.time_indexes: image_set = ImageSet() for fov in self._fields_of_view: for channel_name in self._channels: @@ -113,6 +69,34 @@ class Nd2(Nd2Parser): image_set.add(image) yield image_set + @property + def height(self): + """ + :return: height of each image, in pixels + :rtype: int + + """ + return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiHeight')] + + @property + def width(self): + """ + :return: width of each image, in pixels + :rtype: int + + """ + return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiWidth')] + + def _calculate_field_of_view(self, frame_number): + images_per_cycle = len(self.z_levels) * len(self.channels) + return int((frame_number - (frame_number % images_per_cycle)) / images_per_cycle) % len(self.fields_of_view) + + def _calculate_channel(self, frame_number): + return self._channels[frame_number % len(self.channels)] + + def _calculate_z_level(self, frame_number): + return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / len(self.channels)) % len(self.z_levels))] + def get_image(self, time_index, field_of_view, channel_name, z_level): """ Returns an Image if data exists for the given parameters, otherwise returns None. In general, you should avoid diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 9566f21..5aabcaa 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -63,7 +63,7 @@ class Nd2Parser(object): # The images for the various channels are interleaved within the same array. For example, the second image # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design # a data structure that way, please send the author of this library a message. - image_data = image_group_data[image_data_start::self._channel_count] + image_data = image_group_data[image_data_start::len(self.channels)] # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you # don't have the same number of images each cycle. We discovered this because we only took GFP images every # other cycle to reduce phototoxicity, but NIS Elements still allocated memory as if we were going to take diff --git a/setup.py b/setup.py index d250eec..ee88e1d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from distutils.core import setup -VERSION = "1.0.1" +VERSION = "1.1.0" setup( name="nd2reader", From 454f5529fee06f592f388610c3864605be9d8150 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 04:45:02 +0000 Subject: [PATCH 07/20] #39 fixed image sets --- nd2reader/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 5166932..754c6ef 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -61,9 +61,9 @@ class Nd2(Nd2Parser): """ for time_index in self.time_indexes: image_set = ImageSet() - for fov in self._fields_of_view: - for channel_name in self._channels: - for z_level in self._z_levels: + for fov in self.fields_of_view: + for channel_name in self.channels: + for z_level in self.z_levels: image = self.get_image(time_index, fov, channel_name, z_level) if image is not None: image_set.add(image) From a40b243abb0685a6d873ee554807840154b5c680 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 05:31:06 +0000 Subject: [PATCH 08/20] resolves #40: adds slicing semantics --- nd2reader/__init__.py | 50 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 754c6ef..7f9cf1e 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -34,19 +34,55 @@ class Nd2(Nd2Parser): """ return self._image_count * self._channel_count - def __getitem__(self, frame_number): - if isinstance(frame_number, int): + def __getitem__(self, item): + """ + Allows slicing ND2s. + + >>> nd2 = Nd2("my_images.nd2") + >>> image = nd2[16] # gets 17th frame + >>> for image in nd2[100:200]: # iterate over the 100th to 200th images + >>> do_something(image.data) + >>> for image in nd2[::-1]: # iterate backwards + >>> do_something(image.data) + >>> for image in nd2[37:422:17]: # do something super weird if you really want to + >>> do_something(image.data) + + :type item: int or slice + :rtype: nd2reader.model.Image() or generator + + """ + if isinstance(item, int): try: - channel_offset = frame_number % len(self.channels) - fov = self._calculate_field_of_view(frame_number) - channel = self._calculate_channel(frame_number) - z_level = self._calculate_z_level(frame_number) - timestamp, raw_image_data = self._get_raw_image_data(frame_number, channel_offset) + channel_offset = item % len(self.channels) + fov = self._calculate_field_of_view(item) + channel = self._calculate_channel(item) + z_level = self._calculate_z_level(item) + timestamp, raw_image_data = self._get_raw_image_data(item, channel_offset) image = Image(timestamp, raw_image_data, fov, channel, z_level, self.height, self.width) except (TypeError, ValueError): return None else: return image + elif isinstance(item, slice): + return self._slice(item.start, item.stop, item.step) + raise IndexError + + def _slice(self, start, stop, step): + """ + Allows for iteration over a selection of the entire dataset. + + :type start: int + :type stop: int + :type step: int + :rtype: nd2reader.model.Image() or None + + """ + start = start if start is not None else 0 + step = step if step is not None else 1 + stop = stop if stop is not None else len(self) + # This weird thing with the step allows you to iterate backwards over the images + for i in range(start, stop)[::step]: + yield self[i] @property def image_sets(self): From 3f126dadb98202f45181f691e3f4ff78e0b5b956 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 06:21:45 +0000 Subject: [PATCH 09/20] two dockerfiles as prelude to testing --- Dockerfile => Dockerfile.py27 | 2 +- Dockerfile.py34 | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) rename Dockerfile => Dockerfile.py27 (75%) create mode 100644 Dockerfile.py34 diff --git a/Dockerfile b/Dockerfile.py27 similarity index 75% rename from Dockerfile rename to Dockerfile.py27 index 37f37ee..d2419d3 100644 --- a/Dockerfile +++ b/Dockerfile.py27 @@ -2,7 +2,7 @@ FROM ubuntu MAINTAINER Jim Rybarski RUN mkdir -p /var/nds2 -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ python-numpy COPY . /opt/nd2reader diff --git a/Dockerfile.py34 b/Dockerfile.py34 new file mode 100644 index 0000000..a9181fa --- /dev/null +++ b/Dockerfile.py34 @@ -0,0 +1,13 @@ +FROM ubuntu +MAINTAINER Jim Rybarski + +RUN mkdir -p /var/nds2 +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-numpy + +COPY . /opt/nd2reader +WORKDIR /opt/nd2reader +RUN python3 setup.py install +WORKDIR /var/nd2s + +CMD /usr/bin/python3.4 \ No newline at end of file From 5104a232ffcfdadb3e7291c24ad1062ebbb0acbe Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Tue, 2 Jun 2015 01:37:38 +0000 Subject: [PATCH 10/20] combined python2.7 and 3.4 into one Dockerfile since it's just for testing anyway --- Dockerfile.py34 => Dockerfile | 9 ++++++++- Dockerfile.py27 | 13 ------------- Makefile | 7 +++++-- 3 files changed, 13 insertions(+), 16 deletions(-) rename Dockerfile.py34 => Dockerfile (59%) delete mode 100644 Dockerfile.py27 diff --git a/Dockerfile.py34 b/Dockerfile similarity index 59% rename from Dockerfile.py34 rename to Dockerfile index a9181fa..0209f2a 100644 --- a/Dockerfile.py34 +++ b/Dockerfile @@ -3,10 +3,17 @@ MAINTAINER Jim Rybarski RUN mkdir -p /var/nds2 RUN apt-get update && apt-get install -y --no-install-recommends \ - python3-numpy + python-numpy \ + python3-numpy \ + python-pip \ + python3-pip + +RUN pip install six +RUN pip3 install six COPY . /opt/nd2reader WORKDIR /opt/nd2reader +RUN python setup.py install RUN python3 setup.py install WORKDIR /var/nd2s diff --git a/Dockerfile.py27 b/Dockerfile.py27 deleted file mode 100644 index d2419d3..0000000 --- a/Dockerfile.py27 +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu -MAINTAINER Jim Rybarski - -RUN mkdir -p /var/nds2 -RUN apt-get update && apt-get install -y --no-install-recommends \ - python-numpy - -COPY . /opt/nd2reader -WORKDIR /opt/nd2reader -RUN python setup.py install -WORKDIR /var/nd2s - -CMD /usr/bin/python2.7 diff --git a/Makefile b/Makefile index 2438069..89b185e 100644 --- a/Makefile +++ b/Makefile @@ -3,5 +3,8 @@ build: docker build -t jimrybarski/nd2reader . -shell: - docker run --rm -v ~/Documents/nd2s:/var/nd2s -it jimrybarski/nd2reader +py2shell: + docker run --rm -v ~/Documents/nd2s:/var/nd2s -it jimrybarski/nd2reader python2.7 + +py3shell: + docker run --rm -v ~/Documents/nd2s:/var/nd2s -it jimrybarski/nd2reader python3.4 From ae09b688b289f85dc8604e83f04cbbddf53faa77 Mon Sep 17 00:00:00 2001 From: jim Date: Wed, 3 Jun 2015 12:56:11 -0500 Subject: [PATCH 11/20] #42 refactored and tidied up a bit, created test files --- nd2reader/__init__.py | 4 +- nd2reader/parser.py | 189 +++++++++++++++++++++--------------------- tests.py | 5 ++ tests/__init__.py | 0 4 files changed, 103 insertions(+), 95 deletions(-) create mode 100644 tests.py create mode 100644 tests/__init__.py diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 7f9cf1e..aa0d14f 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -16,7 +16,7 @@ class Nd2(Nd2Parser): def __repr__(self): return "\n".join(["" % self._filename, - "Created: %s" % self._absolute_start.strftime("%Y-%m-%d %H:%M:%S"), + "Created: %s" % self.absolute_start.strftime("%Y-%m-%d %H:%M:%S"), "Image size: %sx%s (HxW)" % (self.height, self.width), "Image cycles: %s" % len(self.time_indexes), "Channels: %s" % ", ".join(["'%s'" % str(channel) for channel in self.channels]), @@ -32,7 +32,7 @@ class Nd2(Nd2Parser): :rtype: int """ - return self._image_count * self._channel_count + return self._total_images_per_channel * self._channel_count def __getitem__(self, item): """ diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 5aabcaa..34acc24 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -19,6 +19,7 @@ class Nd2Parser(object): CHUNK_MAP_END = six.b("ND2 CHUNK MAP SIGNATURE 0000001!") def __init__(self, filename): + self._absolute_start = None self._filename = filename self._fh = None self._channels = None @@ -34,6 +35,99 @@ class Nd2Parser(object): self._parse_metadata() self._z_levels = None + @property + def absolute_start(self): + """ + The date and time when acquisition began. + + :rtype: datetime.datetime() + + """ + if self._absolute_start is None: + for line in self.metadata[six.b('ImageTextInfo')][six.b('SLxImageTextInfo')].values(): + line = line.decode("utf8") + absolute_start_12 = None + absolute_start_24 = None + # ND2s seem to randomly switch between 12- and 24-hour representations. + try: + absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") + except (TypeError, ValueError): + pass + try: + absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") + except (TypeError, ValueError): + pass + if not absolute_start_12 and not absolute_start_24: + continue + return absolute_start_12 if absolute_start_12 else absolute_start_24 + raise ValueError("This ND2 has no recorded start time. This is probably a bug.") + return self._absolute_start + + @property + def channels(self): + """ + These are labels created by the NIS Elements user. Typically they may a short description of the filter cube + used (e.g. "bright field", "GFP", etc.) + + :rtype: str + + """ + if not self._channels: + self._channels = [] + metadata = self.metadata[six.b('ImageMetadataSeq')][six.b('SLxPictureMetadata')][six.b('sPicturePlanes')] + try: + validity = self.metadata[six.b('ImageMetadata')][six.b('SLxExperiment')][six.b('ppNextLevelEx')][six.b('')][0][six.b('ppNextLevelEx')][six.b('')][0][six.b('pItemValid')] + except KeyError: + # If none of the channels have been deleted, there is no validity list, so we just make one + validity = [True for _ in metadata] + # Channel information is contained in dictionaries with the keys a0, a1...an where the number + # indicates the order in which the channel is stored. So by sorting the dicts alphabetically + # we get the correct order. + for (label, chan), valid in zip(sorted(metadata[six.b('sPlaneNew')].items()), validity): + if not valid: + continue + self._channels.append(chan[six.b('sDescription')].decode("utf8")) + return self._channels + + @property + def fields_of_view(self): + """ + The metadata contains information about fields of view, but it contains it even if some fields + of view were cropped. We can't find anything that states which fields of view are actually + in the image data, so we have to calculate it. There probably is something somewhere, since + NIS Elements can figure it out, but we haven't found it yet. + + :rtype: int + + """ + if self._fields_of_view is None: + self._fields_of_view = self._parse_dimension_text(r""".*?XY\((\d+)\).*?""") + return self._fields_of_view + + @property + def time_indexes(self): + """ + The number of cycles. + + :rtype: int + + """ + if self._time_indexes is None: + self._time_indexes = self._parse_dimension_text(r""".*?T'\((\d+)\).*?""") + return self._time_indexes + + @property + def z_levels(self): + """ + The different levels in the Z-plane. Just a sequence from 0 to n. + + :rtype: int + + """ + if self._z_levels is None: + self._z_levels = self._parse_dimension_text(r""".*?Z\((\d+)\).*?""") + return self._z_levels + @property def _file_handle(self): if self._fh is None: @@ -97,32 +191,6 @@ class Nd2Parser(object): raise ValueError("Could not parse metadata dimensions!") return self._dimension_text - @property - def channels(self): - """ - These are labels created by the NIS Elements user. Typically they may a short description of the filter cube - used (e.g. "bright field", "GFP", etc.) - - :rtype: str - - """ - if not self._channels: - self._channels = [] - metadata = self.metadata[six.b('ImageMetadataSeq')][six.b('SLxPictureMetadata')][six.b('sPicturePlanes')] - try: - validity = self.metadata[six.b('ImageMetadata')][six.b('SLxExperiment')][six.b('ppNextLevelEx')][six.b('')][0][six.b('ppNextLevelEx')][six.b('')][0][six.b('pItemValid')] - except KeyError: - # If none of the channels have been deleted, there is no validity list, so we just make one - validity = [True for _ in metadata] - # Channel information is contained in dictionaries with the keys a0, a1...an where the number - # indicates the order in which the channel is stored. So by sorting the dicts alphabetically - # we get the correct order. - for (label, chan), valid in zip(sorted(metadata[six.b('sPlaneNew')].items()), validity): - if not valid: - continue - self._channels.append(chan[six.b('sDescription')].decode("utf8")) - return self._channels - def _calculate_image_group_number(self, time_index, fov, z_level): """ Images are grouped together if they share the same time index, field of view, and z-level. @@ -150,32 +218,6 @@ class Nd2Parser(object): channel_offset[channel] = n return channel_offset - @property - def _absolute_start(self): - """ - The date and time when acquisition began. - - :rtype: datetime.datetime() - - """ - for line in self.metadata[six.b('ImageTextInfo')][six.b('SLxImageTextInfo')].values(): - line = line.decode("utf8") - absolute_start_12 = None - absolute_start_24 = None - # ND2s seem to randomly switch between 12- and 24-hour representations. - try: - absolute_start_24 = datetime.strptime(line, "%m/%d/%Y %H:%M:%S") - except (TypeError, ValueError): - pass - try: - absolute_start_12 = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p") - except (TypeError, ValueError): - pass - if not absolute_start_12 and not absolute_start_24: - continue - return absolute_start_12 if absolute_start_12 else absolute_start_24 - raise ValueError("This ND2 has no recorded start time. This is probably a bug.") - def _parse_dimension_text(self, pattern): try: count = int(re.match(pattern, self._dimensions).group(1)) @@ -188,48 +230,9 @@ class Nd2Parser(object): return list(range(count)) @property - def fields_of_view(self): - """ - The metadata contains information about fields of view, but it contains it even if some fields - of view were cropped. We can't find anything that states which fields of view are actually - in the image data, so we have to calculate it. There probably is something somewhere, since - NIS Elements can figure it out, but we haven't found it yet. - - :rtype: int - - """ - if self._fields_of_view is None: - self._fields_of_view = self._parse_dimension_text(r""".*?XY\((\d+)\).*?""") - return self._fields_of_view - - @property - def time_indexes(self): - """ - The number of cycles. - - :rtype: int - - """ - if self._time_indexes is None: - self._time_indexes = self._parse_dimension_text(r""".*?T'\((\d+)\).*?""") - return self._time_indexes - - @property - def z_levels(self): - """ - The different levels in the Z-plane. Just a sequence from 0 to n. - - :rtype: int - - """ - if self._z_levels is None: - self._z_levels = self._parse_dimension_text(r""".*?Z\((\d+)\).*?""") - return self._z_levels - - @property - def _image_count(self): + def _total_images_per_channel(self): """ - The total number of images in the ND2. Warning: this may be inaccurate as it includes "gap" images. + The total number of images per channel. Warning: this may be inaccurate as it includes "gap" images. :rtype: int diff --git a/tests.py b/tests.py new file mode 100644 index 0000000..49b4387 --- /dev/null +++ b/tests.py @@ -0,0 +1,5 @@ +import unittest +loader = unittest.TestLoader() +tests = loader.discover('tests', pattern='*.py', top_level_dir='.') +testRunner = unittest.TextTestRunner() +testRunner.run(tests) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 07e2297aaf0a70b4556755703a64aaf0a83f1525 Mon Sep 17 00:00:00 2001 From: jim Date: Wed, 3 Jun 2015 13:32:01 -0500 Subject: [PATCH 12/20] #42 fixed dockerfile and makefile, got one passing test --- Dockerfile | 5 +---- Makefile | 4 ++++ nd2reader/__init__.py | 10 ---------- nd2reader/parser.py | 10 ++++++++++ tests/__init__.py | 19 +++++++++++++++++++ 5 files changed, 34 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0209f2a..8f656a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu +FROM ubuntu:15.04 MAINTAINER Jim Rybarski RUN mkdir -p /var/nds2 @@ -15,6 +15,3 @@ COPY . /opt/nd2reader WORKDIR /opt/nd2reader RUN python setup.py install RUN python3 setup.py install -WORKDIR /var/nd2s - -CMD /usr/bin/python3.4 \ No newline at end of file diff --git a/Makefile b/Makefile index 89b185e..87b2d38 100644 --- a/Makefile +++ b/Makefile @@ -8,3 +8,7 @@ py2shell: py3shell: docker run --rm -v ~/Documents/nd2s:/var/nd2s -it jimrybarski/nd2reader python3.4 + +test: build + docker run --rm -it jimrybarski/nd2reader python3.4 /opt/nd2reader/tests.py + diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index aa0d14f..f3f5d7b 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -123,16 +123,6 @@ class Nd2(Nd2Parser): """ return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiWidth')] - def _calculate_field_of_view(self, frame_number): - images_per_cycle = len(self.z_levels) * len(self.channels) - return int((frame_number - (frame_number % images_per_cycle)) / images_per_cycle) % len(self.fields_of_view) - - def _calculate_channel(self, frame_number): - return self._channels[frame_number % len(self.channels)] - - def _calculate_z_level(self, frame_number): - return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / len(self.channels)) % len(self.z_levels))] - def get_image(self, time_index, field_of_view, channel_name, z_level): """ Returns an Image if data exists for the given parameters, otherwise returns None. In general, you should avoid diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 34acc24..f5bbda5 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -128,6 +128,16 @@ class Nd2Parser(object): self._z_levels = self._parse_dimension_text(r""".*?Z\((\d+)\).*?""") return self._z_levels + def _calculate_field_of_view(self, frame_number): + images_per_cycle = len(self.z_levels) * len(self.channels) + return int((frame_number - (frame_number % images_per_cycle)) / images_per_cycle) % len(self.fields_of_view) + + def _calculate_channel(self, frame_number): + return self._channels[frame_number % len(self.channels)] + + def _calculate_z_level(self, frame_number): + return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / len(self.channels)) % len(self.z_levels))] + @property def _file_handle(self): if self._fh is None: diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..531c513 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,19 @@ +from nd2reader.parser import Nd2Parser +import unittest + + +class MockNd2Parser(object): + def __init__(self, channels, fields_of_view, z_levels): + self.channels = channels + self.fields_of_view = fields_of_view + self.z_levels = z_levels + + +class TestNd2Parser(unittest.TestCase): + def test_calculate_field_of_view_simple(self): + """ With a single field of view, the field of view should always be the same number (0). """ + nd2 = MockNd2Parser([''], [0], [0]) + for frame_number in range(1000): + result = Nd2Parser._calculate_field_of_view(nd2, frame_number) + self.assertEqual(result, 0) + From 2b62ed4d2df554348119918da218d292abf006aa Mon Sep 17 00:00:00 2001 From: jim Date: Wed, 3 Jun 2015 13:48:26 -0500 Subject: [PATCH 13/20] #42 many passing tests for calculate_field_of_view --- tests/__init__.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index 531c513..a20d35a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -17,3 +17,67 @@ class TestNd2Parser(unittest.TestCase): result = Nd2Parser._calculate_field_of_view(nd2, frame_number) self.assertEqual(result, 0) + def test_calculate_field_of_view_two_channels(self): + nd2 = MockNd2Parser(['', 'GFP'], [0], [0]) + for frame_number in range(1000): + result = Nd2Parser._calculate_field_of_view(nd2, frame_number) + self.assertEqual(result, 0) + + def test_calculate_field_of_view_three_channels(self): + nd2 = MockNd2Parser(['', 'GFP', 'dsRed'], [0], [0]) + for frame_number in range(1000): + result = Nd2Parser._calculate_field_of_view(nd2, frame_number) + self.assertEqual(result, 0) + + def test_calculate_field_of_view_two_fovs(self): + nd2 = MockNd2Parser([''], [0, 1], [0]) + for frame_number in range(1000): + result = Nd2Parser._calculate_field_of_view(nd2, frame_number) + self.assertEqual(result, frame_number % 2) + + def test_calculate_field_of_view_two_fovs_two_zlevels(self): + nd2 = MockNd2Parser([''], [0, 1], [0, 1]) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 0), 0) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 1), 0) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 2), 1) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 3), 1) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 4), 0) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 5), 0) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 6), 1) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 7), 1) + + def test_calculate_field_of_view_two_everything(self): + nd2 = MockNd2Parser(['', 'GFP'], [0, 1], [0, 1]) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 0), 0) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 1), 0) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 2), 0) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 3), 0) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 4), 1) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 5), 1) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 6), 1) + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, 7), 1) + + def test_calculate_field_of_view_7c2f2z(self): + nd2 = MockNd2Parser(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], [0, 1], [0, 1]) + for i in range(14): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 0) + for i in range(14, 28): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 1) + for i in range(28, 42): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 0) + for i in range(42, 56): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 1) + for i in range(56, 70): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 0) + for i in range(70, 84): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 1) + + def test_calculate_channel_simple(self): + + + + + + + + From 047af59a1a59f3a3f064d63ab3e36266350b1408 Mon Sep 17 00:00:00 2001 From: jim Date: Wed, 3 Jun 2015 14:01:29 -0500 Subject: [PATCH 14/20] #42 many passing tests for calculate_channel --- nd2reader/parser.py | 2 +- tests/__init__.py | 42 +++++++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index f5bbda5..139e2fb 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -133,7 +133,7 @@ class Nd2Parser(object): return int((frame_number - (frame_number % images_per_cycle)) / images_per_cycle) % len(self.fields_of_view) def _calculate_channel(self, frame_number): - return self._channels[frame_number % len(self.channels)] + return self.channels[frame_number % len(self.channels)] def _calculate_z_level(self, frame_number): return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / len(self.channels)) % len(self.z_levels))] diff --git a/tests/__init__.py b/tests/__init__.py index a20d35a..6edf425 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -72,12 +72,36 @@ class TestNd2Parser(unittest.TestCase): for i in range(70, 84): self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 1) - def test_calculate_channel_simple(self): - - - - - - - - + def test_calculate_field_of_view_2c3f5z(self): + """ All prime numbers to elucidate any errors that won't show up when numbers are multiples of each other """ + nd2 = MockNd2Parser(['', 'GFP'], [0, 1, 2], [0, 1, 2, 3, 4]) + for i in range(10): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 0) + for i in range(10, 20): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 1) + for i in range(20, 30): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 2) + for i in range(30, 40): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 0) + for i in range(40, 50): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 1) + for i in range(50, 60): + self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 2) + + def test_calculate_channel(self): + nd2 = MockNd2Parser(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], [0], [0]) + for i in range(1000): + for n, channel in enumerate(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], start=i*7): + self.assertEqual(Nd2Parser._calculate_channel(nd2, n), channel) + + def test_calculate_channel_7c2fov1z(self): + nd2 = MockNd2Parser(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], [0, 1], [0]) + for i in range(1000): + for n, channel in enumerate(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], start=i*7): + self.assertEqual(Nd2Parser._calculate_channel(nd2, n), channel) + + def test_calculate_channel_ludicrous_values(self): + nd2 = MockNd2Parser(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], list(range(31)), list(range(17))) + for i in range(1000): + for n, channel in enumerate(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], start=i*7): + self.assertEqual(Nd2Parser._calculate_channel(nd2, n), channel) \ No newline at end of file From 8ae2bf6d0688172c935159a60105e5060a8533cc Mon Sep 17 00:00:00 2001 From: jim Date: Wed, 3 Jun 2015 14:02:55 -0500 Subject: [PATCH 15/20] #42 added simple calculate_channel test --- tests/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 6edf425..f24514a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -88,6 +88,11 @@ class TestNd2Parser(unittest.TestCase): for i in range(50, 60): self.assertEqual(Nd2Parser._calculate_field_of_view(nd2, i), 2) + def test_calculate_channel_simple(self): + nd2 = MockNd2Parser(['GFP'], [0], [0]) + for i in range(1000): + self.assertEqual(Nd2Parser._calculate_channel(nd2, i), 'GFP') + def test_calculate_channel(self): nd2 = MockNd2Parser(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], [0], [0]) for i in range(1000): @@ -104,4 +109,5 @@ class TestNd2Parser(unittest.TestCase): nd2 = MockNd2Parser(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], list(range(31)), list(range(17))) for i in range(1000): for n, channel in enumerate(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], start=i*7): - self.assertEqual(Nd2Parser._calculate_channel(nd2, n), channel) \ No newline at end of file + self.assertEqual(Nd2Parser._calculate_channel(nd2, n), channel) + From c16dccd13abcd3f101803d48541f838c85584099 Mon Sep 17 00:00:00 2001 From: jim Date: Wed, 3 Jun 2015 14:15:29 -0500 Subject: [PATCH 16/20] #42 tests for calculate_z_level --- tests/__init__.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index f24514a..0a3def9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -107,7 +107,77 @@ class TestNd2Parser(unittest.TestCase): def test_calculate_channel_ludicrous_values(self): nd2 = MockNd2Parser(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], list(range(31)), list(range(17))) - for i in range(1000): + for i in range(10000): for n, channel in enumerate(['', 'GFP', 'dsRed', 'dTomato', 'lulzBlue', 'jimbotronPurple', 'orange'], start=i*7): self.assertEqual(Nd2Parser._calculate_channel(nd2, n), channel) + def test_calculate_z_level(self): + nd2 = MockNd2Parser([''], [0], [0]) + for frame_number in range(1000): + result = Nd2Parser._calculate_z_level(nd2, frame_number) + self.assertEqual(result, 0) + + def test_calculate_z_level_1c1f2z(self): + nd2 = MockNd2Parser([''], [0], [0, 1]) + for frame_number in range(1000): + result = Nd2Parser._calculate_z_level(nd2, frame_number) + self.assertEqual(result, frame_number % 2) + + def test_calculate_z_level_31c17f1z(self): + nd2 = MockNd2Parser(list(range(31)), list(range(17)), [0]) + for frame_number in range(1000): + result = Nd2Parser._calculate_z_level(nd2, frame_number) + self.assertEqual(result, 0) + + def test_calculate_z_level_2c1f2z(self): + nd2 = MockNd2Parser(['', 'GFP'], [0], [0, 1]) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 0), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 1), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 2), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 3), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 4), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 5), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 6), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 7), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 8), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 9), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 10), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 11), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 12), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 13), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 14), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 15), 1) + + def test_calculate_z_level_2c3f5z(self): + nd2 = MockNd2Parser(['', 'GFP'], [0, 1, 2], [0, 1, 2, 3, 4]) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 0), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 1), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 2), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 3), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 4), 2) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 5), 2) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 6), 3) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 7), 3) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 8), 4) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 9), 4) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 10), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 11), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 12), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 13), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 14), 2) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 15), 2) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 16), 3) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 17), 3) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 18), 4) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 19), 4) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 20), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 21), 0) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 22), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 23), 1) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 24), 2) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 25), 2) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 26), 3) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 27), 3) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 28), 4) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 29), 4) + self.assertEqual(Nd2Parser._calculate_z_level(nd2, 30), 0) \ No newline at end of file From 18bb3f539243128c1dd47ccf40a6ca966e5e555d Mon Sep 17 00:00:00 2001 From: jim Date: Wed, 3 Jun 2015 14:58:42 -0500 Subject: [PATCH 17/20] resolves #54: raises Index error when you go out of bounds --- nd2reader/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index f3f5d7b..979f5ec 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -61,6 +61,8 @@ class Nd2(Nd2Parser): image = Image(timestamp, raw_image_data, fov, channel, z_level, self.height, self.width) except (TypeError, ValueError): return None + except KeyError: + raise IndexError("Invalid frame number.") else: return image elif isinstance(item, slice): From 49737f678f9405925ab1ff094719fd3b86cc30a8 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Thu, 4 Jun 2015 03:11:10 +0000 Subject: [PATCH 18/20] resolves #56 --- CHANGELOG.md | 15 +++++ README.md | 136 +++++++++++++++++++++++++++++------------- nd2reader/__init__.py | 5 +- 3 files changed, 112 insertions(+), 44 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dfe4736 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +## [1.1.0] - 2015-06-03 +### ADDED +- Indexing and slicing of images +- Python 3 support +- Dockerfile support for Python 3.4 +- Makefile commands for convenient testing in Docker +- Unit tests + +### CHANGED +- Made the interface for most metadata public. +- Refactored some poorly-named things + +## [1.0.0] - 2015-05-23 +### Added +- First stable release! diff --git a/README.md b/README.md index d17653f..dd0763d 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,58 @@ Fields of View: 8 Z-Levels: 3 ``` -### Simple Iteration +You can also get some metadata about the nd2 programatically: -For most cases, you'll just want to iterate over each image: +```python +>>> nd2.height +1280 +>>> nd2.width +800 +>>> len(nd2) +30528 +``` + +### Images + +`nd2reader` will always return an `Image` object, which contains some metadata about the image as well as the +raw pixel data itself. Images are always a 16-bit grayscale image. The `data` attribute holds the numpy array +with the image data: + +```python +>>> image = nd2[20] +>>> print(image.data) +array([[1894, 1949, 1941, ..., 2104, 2135, 2114], + [1825, 1846, 1848, ..., 1994, 2149, 2064], + [1909, 1820, 1821, ..., 1995, 1952, 2062], + ..., + [3487, 3512, 3594, ..., 3603, 3643, 3492], + [3642, 3475, 3525, ..., 3712, 3682, 3609], + [3687, 3777, 3738, ..., 3784, 3870, 4008]], dtype=uint16) +``` + +You can get a quick summary of image data by examining the `Image` object: + +```python +>>> image + +1280x800 (HxW) +Timestamp: 1699.79478134 +Field of View: 2 +Channel: GFP +Z-Level: 1 +``` + +Or you can access it programmatically: + +```python +image = nd2[0] +print(image.timestamp) +print(image.field_of_view) +print(image.channel) +print(image.z_level) +``` + +Often, you may want to just iterate over each image: ```python import nd2reader @@ -48,10 +97,42 @@ for image in nd2: do_something(image.data) ``` +You can also get an image directly by indexing. Here, we look at the 38th image: + +```python +>>> nd2[37] + +1280x800 (HxW) +Timestamp: 1699.79478134 +Field of View: 2 +Channel: GFP +Z-Level: 1 +``` + +Slicing is also supported and is extremely memory efficient, as images are only read when directly accessed: + +```python +my_subset = nd2[50:433] +for image in my_subset: + do_something(image.data) +``` + +Step sizes are also accepted: + +```python +for image in nd2[:100:2]: + # gets every other image in the first 100 images + do_something(image.data) + +for image in nd2[::-1]: + # iterate backwards over every image, if you're into that kind of thing + do_something_image.data) +``` + ### Image Sets If you have complicated hierarchical data, it may be easier to use image sets, which groups images together if they -share the same time index and field of view: +share the same time index (not timestamp!) and field of view: ```python import nd2reader @@ -59,53 +140,26 @@ nd2 = nd2reader.Nd2("/path/to/my_complicated_images.nd2") for image_set in nd2.image_sets: # you can select images by channel gfp_image = image_set.get("GFP") - do_something_gfp_related(gfp_image) + do_something_gfp_related(gfp_image.data) # you can also specify the z-level. this defaults to 0 if not given out_of_focus_image = image_set.get("Bright Field", z_level=1) - do_something_out_of_focus_related(out_of_focus_image) + do_something_out_of_focus_related(out_of_focus_image.data) ``` -### Direct Image Access +To get an image from an image set, you must specify a channel. It defaults to the 0th z-level, so if you have +more than one z-level you will need to specify it when using `get`: -There is a method, `get_image`, which allows random access to images. This might not always return an image, however, -if you acquired different numbers of images in each cycle of a program. For example, if you acquire GFP images every -other minute, but acquire bright field images every minute, `get_image` will return `None` at certain time indexes. - -### Images +```python +image = image_set.get("YFP") +image = image_set.get("YFP", z_level=2) +``` -`Image` objects provide several pieces of useful data. +You can also see how many images are in your image set: ```python ->>> import nd2reader ->>> nd2 = nd2reader.Nd2("/path/to/my_images.nd2") ->>> image = nd2.get_image(14, 2, "GFP", 1) ->>> image.data -array([[1809, 1783, 1830, ..., 1923, 1920, 1914], - [1687, 1855, 1792, ..., 1986, 1903, 1889], - [1758, 1901, 1849, ..., 1911, 2010, 1954], - ..., - [3363, 3370, 3570, ..., 3565, 3601, 3459], - [3480, 3428, 3328, ..., 3542, 3461, 3575], - [3497, 3666, 3635, ..., 3817, 3867, 3779]]) ->>> image.channel -'GFP' ->>> image.timestamp -1699.7947813408175 ->>> image.field_of_view -2 ->>> image.z_level -1 - -# You can also get a quick summary of image data: - ->>> image - -1280x800 (HxW) -Timestamp: 1699.79478134 -Field of View: 2 -Channel: GFP -Z-Level: 1 +>>> len(image_set) +7 ``` ### Bug Reports and Features diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 979f5ec..2337034 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -32,7 +32,7 @@ class Nd2(Nd2Parser): :rtype: int """ - return self._total_images_per_channel * self._channel_count + return self._total_images_per_channel * len(self.channels) def __getitem__(self, item): """ @@ -128,8 +128,7 @@ class Nd2(Nd2Parser): def get_image(self, time_index, field_of_view, channel_name, z_level): """ Returns an Image if data exists for the given parameters, otherwise returns None. In general, you should avoid - using this method unless you're very familiar with the structure of ND2 files. If you have a use case that - cannot be met by the `__iter__` or `image_sets` methods above, please create an issue on Github. + using this method unless you're very familiar with the structure of ND2 files. :param time_index: the frame number :type time_index: int From 1fc449a6e042d90b2d8755ba7e24878eb5da3b12 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Thu, 4 Jun 2015 04:32:49 +0000 Subject: [PATCH 19/20] #56 clearer README, auto-install dependencies, build numpy from source in Docker --- CHANGELOG.md | 3 ++- Dockerfile | 13 ++++++++----- Makefile | 2 +- README.md | 23 +++++++++++++++-------- setup.py | 6 +++++- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfe4736..cdd0791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ - Unit tests ### CHANGED -- Made the interface for most metadata public. +- Switched to setuptools to automatically install missing dependencies +- Made the interface for most metadata public - Refactored some poorly-named things ## [1.0.0] - 2015-05-23 diff --git a/Dockerfile b/Dockerfile index 8f656a9..969c5b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,17 @@ MAINTAINER Jim Rybarski RUN mkdir -p /var/nds2 RUN apt-get update && apt-get install -y --no-install-recommends \ - python-numpy \ - python3-numpy \ + build-essential \ + libatlas3-base \ + liblapack-dev \ + libblas-dev \ + python \ + python3 \ + python-dev \ + python3-dev \ python-pip \ python3-pip -RUN pip install six -RUN pip3 install six - COPY . /opt/nd2reader WORKDIR /opt/nd2reader RUN python setup.py install diff --git a/Makefile b/Makefile index 87b2d38..8a6a193 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build shell +.PHONY: build py2shell py3shell test build: docker build -t jimrybarski/nd2reader . diff --git a/README.md b/README.md index dd0763d..fa0dc03 100644 --- a/README.md +++ b/README.md @@ -10,16 +10,19 @@ ### Installation -Just use pip (`numpy` is required): +Dependencies will automatically be installed if you don't have them. That said, for optimal performance, you should +install the following packages before installing nd2reader: -`pip install numpy nd2reader` +#### Ubuntu +`apt-get install python-numpy python-six` (Python 2.x) +`apt-get install python3-numpy python3-six` (Python 3.x) -If you want to install via git, clone the repo and run: +#### Other operating systems +These have not been tested yet. -``` -pip install numpy -python setup.py install -``` +nd2reader is compatible with both Python 2.x and 3.x. I recommend installing using pip: + +`pip install nd2reader` (Python 2.x) or `pip3 install nd2reader` (Python 3.x) ### ND2s @@ -50,7 +53,7 @@ You can also get some metadata about the nd2 programatically: ### Images -`nd2reader` will always return an `Image` object, which contains some metadata about the image as well as the +Every method returns an `Image` object, which contains some metadata about the image as well as the raw pixel data itself. Images are always a 16-bit grayscale image. The `data` attribute holds the numpy array with the image data: @@ -162,6 +165,10 @@ You can also see how many images are in your image set: 7 ``` +### Protips + +nd2reader is about 14 times faster under Python 3.4 compared to Python 2.7. If you know why, please get in touch! + ### Bug Reports and Features If this fails to work exactly as expected, please open a Github issue. If you get an unhandled exception, please diff --git a/setup.py b/setup.py index ee88e1d..ee1b087 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,14 @@ -from distutils.core import setup +from setuptools import setup VERSION = "1.1.0" setup( name="nd2reader", packages=['nd2reader', 'nd2reader.model'], + install_requires=[ + 'numpy>=1.6.2, <2.0', + 'six>=1.4, <2.0' + ], version=VERSION, description='A tool for reading ND2 files produced by NIS Elements', author='Jim Rybarski', From 092a993dc6d1380bf78a9cf43a73ff070970c9f0 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Thu, 4 Jun 2015 04:46:33 +0000 Subject: [PATCH 20/20] #56 fix six version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ed8da63..b8e18f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ numpy>=1.9.2 -six +six>=1.4