From 095814450c7d25b6a1a0fb3b9ddb599fe3725eec Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Tue, 26 May 2015 06:05:41 +0000 Subject: [PATCH 1/7] #39 halfway implemented getitem. can you get non-cyclical field of view orders? if so that would invalidate this approach --- nd2reader/__init__.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 26a6f1b..75c0741 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -56,6 +56,31 @@ class Nd2(Nd2Parser): if image is not None: yield image + def __getitem__(self, item): + if isinstance(item, int): + try: + channel_offset = item % self._channel_count + fov = self._calculate_field_of_view(item) + channel = self._calculate_channel(item) + z_level = self._calculate_z_level(item) + item -= channel_offset + item /= self._channel_count + timestamp, raw_image_data = self._get_raw_image_data(item, channel_offset) + image = Image(timestamp, raw_image_data, fov, channel, z_level, self.height, self.width) + except TypeError: + return None + else: + return image + + def _calculate_field_of_view(self, frame_number): + return (frame_number - (frame_number % (self._channel_count + self._z_level_count))) % self._field_of_view_count + + def _calculate_channel(self, frame_number): + pass + + def _calculate_z_level(self, frame_number): + pass + @property def image_sets(self): """ From 8733b366d8ec482d8d5a52aae4193b0036abd81f Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sat, 30 May 2015 22:35:09 +0000 Subject: [PATCH 2/7] #39 --- nd2reader/__init__.py | 10 ++--- nd2reader/parser.py | 95 +++++++++++++++++++++---------------------- 2 files changed, 52 insertions(+), 53 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 75c0741..efad3a9 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -73,13 +73,13 @@ class Nd2(Nd2Parser): return image def _calculate_field_of_view(self, frame_number): - return (frame_number - (frame_number % (self._channel_count + self._z_level_count))) % self._field_of_view_count + return (frame_number - (frame_number % (len(self.channels) + len(self.z_levels))) % len(self.fields_of_view) def _calculate_channel(self, frame_number): - pass + return self._channels[frame_number % self._channel_count] def _calculate_z_level(self, frame_number): - pass + return self._z_levels[] @property def image_sets(self): @@ -119,9 +119,9 @@ class Nd2(Nd2Parser): :rtype: nd2reader.model.Image() or None """ - image_set_number = self._calculate_image_group_number(time_index, field_of_view, z_level) + image_group_number = self._calculate_image_group_number(time_index, field_of_view, z_level) try: - timestamp, raw_image_data = self._get_raw_image_data(image_set_number, self._channel_offset[channel_name]) + timestamp, raw_image_data = self._get_raw_image_data(image_group_number, self._channel_offset[channel_name]) image = Image(timestamp, raw_image_data, field_of_view, channel_name, z_level, self.height, self.width) except TypeError: return None diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 58914a3..2758879 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -21,13 +21,18 @@ class Nd2Parser(object): def __init__(self, filename): self._filename = filename self._fh = None + self._channels = None + self._channel_count = None self._chunk_map_start_location = None self._cursor_position = 0 self._dimension_text = None + self._fields_of_view = None self._label_map = {} self.metadata = {} self._read_map() + self._time_indexes = None self._parse_metadata() + self._z_levels = None @property def _file_handle(self): @@ -93,7 +98,7 @@ class Nd2Parser(object): return self._dimension_text @property - def _channels(self): + def channels(self): """ These are labels created by the NIS Elements user. Typically they may a short description of the filter cube used (e.g. "bright field", "GFP", etc.) @@ -101,19 +106,21 @@ class Nd2Parser(object): :rtype: str """ - metadata = self.metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] - try: - validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] - except KeyError: - # If none of the channels have been deleted, there is no validity list, so we just make one - validity = [True for _ in metadata] - # Channel information is contained in dictionaries with the keys a0, a1...an where the number - # indicates the order in which the channel is stored. So by sorting the dicts alphabetically - # we get the correct order. - for (label, chan), valid in zip(sorted(metadata['sPlaneNew'].items()), validity): - if not valid: - continue - yield chan['sDescription'] + if not self._channels: + metadata = self.metadata['ImageMetadataSeq']['SLxPictureMetadata']['sPicturePlanes'] + try: + validity = self.metadata['ImageMetadata']['SLxExperiment']['ppNextLevelEx'][''][0]['ppNextLevelEx'][''][0]['pItemValid'] + except KeyError: + # If none of the channels have been deleted, there is no validity list, so we just make one + validity = [True for _ in metadata] + # Channel information is contained in dictionaries with the keys a0, a1...an where the number + # indicates the order in which the channel is stored. So by sorting the dicts alphabetically + # we get the correct order. + for (label, chan), valid in zip(sorted(metadata['sPlaneNew'].items()), validity): + if not valid: + continue + self._channels.append(chan['sDescription']) + return self._channels def _calculate_image_group_number(self, time_index, fov, z_level): """ @@ -126,7 +133,7 @@ class Nd2Parser(object): :rtype: int """ - return time_index * self._field_of_view_count * self._z_level_count + (fov * self._z_level_count + z_level) + return time_index * len(self.fields_of_view) * len(self.z_levels) + (fov * len(self.z_levels) + z_level) @property def _channel_offset(self): @@ -167,24 +174,28 @@ class Nd2Parser(object): return absolute_start_12 if absolute_start_12 else absolute_start_24 raise ValueError("This ND2 has no recorded start time. This is probably a bug.") + def _parse_dimension_text(self, pattern): + try: + count = int(re.match(pattern, self._dimensions).group(1)) + except AttributeError: + return [0] + else: + return range(count) + @property - def _channel_count(self): + def channel_count(self): """ The number of different channels used, including bright field. :rtype: int """ - pattern = r""".*?λ\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count + if self._channel_count is None: + self._channel_count = self._parse_dimension_text(r""".*?λ\((\d+)\).*?""") + return self._channel_count @property - def _field_of_view_count(self): + def fields_of_view(self): """ The metadata contains information about fields of view, but it contains it even if some fields of view were cropped. We can't find anything that states which fields of view are actually @@ -194,45 +205,33 @@ class Nd2Parser(object): :rtype: int """ - pattern = r""".*?XY\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count + if self._fields_of_view is None: + self._fields_of_view = self._parse_dimension_text(r""".*?XY\((\d+)\).*?""") + return self._fields_of_view @property - def _time_index_count(self): + def time_indexes(self): """ The number of cycles. :rtype: int """ - pattern = r""".*?T'\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count + if self._time_indexes is None: + self._time_indexes = self._parse_dimension_text(r""".*?T'\((\d+)\).*?""") + return self._time_indexes @property - def _z_level_count(self): + def z_levels(self): """ - The number of different levels in the Z-plane. + The different levels in the Z-plane. Just a sequence from 0 to n. :rtype: int """ - pattern = r""".*?Z\((\d+)\).*?""" - try: - count = int(re.match(pattern, self._dimensions).group(1)) - except AttributeError: - return 1 - else: - return count + if self._z_levels is None: + self._z_levels = self._parse_dimension_text(r""".*?Z\((\d+)\).*?""") + return self._z_levels @property def _image_count(self): From c9d3a82930e2a3c9a8ae2f76dddf6facf470ed28 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 02:31:11 +0000 Subject: [PATCH 3/7] #39: getting individual images works, however, z_level and field of view are wrong --- nd2reader/__init__.py | 28 ++++++++++++++-------------- nd2reader/parser.py | 19 ++++--------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index babff0c..6905c98 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -18,10 +18,10 @@ class Nd2(Nd2Parser): return "\n".join(["" % self._filename, "Created: %s" % self._absolute_start.strftime("%Y-%m-%d %H:%M:%S"), "Image size: %sx%s (HxW)" % (self.height, self.width), - "Image cycles: %s" % self._time_index_count, - "Channels: %s" % ", ".join(["'%s'" % str(channel) for channel in self._channels]), - "Fields of View: %s" % self._field_of_view_count, - "Z-Levels: %s" % self._z_level_count + "Image cycles: %s" % len(self.time_indexes), + "Channels: %s" % ", ".join(["'%s'" % str(channel) for channel in self.channels]), + "Fields of View: %s" % len(self.fields_of_view), + "Z-Levels: %s" % len(self.z_levels) ]) def __len__(self): @@ -60,8 +60,8 @@ class Nd2(Nd2Parser): """ for i in range(self._image_count): - for fov in range(self._field_of_view_count): - for z_level in range(self._z_level_count): + for fov in self._fields_of_view: + for z_level in self._z_levels: for channel_name in self._channels: image = self.get_image(i, fov, channel_name, z_level) if image is not None: @@ -70,12 +70,12 @@ class Nd2(Nd2Parser): def __getitem__(self, item): if isinstance(item, int): try: - channel_offset = item % self._channel_count + channel_offset = item % len(self.channels) fov = self._calculate_field_of_view(item) channel = self._calculate_channel(item) z_level = self._calculate_z_level(item) item -= channel_offset - item /= self._channel_count + item /= len(self.channels) timestamp, raw_image_data = self._get_raw_image_data(item, channel_offset) image = Image(timestamp, raw_image_data, fov, channel, z_level, self.height, self.width) except TypeError: @@ -84,13 +84,13 @@ class Nd2(Nd2Parser): return image def _calculate_field_of_view(self, frame_number): - return (frame_number - (frame_number % (len(self.channels) + len(self.z_levels))) % len(self.fields_of_view) + return (frame_number - (frame_number % (len(self.channels) + len(self.z_levels)))) % len(self.fields_of_view) def _calculate_channel(self, frame_number): - return self._channels[frame_number % self._channel_count] + return self._channels[frame_number % len(self.channels)] def _calculate_z_level(self, frame_number): - return self._z_levels[] + return self.z_levels[frame_number % len(self.channels) % len(self.fields_of_view)] @property def image_sets(self): @@ -103,11 +103,11 @@ class Nd2(Nd2Parser): :return: model.ImageSet() """ - for time_index in range(self._time_index_count): + for time_index in self._time_indexes: image_set = ImageSet() - for fov in range(self._field_of_view_count): + for fov in self._fields_of_view: for channel_name in self._channels: - for z_level in range(self._z_level_count): + for z_level in self._z_levels: image = self.get_image(time_index, fov, channel_name, z_level) if image is not None: image_set.add(image) diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 9ec7277..9566f21 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -95,8 +95,6 @@ class Nd2Parser(object): break else: raise ValueError("Could not parse metadata dimensions!") - if six.PY3: - return self._dimension_text.decode("utf8") return self._dimension_text @property @@ -183,20 +181,11 @@ class Nd2Parser(object): count = int(re.match(pattern, self._dimensions).group(1)) except AttributeError: return [0] + except TypeError: + count = int(re.match(pattern, self._dimensions.decode("utf8")).group(1)) + return list(range(count)) else: - return range(count) - - @property - def channel_count(self): - """ - The number of different channels used, including bright field. - - :rtype: int - - """ - if self._channel_count is None: - self._channel_count = self._parse_dimension_text(r""".*?λ\((\d+)\).*?""") - return self._channel_count + return list(range(count)) @property def fields_of_view(self): From e8cb08f4ccec268a8a9c5212905986db881d6a07 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 03:14:05 +0000 Subject: [PATCH 4/7] #39: calculating channel and z_level works right --- nd2reader/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 6905c98..47ccdaa 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -84,13 +84,13 @@ class Nd2(Nd2Parser): return image def _calculate_field_of_view(self, frame_number): - return (frame_number - (frame_number % (len(self.channels) + len(self.z_levels)))) % len(self.fields_of_view) + return frame_number - (frame_number % (len(self.z_levels) + len(self.channels))) def _calculate_channel(self, frame_number): return self._channels[frame_number % len(self.channels)] def _calculate_z_level(self, frame_number): - return self.z_levels[frame_number % len(self.channels) % len(self.fields_of_view)] + return self.z_levels[(frame_number * len(self.channels)) % len(self.z_levels) - frame_number % len(self.channels)] @property def image_sets(self): From 0f9c6128e3af7cfbb738c27e745d049bea15db0f Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 04:00:11 +0000 Subject: [PATCH 5/7] #39: field of view works --- nd2reader/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 47ccdaa..0f40628 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -84,13 +84,13 @@ class Nd2(Nd2Parser): return image def _calculate_field_of_view(self, frame_number): - return frame_number - (frame_number % (len(self.z_levels) + len(self.channels))) + return int((frame_number - (frame_number % (len(self.z_levels) * len(self.channels)))) / (len(self.z_levels) * len(self.channels))) def _calculate_channel(self, frame_number): return self._channels[frame_number % len(self.channels)] def _calculate_z_level(self, frame_number): - return self.z_levels[(frame_number * len(self.channels)) % len(self.z_levels) - frame_number % len(self.channels)] + return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / 2) % len(self.z_levels))] @property def image_sets(self): From 89ddce3a7bf3acc049eed2155f91723d7a4309a4 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 04:43:06 +0000 Subject: [PATCH 6/7] resolves #39 --- nd2reader/__init__.py | 90 ++++++++++++++++++------------------------- nd2reader/parser.py | 2 +- setup.py | 2 +- 3 files changed, 39 insertions(+), 55 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 0f40628..5166932 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -34,64 +34,20 @@ class Nd2(Nd2Parser): """ return self._image_count * self._channel_count - @property - def height(self): - """ - :return: height of each image, in pixels - :rtype: int - - """ - return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiHeight')] - - @property - def width(self): - """ - :return: width of each image, in pixels - :rtype: int - - """ - return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiWidth')] - - def __iter__(self): - """ - Iterates over every image, in the order they were taken. - - :return: model.Image() - - """ - for i in range(self._image_count): - for fov in self._fields_of_view: - for z_level in self._z_levels: - for channel_name in self._channels: - image = self.get_image(i, fov, channel_name, z_level) - if image is not None: - yield image - - def __getitem__(self, item): - if isinstance(item, int): + def __getitem__(self, frame_number): + if isinstance(frame_number, int): try: - channel_offset = item % len(self.channels) - fov = self._calculate_field_of_view(item) - channel = self._calculate_channel(item) - z_level = self._calculate_z_level(item) - item -= channel_offset - item /= len(self.channels) - timestamp, raw_image_data = self._get_raw_image_data(item, channel_offset) + channel_offset = frame_number % len(self.channels) + fov = self._calculate_field_of_view(frame_number) + channel = self._calculate_channel(frame_number) + z_level = self._calculate_z_level(frame_number) + timestamp, raw_image_data = self._get_raw_image_data(frame_number, channel_offset) image = Image(timestamp, raw_image_data, fov, channel, z_level, self.height, self.width) - except TypeError: + except (TypeError, ValueError): return None else: return image - def _calculate_field_of_view(self, frame_number): - return int((frame_number - (frame_number % (len(self.z_levels) * len(self.channels)))) / (len(self.z_levels) * len(self.channels))) - - def _calculate_channel(self, frame_number): - return self._channels[frame_number % len(self.channels)] - - def _calculate_z_level(self, frame_number): - return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / 2) % len(self.z_levels))] - @property def image_sets(self): """ @@ -103,7 +59,7 @@ class Nd2(Nd2Parser): :return: model.ImageSet() """ - for time_index in self._time_indexes: + for time_index in self.time_indexes: image_set = ImageSet() for fov in self._fields_of_view: for channel_name in self._channels: @@ -113,6 +69,34 @@ class Nd2(Nd2Parser): image_set.add(image) yield image_set + @property + def height(self): + """ + :return: height of each image, in pixels + :rtype: int + + """ + return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiHeight')] + + @property + def width(self): + """ + :return: width of each image, in pixels + :rtype: int + + """ + return self.metadata[six.b('ImageAttributes')][six.b('SLxImageAttributes')][six.b('uiWidth')] + + def _calculate_field_of_view(self, frame_number): + images_per_cycle = len(self.z_levels) * len(self.channels) + return int((frame_number - (frame_number % images_per_cycle)) / images_per_cycle) % len(self.fields_of_view) + + def _calculate_channel(self, frame_number): + return self._channels[frame_number % len(self.channels)] + + def _calculate_z_level(self, frame_number): + return self.z_levels[int(((frame_number - (frame_number % len(self.channels))) / len(self.channels)) % len(self.z_levels))] + def get_image(self, time_index, field_of_view, channel_name, z_level): """ Returns an Image if data exists for the given parameters, otherwise returns None. In general, you should avoid diff --git a/nd2reader/parser.py b/nd2reader/parser.py index 9566f21..5aabcaa 100644 --- a/nd2reader/parser.py +++ b/nd2reader/parser.py @@ -63,7 +63,7 @@ class Nd2Parser(object): # The images for the various channels are interleaved within the same array. For example, the second image # of a four image group will be composed of bytes 2, 6, 10, etc. If you understand why someone would design # a data structure that way, please send the author of this library a message. - image_data = image_group_data[image_data_start::self._channel_count] + image_data = image_group_data[image_data_start::len(self.channels)] # Skip images that are all zeros! This is important, since NIS Elements creates blank "gap" images if you # don't have the same number of images each cycle. We discovered this because we only took GFP images every # other cycle to reduce phototoxicity, but NIS Elements still allocated memory as if we were going to take diff --git a/setup.py b/setup.py index d250eec..ee88e1d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from distutils.core import setup -VERSION = "1.0.1" +VERSION = "1.1.0" setup( name="nd2reader", From 454f5529fee06f592f388610c3864605be9d8150 Mon Sep 17 00:00:00 2001 From: Jim Rybarski Date: Sun, 31 May 2015 04:45:02 +0000 Subject: [PATCH 7/7] #39 fixed image sets --- nd2reader/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nd2reader/__init__.py b/nd2reader/__init__.py index 5166932..754c6ef 100644 --- a/nd2reader/__init__.py +++ b/nd2reader/__init__.py @@ -61,9 +61,9 @@ class Nd2(Nd2Parser): """ for time_index in self.time_indexes: image_set = ImageSet() - for fov in self._fields_of_view: - for channel_name in self._channels: - for z_level in self._z_levels: + for fov in self.fields_of_view: + for channel_name in self.channels: + for z_level in self.z_levels: image = self.get_image(time_index, fov, channel_name, z_level) if image is not None: image_set.add(image)