zolfa
/
zolfa-nd2reader
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

Complete refactor 

- use only one parser for v3 files
- merge some classes
- return metadata as raw metadata and dictionary
feature/load_slices
Ruben Verweij 7 years ago
parent
6f151e8701
commit
cb2c06f9a5
21 changed files with 441 additions and 983 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +70
    -0
      nd2reader/common.py
  2. +0
    -0
      nd2reader/common/__init__.py
  3. +0
    -0
      nd2reader/driver/__init__.py
  4. +0
    -0
      nd2reader/label_map.py
  5. +0
    -236
      nd2reader/legacy.py
  6. +0
    -1
      nd2reader/model/__init__.py
  7. +0
    -135
      nd2reader/model/image.py
  8. +0
    -106
      nd2reader/model/metadata.py
  9. +0
    -81
      nd2reader/model/roi.py
  10. +120
    -38
      nd2reader/nd2parser.py
  11. +35
    -48
      nd2reader/nd2reader.py
  12. +0
    -1
      nd2reader/parser/__init__.py
  13. +0
    -17
      nd2reader/parser/base.py
  14. +0
    -55
      nd2reader/parser/parser.py
  15. +211
    -219
      nd2reader/raw_metadata.py
  16. +1
    -1
      test.py
  17. +0
    -0
      tests/driver/__init__.py
  18. +0
    -0
      tests/model/__init__.py
  19. +0
    -42
      tests/model/image.py
  20. +3
    -2
      tests/test_common.py
  21. +1
    -1
      tests/test_version.py

nd2reader/common/v3.py → nd2reader/common.py View File


+ 0
- 0
nd2reader/common/__init__.py View File


+ 0
- 0
nd2reader/driver/__init__.py View File


nd2reader/model/label.py → nd2reader/label_map.py View File


+ 0
- 236
nd2reader/legacy.py View File

@ -1,236 +0,0 @@
# -*- coding: utf-8 -*-
from nd2reader.parser import get_parser, get_version
import six
class Nd2(object):
""" Allows easy access to NIS Elements .nd2 image files. """
def __init__(self, filename):
self._filename = filename
self._fh = open(filename, "rb")
major_version, minor_version = get_version(self._fh)
self._parser = get_parser(self._fh, major_version, minor_version)
self._metadata = self._parser.metadata
self._roi_metadata = self._parser.roi_metadata
def __repr__(self):
return "\n".join(["<ND2 %s>" % self._filename,
"Created: %s" % (self.date if self.date is not None else "Unknown"),
"Image size: %sx%s (HxW)" % (self.height, self.width),
"Frames: %s" % len(self.frames),
"Channels: %s" % ", ".join(["%s" % str(channel) for channel in self.channels]),
"Fields of View: %s" % len(self.fields_of_view),
"Z-Levels: %s" % len(self.z_levels)
])
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self._fh is not None:
self._fh.close()
def __len__(self):
"""
This should be the total number of images in the ND2, but it may be inaccurate. If the ND2 contains a
different number of images in a cycle (i.e. there are "gap" images) it will be higher than reality.
:rtype: int
"""
return self._metadata.total_images_per_channel * len(self.channels)
def __getitem__(self, item):
"""
Allows slicing ND2s.
:type item: int or slice
:rtype: nd2reader.model.Image() or generator
"""
if isinstance(item, int):
try:
image = self._parser.driver.get_image(item)
except KeyError:
raise IndexError
else:
return image
elif isinstance(item, slice):
return self._slice(item.start, item.stop, item.step)
raise IndexError
def select(self, fields_of_view=None, channels=None, z_levels=None, start=0, stop=None):
"""
Iterates over images matching the given criteria. This can be 2-10 times faster than manually iterating over
the Nd2 and checking the attributes of each image, as this method skips disk reads for any images that don't
meet the criteria.
:type fields_of_view: int or tuple or list
:type channels: str or tuple or list
:type z_levels: int or tuple or list
:type start: int
:type stop: int
"""
fields_of_view = self._to_tuple(fields_of_view, self.fields_of_view)
channels = self._to_tuple(channels, self.channels)
z_levels = self._to_tuple(z_levels, self.z_levels)
# By default, we stop after the last image. Otherwise we make sure the user-provided value is valid
stop = len(self) if stop is None else max(0, min(stop, len(self)))
for frame in range(start, stop):
field_of_view, channel, z_level = self._parser.driver.calculate_image_properties(frame)
if field_of_view in fields_of_view and channel in channels and z_level in z_levels:
image = self._parser.driver.get_image(frame)
if image is not None:
yield image
@property
def height(self):
"""
The height of each image in pixels.
:rtype: int
"""
return self._metadata.height
@property
def width(self):
"""
The width of each image in pixels.
:rtype: int
"""
return self._metadata.width
@property
def z_levels(self):
"""
A list of integers that represent the different levels on the Z-axis that images were taken. Currently this is
just a list of numbers from 0 to N. For example, an ND2 where images were taken at -3µm, 0µm, and +5µm from a
set position would be represented by 0, 1 and 2, respectively. ND2s do store the actual offset of each image
in micrometers and in the future this will hopefully be available. For now, however, you will have to match up
the order yourself.
:return: list of int
"""
return self._metadata.z_levels
@property
def fields_of_view(self):
"""
A list of integers representing the various stage locations, in the order they were taken in the first round
of acquisition.
:return: list of int
"""
return self._metadata.fields_of_view
@property
def channels(self):
"""
A list of channel (i.e. wavelength) names. These are set by the user in NIS Elements.
:return: list of str
"""
return self._metadata.channels
@property
def frames(self):
"""
A list of integers representing groups of images. ND2s consider images to be part of the same frame if they
are in the same field of view and don't have the same channel. So if you take a bright field and GFP image at
four different fields of view over and over again, you'll have 8 images and 4 frames per cycle.
:return: list of int
"""
return self._metadata.frames
@property
def date(self):
"""
The date and time that the acquisition began. Not guaranteed to have been recorded.
:rtype: datetime.datetime() or None
"""
return self._metadata.date
@property
def pixel_microns(self):
"""
The width of a pixel in microns. Note that the user can override this in NIS Elements so it may not reflect reality.
:rtype: float
"""
return self._metadata.pixel_microns
def get_image(self, frame_number, field_of_view, channel_name, z_level):
"""
Attempts to return the image with the unique combination of given attributes. None will be returned if a match
is not found.
:type frame_number: int
:param field_of_view: the label for the place in the XY-plane where this image was taken.
:type field_of_view: int
:param channel_name: the name of the color of this image
:type channel_name: str
:param z_level: the label for the location in the Z-plane where this image was taken.
:type z_level: int
:rtype: nd2reader.model.Image() or None
"""
return self._parser.driver.get_image_by_attributes(frame_number,
field_of_view,
channel_name,
z_level,
self.height,
self.width)
def close(self):
"""
Closes the file handle to the image. This actually sometimes will prevent problems so it's good to do this or
use Nd2 as a context manager.
"""
self._fh.close()
def _slice(self, start, stop, step):
"""
Allows for iteration over a selection of the entire dataset.
:type start: int
:type stop: int
:type step: int
:rtype: nd2reader.model.Image()
"""
start = start if start is not None else 0
step = step if step is not None else 1
stop = stop if stop is not None else len(self)
# This weird thing with the step allows you to iterate backwards over the images
for i in range(start, stop)[::step]:
yield self[i]
def _to_tuple(self, value, default):
"""
Idempotently converts a value to a tuple. This allows users to pass in scalar values and iterables to
select(), which is more ergonomic than having to remember to pass in single-member lists
:type value: int or str or tuple or list
:type default: tuple or list
:rtype: tuple
"""
value = default if value is None else value
return (value,) if isinstance(value, int) or isinstance(value, six.string_types) else tuple(value)

+ 0
- 1
nd2reader/model/__init__.py View File

@ -1 +0,0 @@
from nd2reader.model.image import Image

+ 0
- 135
nd2reader/model/image.py View File

@ -1,135 +0,0 @@
# -*- coding: utf-8 -*-
import numpy as np
class Image(np.ndarray):
"""
Holds the raw pixel data of an image and provides access to some metadata.
"""
def __new__(cls, array):
return np.asarray(array).view(cls)
def __init__(self, array):
self._index = None
self._timestamp = None
self._frame_number = None
self._field_of_view = None
self._channel = None
self._z_level = None
def __array_wrap__(self, obj, *args):
if len(obj.shape) == 0:
return obj[()]
else:
return obj
def add_params(self, index, timestamp, frame_number, field_of_view, channel, z_level):
"""
:param index: The integer that can be used to directly index this image
:type index: int
:param timestamp: The number of milliseconds after the beginning of the acquisition that this image was taken.
:type timestamp: float
:param frame_number: The order in which this image was taken, with images of different channels/z-levels
at the same field of view treated as being in the same frame.
:type frame_number: int
:param field_of_view: The label for the place in the XY-plane where this image was taken.
:type field_of_view: int
:param channel: The name of the color of this image
:type channel: str
:param z_level: The label for the location in the Z-plane where this image was taken.
:type z_level: int
"""
self._index = index
self._timestamp = timestamp
self._frame_number = int(frame_number)
self._field_of_view = field_of_view
self._channel = channel
self._z_level = z_level
@property
def index(self):
return self._index
@property
def height(self):
"""
The height in pixels.
:rtype: int
"""
return self.shape[0]
@property
def width(self):
"""
The width in pixels.
:rtype: int
"""
return self.shape[1]
@property
def field_of_view(self):
"""
The index of the stage location where this image was acquired.
:rtype: int
"""
return self._field_of_view
@property
def timestamp(self):
"""
The number of seconds after the beginning of the acquisition that the image was taken. Note that for a given
field of view and z-level offset, if you have images of multiple channels, they will all be given the same
timestamp. That's just how ND2s are structured, so if your experiment depends on millisecond accuracy,
you need to find an alternative imaging system.
:rtype: float
"""
# data is actually stored in milliseconds
return self._timestamp / 1000.0
@property
def frame_number(self):
"""
The index of the group of images taken sequentially that all have the same group number and field of view.
:rtype: int
"""
return self._frame_number
@property
def channel(self):
"""
The name of the filter used to acquire this image. These are user-supplied in NIS Elements.
:rtype: str
"""
return self._channel
@property
def z_level(self):
"""
The vertical offset of the image. These are simple integers starting from 0, where the 0 is the lowest
z-level and each subsequent level incremented by 1.
For example, if you acquired images at -3 µm, 0 µm, and +3 µm, your z-levels would be:
-3 µm: 0
0 µm: 1
+3 µm: 2
:rtype: int
"""
return self._z_level

+ 0
- 106
nd2reader/model/metadata.py View File

@ -1,106 +0,0 @@
class Metadata(object):
""" A simple container for ND2 metadata. """
def __init__(self, height, width, channels, date, fields_of_view, frames, z_levels, total_images_per_channel, pixel_microns):
self._height = height
self._width = width
self._channels = channels
self._date = date
self._fields_of_view = fields_of_view
self._frames = frames
self._z_levels = z_levels
self._total_images_per_channel = total_images_per_channel
self._pixel_microns = pixel_microns
@property
def height(self):
"""
The image height in pixels.
:rtype: int
"""
return self._height
@property
def width(self):
"""
The image width in pixels.
:rtype: int
"""
return self._width
@property
def date(self):
"""
The date and time when acquisition began.
:rtype: datetime.datetime() or None
"""
return self._date
@property
def channels(self):
"""
These are labels created by the NIS Elements user. Typically they may a short description of the filter cube
used (e.g. "bright field", "GFP", etc.)
:rtype: list
"""
return self._channels
@property
def fields_of_view(self):
"""
The metadata contains information about fields of view, but it contains it even if some fields
of view were cropped. We can't find anything that states which fields of view are actually
in the image data, so we have to calculate it. There probably is something somewhere, since
NIS Elements can figure it out, but we haven't found it yet.
:rtype: list
"""
return self._fields_of_view
@property
def frames(self):
"""
The number of cycles.
:rtype: list
"""
return self._frames
@property
def z_levels(self):
"""
The different levels in the Z-plane. Just a sequence from 0 to n.
:rtype: list
"""
return self._z_levels
@property
def total_images_per_channel(self):
"""
The total number of images of a particular channel (wavelength, filter, etc) in the entire ND2.
:rtype: int
"""
return self._total_images_per_channel
@property
def pixel_microns(self):
"""
The width of a pixel in microns.
:rtype: float
"""
return self._pixel_microns

+ 0
- 81
nd2reader/model/roi.py View File

@ -1,81 +0,0 @@
import six
import numpy as np
class Roi(object):
"""
A ND2 ROI representation.
Coordinates are the center coordinates of the ROI in (x, y, z) order in micron.
Sizes are the sizes of the ROI in (x, y, z) order in micron.
Shapes are represented by numbers, defined by constants in this class.
All these properties can be set for multiple time points (in ms).
"""
SHAPE_RECTANGLE = 3
SHAPE_CIRCLE = 9
TYPE_BACKGROUND = 2
def __init__(self, raw_roi_dict, metadata):
"""
:param raw_roi_dict:
:param metadata
"""
self.timepoints = []
self.positions = []
self.sizes = []
self.shape = self.SHAPE_CIRCLE
self.type = self.TYPE_BACKGROUND
self._img_width_micron = metadata.width * metadata.pixel_microns
self._img_height_micron = metadata.height * metadata.pixel_microns
self._pixel_microns = metadata.pixel_microns
self._extract_vect_anims(raw_roi_dict)
def _extract_vect_anims(self, raw_roi_dict):
"""
Extract the vector animation parameters from the ROI.
This includes the position and size at the given timepoints.
:param raw_roi_dict:
:return:
"""
number_of_timepoints = raw_roi_dict[six.b('m_vectAnimParams_Size')]
for i in range(number_of_timepoints):
self._parse_vect_anim(raw_roi_dict[six.b('m_vectAnimParams_%d') % i])
self.shape = raw_roi_dict[six.b('m_sInfo')][six.b('m_uiShapeType')]
self.type = raw_roi_dict[six.b('m_sInfo')][six.b('m_uiInterpType')]
# convert to NumPy arrays
self.timepoints = np.array(self.timepoints, dtype=np.float)
self.positions = np.array(self.positions, dtype=np.float)
self.sizes = np.array(self.sizes, dtype=np.float)
def _parse_vect_anim(self, animation_dict):
"""
Parses a ROI vector animation object and adds it to the global list of timepoints and positions.
:param animation_dict:
:return:
"""
self.timepoints.append(animation_dict[six.b('m_dTimeMs')])
# positions are taken from the center of the image as a fraction of the half width/height of the image
position = np.array((0.5 * self._img_width_micron * (1 + animation_dict[six.b('m_dCenterX')]),
0.5 * self._img_height_micron * (1 + animation_dict[six.b('m_dCenterY')]),
animation_dict[six.b('m_dCenterZ')]))
self.positions.append(position)
size_dict = animation_dict[six.b('m_sBoxShape')]
# sizes are fractions of the half width/height of the image
self.sizes.append((size_dict[six.b('m_dSizeX')] * 0.25 * self._img_width_micron,
size_dict[six.b('m_dSizeY')] * 0.25 * self._img_height_micron,
size_dict[six.b('m_dSizeZ')]))
def is_circle(self):
return self.shape == self.SHAPE_CIRCLE
def is_rectangle(self):
return self.shape == self.SHAPE_RECTANGLE

nd2reader/driver/v3.py → nd2reader/nd2parser.py View File


+ 35
- 48
nd2reader/nd2reader.py View File

@ -1,14 +1,11 @@
from pims import FramesSequenceND, Frame
import numpy as np
from nd2reader.exc import NoImageError
from nd2reader.parser import get_parser
import six
from nd2reader.nd2parser import ND2Parser
class ND2Reader(FramesSequenceND):
"""
PIMS wrapper for the ND2 reader
PIMS wrapper for the ND2 parser
"""
def __init__(self, filename):
@ -16,43 +13,30 @@ class ND2Reader(FramesSequenceND):
# first use the parser to parse the file
self._fh = open(filename, "rb")
self._parser = get_parser(self._fh)
self._metadata = self._parser.metadata
self._roi_metadata = self._parser.roi_metadata
self._parser = ND2Parser(self._fh)
# Setup metadata
self.metadata = self._parser.metadata
# Set data type
self._dtype = self._get_dtype_from_metadata()
self._dtype = self._parser.get_dtype_from_metadata()
# Setup the axes
self._init_axis('x', self._metadata.width)
self._init_axis('y', self._metadata.height)
self._init_axis('c', len(self._metadata.channels))
self._init_axis('t', len(self._metadata.frames))
self._init_axis('z', len(self._metadata.z_levels))
self._setup_axes()
# provide the default
self.iter_axes = 't'
def _get_dtype_from_metadata(self):
@classmethod
def class_exts(cls):
"""
Determine the data type from the metadata.
So PIMS open function can use this reader for opening .nd2 files
:return:
"""
bit_depth = self._parser.raw_metadata.image_attributes[six.b('SLxImageAttributes')][six.b('uiBpcInMemory')]
if bit_depth <= 16:
self._dtype = np.float16
elif bit_depth <= 32:
self._dtype = np.float32
else:
self._dtype = np.float64
return self._dtype
@classmethod
def class_exts(cls):
return {'nd2'} | super(ND2Reader, cls).class_exts()
def close(self):
"""
Correctly close the file handle
:return:
"""
if self._fh is not None:
self._fh.close()
@ -64,30 +48,33 @@ class ND2Reader(FramesSequenceND):
:param z:
:return:
"""
c_name = self._metadata.channels[c]
c_name = self.metadata["channels"][c]
try:
image = self._parser.driver.get_image_by_attributes(t, 0, c_name, z, self._metadata.width,
self._metadata.height)
image = self._parser.get_image_by_attributes(t, 0, c_name, z, self.metadata["width"],
self.metadata["height"])
except (TypeError, NoImageError):
return Frame([])
else:
return Frame(image, frame_no=image.frame_number, metadata=self._get_frame_metadata())
return image
def _get_frame_metadata(self):
@property
def pixel_type(self):
"""
Get the metadata for one frame
Return the pixel data type
:return:
"""
frame_metadata = {
"height": self._metadata.height,
"width": self._metadata.width,
"date": self._metadata.date,
"pixel_microns": self._metadata.pixel_microns,
"rois": self._roi_metadata
}
return self._dtype
return frame_metadata
def _setup_axes(self):
"""
Setup the xyctz axes, iterate over t axis by default
:return:
"""
self._init_axis('x', self.metadata["width"])
self._init_axis('y', self.metadata["height"])
self._init_axis('c', len(self.metadata["channels"]))
self._init_axis('t', len(self.metadata["frames"]))
self._init_axis('z', len(self.metadata["z_levels"]))
@property
def pixel_type(self):
return self._dtype
# provide the default
self.iter_axes = 't'

+ 0
- 1
nd2reader/parser/__init__.py View File

@ -1 +0,0 @@
from . parser import get_parser, get_version, parse_version

+ 0
- 17
nd2reader/parser/base.py View File

@ -1,17 +0,0 @@
from abc import abstractproperty
class BaseParser(object):
def __init__(self, fh):
self._fh = fh
self.camera_metadata = None
self.metadata = None
self.roi_metadata = None
@abstractproperty
def driver(self):
"""
Must return an object that can look up and read images.
"""
raise NotImplementedError

+ 0
- 55
nd2reader/parser/parser.py View File

@ -1,55 +0,0 @@
from nd2reader.parser.v3 import V3Parser
import re
from nd2reader.exc import InvalidVersionError
def get_parser(fh):
"""
Picks the appropriate parser based on the ND2 version.
:type fh: file
:type major_version: int
:type minor_version: int
:rtype: a parser object
"""
major_version, minor_version = get_version(fh)
parsers = {(3, None): V3Parser}
parser = parsers.get((major_version, minor_version)) or parsers.get((major_version, None))
if not parser:
raise InvalidVersionError("No parser is available for that version.")
return parser(fh)
def get_version(fh):
"""
Determines what version the ND2 is.
:param fh: an open file handle to the ND2
:type fh: file
"""
# the first 16 bytes seem to have no meaning, so we skip them
fh.seek(16)
# the next 38 bytes contain the string that we want to parse. Unlike most of the ND2, this is in UTF-8
data = fh.read(38).decode("utf8")
return parse_version(data)
def parse_version(data):
"""
Parses a string with the version data in it.
:param data: the 19th through 54th byte of the ND2, representing the version
:type data: unicode
"""
match = re.search(r"""^ND2 FILE SIGNATURE CHUNK NAME01!Ver(?P<major>\d)\.(?P<minor>\d)$""", data)
if match:
# We haven't seen a lot of ND2s but the ones we have seen conform to this
return int(match.group('major')), int(match.group('minor'))
raise InvalidVersionError("The version of the ND2 you specified is not supported.")

nd2reader/parser/v3.py → nd2reader/raw_metadata.py View File


+ 1
- 1
test.py View File

@ -1,5 +1,5 @@
import unittest
loader = unittest.TestLoader()
tests = loader.discover('tests', pattern='*.py', top_level_dir='.')
tests = loader.discover('tests', pattern='test_*.py', top_level_dir='.')
testRunner = unittest.TextTestRunner()
testRunner.run(tests)

+ 0
- 0
tests/driver/__init__.py View File


+ 0
- 0
tests/model/__init__.py View File


+ 0
- 42
tests/model/image.py View File

@ -1,42 +0,0 @@
from nd2reader.model.image import Image
import numpy as np
import unittest
class ImageTests(unittest.TestCase):
"""
Basically just tests that the Image API works and that Images act as Numpy arrays. There's very little going on
here other than simply storing data.
"""
def setUp(self):
array = np.array([[0, 1, 254],
[45, 12, 9],
[12, 12, 99]])
self.image = Image(array)
self.image.add_params(1, 1200.314, 17, 2, 'GFP', 1)
def test_size(self):
self.assertEqual(self.image.height, 3)
self.assertEqual(self.image.width, 3)
def test_timestamp(self):
self.assertEqual(self.image.timestamp, 1.200314)
def test_frame_number(self):
self.assertEqual(self.image.frame_number, 17)
def test_fov(self):
self.assertEqual(self.image.field_of_view, 2)
def test_channel(self):
self.assertEqual(self.image.channel, 'GFP')
def test_z_level(self):
self.assertEqual(self.image.z_level, 1)
def test_slice(self):
subimage = self.image[:2, :2]
expected = np.array([[0, 1],
[45, 12]])
self.assertTrue(np.array_equal(subimage, expected))

tests/driver/version.py → tests/test_common.py View File


tests/version.py → tests/test_version.py View File


Write Preview
Loading…
Cancel
Save