You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

356 lines
9.0 KiB

8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
  1. import os
  2. import struct
  3. import array
  4. from datetime import datetime
  5. import six
  6. import re
  7. from nd2reader.exceptions import InvalidVersionError
  8. def get_version(fh):
  9. """Determines what version the ND2 is.
  10. Args:
  11. fh: File handle of the .nd2 file
  12. Returns:
  13. tuple: Major and minor version
  14. """
  15. # the first 16 bytes seem to have no meaning, so we skip them
  16. fh.seek(16)
  17. # the next 38 bytes contain the string that we want to parse. Unlike most of the ND2, this is in UTF-8
  18. data = fh.read(38).decode("utf8")
  19. return parse_version(data)
  20. def parse_version(data):
  21. """Parses a string with the version data in it.
  22. Args:
  23. data (unicode): the 19th through 54th byte of the ND2, representing the version
  24. Returns:
  25. tuple: Major and minor version
  26. """
  27. match = re.search(r"""^ND2 FILE SIGNATURE CHUNK NAME01!Ver(?P<major>\d)\.(?P<minor>\d)$""", data)
  28. if match:
  29. # We haven't seen a lot of ND2s but the ones we have seen conform to this
  30. return int(match.group('major')), int(match.group('minor'))
  31. raise InvalidVersionError("The version of the ND2 you specified is not supported.")
  32. def read_chunk(fh, chunk_location):
  33. """Reads a piece of data given the location of its pointer.
  34. Args:
  35. fh: an open file handle to the ND2
  36. chunk_location (int): location to read
  37. Returns:
  38. bytes: the data at the chunk location
  39. """
  40. if chunk_location is None or fh is None:
  41. return None
  42. fh.seek(chunk_location)
  43. # The chunk metadata is always 16 bytes long
  44. chunk_metadata = fh.read(16)
  45. header, relative_offset, data_length = struct.unpack("IIQ", chunk_metadata)
  46. if header != 0xabeceda:
  47. raise ValueError("The ND2 file seems to be corrupted.")
  48. # We start at the location of the chunk metadata, skip over the metadata, and then proceed to the
  49. # start of the actual data field, which is at some arbitrary place after the metadata.
  50. fh.seek(chunk_location + 16 + relative_offset)
  51. return fh.read(data_length)
  52. def read_array(fh, kind, chunk_location):
  53. """
  54. Args:
  55. fh: File handle of the nd2 file
  56. kind: data type, can be one of 'double', 'int' or 'float'
  57. chunk_location: the location of the array chunk in the binary nd2 file
  58. Returns:
  59. array.array: an array of the data
  60. """
  61. kinds = {'double': 'd',
  62. 'int': 'i',
  63. 'float': 'f'}
  64. if kind not in kinds:
  65. raise ValueError('You attempted to read an array of an unknown type.')
  66. raw_data = read_chunk(fh, chunk_location)
  67. if raw_data is None:
  68. return None
  69. return array.array(kinds[kind], raw_data)
  70. def _parse_unsigned_char(data):
  71. """
  72. Args:
  73. data: binary data
  74. Returns:
  75. char: the data converted to unsigned char
  76. """
  77. return struct.unpack("B", data.read(1))[0]
  78. def _parse_unsigned_int(data):
  79. """
  80. Args:
  81. data: binary data
  82. Returns:
  83. int: the data converted to unsigned int
  84. """
  85. return struct.unpack("I", data.read(4))[0]
  86. def _parse_unsigned_long(data):
  87. """
  88. Args:
  89. data: binary data
  90. Returns:
  91. long: the data converted to unsigned long
  92. """
  93. return struct.unpack("Q", data.read(8))[0]
  94. def _parse_double(data):
  95. """
  96. Args:
  97. data: binary data
  98. Returns:
  99. double: the data converted to double
  100. """
  101. return struct.unpack("d", data.read(8))[0]
  102. def _parse_string(data):
  103. """
  104. Args:
  105. data: binary data
  106. Returns:
  107. string: the data converted to string
  108. """
  109. value = data.read(2)
  110. # the string ends at the first instance of \x00\x00
  111. while not value.endswith(six.b("\x00\x00")):
  112. next_data = data.read(2)
  113. if len(next_data) == 0:
  114. break
  115. value += next_data
  116. try:
  117. decoded = value.decode("utf16")[:-1].encode("utf8")
  118. except UnicodeDecodeError:
  119. decoded = value.decode('utf8')
  120. return decoded
  121. def _parse_char_array(data):
  122. """
  123. Args:
  124. data: binary data
  125. Returns:
  126. array.array: the data converted to an array
  127. """
  128. array_length = struct.unpack("Q", data.read(8))[0]
  129. return array.array("B", data.read(array_length))
  130. def parse_date(text_info):
  131. """
  132. The date and time when acquisition began.
  133. Args:
  134. text_info: the text that contains the date and time information
  135. Returns:
  136. datetime: the date and time of the acquisition
  137. """
  138. for line in text_info.values():
  139. line = line.decode("utf8")
  140. # ND2s seem to randomly switch between 12- and 24-hour representations.
  141. try:
  142. absolute_start = datetime.strptime(line, "%m/%d/%Y %H:%M:%S")
  143. except (TypeError, ValueError):
  144. try:
  145. absolute_start = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p")
  146. except (TypeError, ValueError):
  147. absolute_start = None
  148. if absolute_start is not None:
  149. return absolute_start
  150. return None
  151. def _parse_metadata_item(data, cursor_position):
  152. """Reads hierarchical data, analogous to a Python dict.
  153. Args:
  154. data: the binary data that needs to be parsed
  155. cursor_position: the position in the binary nd2 file
  156. Returns:
  157. dict: a dictionary containing the metadata item
  158. """
  159. new_count, length = struct.unpack("<IQ", data.read(12))
  160. length -= data.tell() - cursor_position
  161. next_data_length = data.read(length)
  162. value = read_metadata(next_data_length, new_count)
  163. # Skip some offsets
  164. data.read(new_count * 8)
  165. return value
  166. def _get_value(data, data_type, cursor_position):
  167. """ND2s use various codes to indicate different data types, which we translate here.
  168. Args:
  169. data: the binary data
  170. data_type: the data type (unsigned char = 1, unsigned int = 2 or 3, unsigned long = 5, double = 6, string = 8,
  171. char array = 9, metadata item = 11)
  172. cursor_position: the cursor position in the binary nd2 file
  173. Returns:
  174. mixed: the parsed value
  175. """
  176. parser = {1: _parse_unsigned_char,
  177. 2: _parse_unsigned_int,
  178. 3: _parse_unsigned_int,
  179. 5: _parse_unsigned_long,
  180. 6: _parse_double,
  181. 8: _parse_string,
  182. 9: _parse_char_array,
  183. 11: _parse_metadata_item}
  184. try:
  185. value = parser[data_type](data) if data_type < 11 else parser[data_type](data, cursor_position)
  186. except (KeyError, struct.error):
  187. value = None
  188. return value
  189. def read_metadata(data, count):
  190. """
  191. Iterates over each element of some section of the metadata and parses it.
  192. Args:
  193. data: the metadata in binary form
  194. count: the number of metadata elements
  195. Returns:
  196. dict: a dictionary containing the parsed metadata
  197. """
  198. if data is None:
  199. return None
  200. data = six.BytesIO(data)
  201. metadata = {}
  202. for _ in range(count):
  203. cursor_position = data.tell()
  204. header = data.read(2)
  205. if not header:
  206. # We've reached the end of some hierarchy of data
  207. break
  208. data_type, name_length = struct.unpack('BB', header)
  209. name = data.read(name_length * 2).decode("utf16")[:-1].encode("utf8")
  210. value = _get_value(data, data_type, cursor_position)
  211. metadata = _add_to_metadata(metadata, name, value)
  212. return metadata
  213. def _add_to_metadata(metadata, name, value):
  214. """
  215. Add the name value pair to the metadata dict
  216. Args:
  217. metadata (dict): a dictionary containing the metadata
  218. name (string): the dictionary key
  219. value: the value to add
  220. Returns:
  221. dict: the new metadata dictionary
  222. """
  223. if name not in metadata.keys():
  224. metadata[name] = value
  225. else:
  226. if not isinstance(metadata[name], list):
  227. # We have encountered this key exactly once before. Since we're seeing it again, we know we
  228. # need to convert it to a list before proceeding.
  229. metadata[name] = [metadata[name]]
  230. # We've encountered this key before so we're guaranteed to be dealing with a list. Thus we append
  231. # the value to the already-existing list.
  232. metadata[name].append(value)
  233. return metadata
  234. def get_from_dict_if_exists(key, dictionary, convert_key_to_binary=True):
  235. """
  236. Get the entry from the dictionary if it exists
  237. Args:
  238. key: key to lookup
  239. dictionary: dictionary to look in
  240. convert_key_to_binary: convert the key from string to binary if true
  241. Returns:
  242. the value of dictionary[key] or None
  243. """
  244. if convert_key_to_binary:
  245. key = six.b(key)
  246. if key not in dictionary:
  247. return None
  248. return dictionary[key]
  249. def check_or_make_dir(directory):
  250. """
  251. Check if a directory exists, if not, create it
  252. Args:
  253. directory: the path to the directory
  254. """
  255. if not os.path.exists(directory):
  256. os.makedirs(directory)