You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

353 lines
8.9 KiB

8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
  1. import os
  2. import struct
  3. import array
  4. from datetime import datetime
  5. import six
  6. import re
  7. from nd2reader.exceptions import InvalidVersionError
  8. def get_version(fh):
  9. """Determines what version the ND2 is.
  10. Args:
  11. fh: File handle of the .nd2 file
  12. Returns:
  13. tuple: Major and minor version
  14. """
  15. # the first 16 bytes seem to have no meaning, so we skip them
  16. fh.seek(16)
  17. # the next 38 bytes contain the string that we want to parse. Unlike most of the ND2, this is in UTF-8
  18. data = fh.read(38).decode("utf8")
  19. return parse_version(data)
  20. def parse_version(data):
  21. """Parses a string with the version data in it.
  22. Args:
  23. data (unicode): the 19th through 54th byte of the ND2, representing the version
  24. Returns:
  25. tuple: Major and minor version
  26. """
  27. match = re.search(r"""^ND2 FILE SIGNATURE CHUNK NAME01!Ver(?P<major>\d)\.(?P<minor>\d)$""", data)
  28. if match:
  29. # We haven't seen a lot of ND2s but the ones we have seen conform to this
  30. return int(match.group('major')), int(match.group('minor'))
  31. raise InvalidVersionError("The version of the ND2 you specified is not supported.")
  32. def read_chunk(fh, chunk_location):
  33. """Reads a piece of data given the location of its pointer.
  34. Args:
  35. fh: an open file handle to the ND2
  36. chunk_location (int): location to read
  37. Returns:
  38. bytes: the data at the chunk location
  39. """
  40. if chunk_location is None or fh is None:
  41. return None
  42. fh.seek(chunk_location)
  43. # The chunk metadata is always 16 bytes long
  44. chunk_metadata = fh.read(16)
  45. header, relative_offset, data_length = struct.unpack("IIQ", chunk_metadata)
  46. if header != 0xabeceda:
  47. raise ValueError("The ND2 file seems to be corrupted.")
  48. # We start at the location of the chunk metadata, skip over the metadata, and then proceed to the
  49. # start of the actual data field, which is at some arbitrary place after the metadata.
  50. fh.seek(chunk_location + 16 + relative_offset)
  51. return fh.read(data_length)
  52. def read_array(fh, kind, chunk_location):
  53. """
  54. Args:
  55. fh: File handle of the nd2 file
  56. kind: data type, can be one of 'double', 'int' or 'float'
  57. chunk_location: the location of the array chunk in the binary nd2 file
  58. Returns:
  59. array.array: an array of the data
  60. """
  61. kinds = {'double': 'd',
  62. 'int': 'i',
  63. 'float': 'f'}
  64. if kind not in kinds:
  65. raise ValueError('You attempted to read an array of an unknown type.')
  66. raw_data = read_chunk(fh, chunk_location)
  67. if raw_data is None:
  68. return None
  69. return array.array(kinds[kind], raw_data)
  70. def _parse_unsigned_char(data):
  71. """
  72. Args:
  73. data: binary data
  74. Returns:
  75. char: the data converted to unsigned char
  76. """
  77. return struct.unpack("B", data.read(1))[0]
  78. def _parse_unsigned_int(data):
  79. """
  80. Args:
  81. data: binary data
  82. Returns:
  83. int: the data converted to unsigned int
  84. """
  85. return struct.unpack("I", data.read(4))[0]
  86. def _parse_unsigned_long(data):
  87. """
  88. Args:
  89. data: binary data
  90. Returns:
  91. long: the data converted to unsigned long
  92. """
  93. return struct.unpack("Q", data.read(8))[0]
  94. def _parse_double(data):
  95. """
  96. Args:
  97. data: binary data
  98. Returns:
  99. double: the data converted to double
  100. """
  101. return struct.unpack("d", data.read(8))[0]
  102. def _parse_string(data):
  103. """
  104. Args:
  105. data: binary data
  106. Returns:
  107. string: the data converted to string
  108. """
  109. value = data.read(2)
  110. # the string ends at the first instance of \x00\x00
  111. while not value.endswith(six.b("\x00\x00")):
  112. next_data = data.read(2)
  113. if len(next_data) == 0:
  114. break
  115. value += next_data
  116. try:
  117. decoded = value.decode("utf16")[:-1].encode("utf8")
  118. except UnicodeDecodeError:
  119. decoded = value.decode('utf8').encode("utf8")
  120. return decoded
  121. def _parse_char_array(data):
  122. """
  123. Args:
  124. data: binary data
  125. Returns:
  126. array.array: the data converted to an array
  127. """
  128. array_length = struct.unpack("Q", data.read(8))[0]
  129. return array.array("B", data.read(array_length))
  130. def parse_date(text_info):
  131. """
  132. The date and time when acquisition began.
  133. Args:
  134. text_info: the text that contains the date and time information
  135. Returns:
  136. datetime: the date and time of the acquisition
  137. """
  138. for line in text_info.values():
  139. line = line.decode("utf8")
  140. # ND2s seem to randomly switch between 12- and 24-hour representations.
  141. possible_formats = ["%m/%d/%Y %H:%M:%S", "%m/%d/%Y %I:%M:%S %p", "%d/%m/%Y %H:%M:%S"]
  142. for date_format in possible_formats:
  143. try:
  144. absolute_start = datetime.strptime(line, date_format)
  145. except (TypeError, ValueError):
  146. continue
  147. return absolute_start
  148. return None
  149. def _parse_metadata_item(data, cursor_position):
  150. """Reads hierarchical data, analogous to a Python dict.
  151. Args:
  152. data: the binary data that needs to be parsed
  153. cursor_position: the position in the binary nd2 file
  154. Returns:
  155. dict: a dictionary containing the metadata item
  156. """
  157. new_count, length = struct.unpack("<IQ", data.read(12))
  158. length -= data.tell() - cursor_position
  159. next_data_length = data.read(length)
  160. value = read_metadata(next_data_length, new_count)
  161. # Skip some offsets
  162. data.read(new_count * 8)
  163. return value
  164. def _get_value(data, data_type, cursor_position):
  165. """ND2s use various codes to indicate different data types, which we translate here.
  166. Args:
  167. data: the binary data
  168. data_type: the data type (unsigned char = 1, unsigned int = 2 or 3, unsigned long = 5, double = 6, string = 8,
  169. char array = 9, metadata item = 11)
  170. cursor_position: the cursor position in the binary nd2 file
  171. Returns:
  172. mixed: the parsed value
  173. """
  174. parser = {1: _parse_unsigned_char,
  175. 2: _parse_unsigned_int,
  176. 3: _parse_unsigned_int,
  177. 5: _parse_unsigned_long,
  178. 6: _parse_double,
  179. 8: _parse_string,
  180. 9: _parse_char_array,
  181. 11: _parse_metadata_item}
  182. try:
  183. value = parser[data_type](data) if data_type < 11 else parser[data_type](data, cursor_position)
  184. except (KeyError, struct.error):
  185. value = None
  186. return value
  187. def read_metadata(data, count):
  188. """
  189. Iterates over each element of some section of the metadata and parses it.
  190. Args:
  191. data: the metadata in binary form
  192. count: the number of metadata elements
  193. Returns:
  194. dict: a dictionary containing the parsed metadata
  195. """
  196. if data is None:
  197. return None
  198. data = six.BytesIO(data)
  199. metadata = {}
  200. for _ in range(count):
  201. cursor_position = data.tell()
  202. header = data.read(2)
  203. if not header:
  204. # We've reached the end of some hierarchy of data
  205. break
  206. data_type, name_length = struct.unpack('BB', header)
  207. name = data.read(name_length * 2).decode("utf16")[:-1].encode("utf8")
  208. value = _get_value(data, data_type, cursor_position)
  209. metadata = _add_to_metadata(metadata, name, value)
  210. return metadata
  211. def _add_to_metadata(metadata, name, value):
  212. """
  213. Add the name value pair to the metadata dict
  214. Args:
  215. metadata (dict): a dictionary containing the metadata
  216. name (string): the dictionary key
  217. value: the value to add
  218. Returns:
  219. dict: the new metadata dictionary
  220. """
  221. if name not in metadata.keys():
  222. metadata[name] = value
  223. else:
  224. if not isinstance(metadata[name], list):
  225. # We have encountered this key exactly once before. Since we're seeing it again, we know we
  226. # need to convert it to a list before proceeding.
  227. metadata[name] = [metadata[name]]
  228. # We've encountered this key before so we're guaranteed to be dealing with a list. Thus we append
  229. # the value to the already-existing list.
  230. metadata[name].append(value)
  231. return metadata
  232. def get_from_dict_if_exists(key, dictionary, convert_key_to_binary=True):
  233. """
  234. Get the entry from the dictionary if it exists
  235. Args:
  236. key: key to lookup
  237. dictionary: dictionary to look in
  238. convert_key_to_binary: convert the key from string to binary if true
  239. Returns:
  240. the value of dictionary[key] or None
  241. """
  242. if convert_key_to_binary:
  243. key = six.b(key)
  244. if key not in dictionary:
  245. return None
  246. return dictionary[key]
  247. def check_or_make_dir(directory):
  248. """
  249. Check if a directory exists, if not, create it
  250. Args:
  251. directory: the path to the directory
  252. """
  253. if not os.path.exists(directory):
  254. os.makedirs(directory)