You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

210 lines
9.4 KiB

  1. from __future__ import unicode_literals
  2. import re
  3. import binascii
  4. try:
  5. from Crypto.Cipher import AES
  6. can_decrypt_frag = True
  7. except ImportError:
  8. can_decrypt_frag = False
  9. from .fragment import FragmentFD
  10. from .external import FFmpegFD
  11. from ..compat import (
  12. compat_urllib_error,
  13. compat_urlparse,
  14. compat_struct_pack,
  15. )
  16. from ..utils import (
  17. parse_m3u8_attributes,
  18. update_url_query,
  19. )
  20. class HlsFD(FragmentFD):
  21. """ A limited implementation that does not require ffmpeg """
  22. FD_NAME = 'hlsnative'
  23. @staticmethod
  24. def can_download(manifest, info_dict):
  25. UNSUPPORTED_FEATURES = (
  26. r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)', # encrypted streams [1]
  27. # r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2]
  28. # Live streams heuristic does not always work (e.g. geo restricted to Germany
  29. # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
  30. # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)', # live streams [3]
  31. # This heuristic also is not correct since segments may not be appended as well.
  32. # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
  33. # no segments will definitely be appended to the end of the playlist.
  34. # r'#EXT-X-PLAYLIST-TYPE:EVENT', # media segments may be appended to the end of
  35. # # event media playlists [4]
  36. # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
  37. # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
  38. # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
  39. # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
  40. )
  41. check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
  42. is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
  43. check_results.append(can_decrypt_frag or not is_aes128_enc)
  44. check_results.append(not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest))
  45. check_results.append(not info_dict.get('is_live'))
  46. return all(check_results)
  47. def real_download(self, filename, info_dict):
  48. man_url = info_dict['url']
  49. self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
  50. urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
  51. man_url = urlh.geturl()
  52. s = urlh.read().decode('utf-8', 'ignore')
  53. if not self.can_download(s, info_dict):
  54. if info_dict.get('extra_param_to_segment_url'):
  55. self.report_error('pycrypto not found. Please install it.')
  56. return False
  57. self.report_warning(
  58. 'hlsnative has detected features it does not support, '
  59. 'extraction will be delegated to ffmpeg')
  60. fd = FFmpegFD(self.ydl, self.params)
  61. for ph in self._progress_hooks:
  62. fd.add_progress_hook(ph)
  63. return fd.real_download(filename, info_dict)
  64. def is_ad_fragment_start(s):
  65. return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
  66. or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad'))
  67. def is_ad_fragment_end(s):
  68. return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s
  69. or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',segment'))
  70. media_frags = 0
  71. ad_frags = 0
  72. ad_frag_next = False
  73. for line in s.splitlines():
  74. line = line.strip()
  75. if not line:
  76. continue
  77. if line.startswith('#'):
  78. if is_ad_fragment_start(line):
  79. ad_frag_next = True
  80. elif is_ad_fragment_end(line):
  81. ad_frag_next = False
  82. continue
  83. if ad_frag_next:
  84. ad_frags += 1
  85. continue
  86. media_frags += 1
  87. ctx = {
  88. 'filename': filename,
  89. 'total_frags': media_frags,
  90. 'ad_frags': ad_frags,
  91. }
  92. self._prepare_and_start_frag_download(ctx)
  93. fragment_retries = self.params.get('fragment_retries', 0)
  94. skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
  95. test = self.params.get('test', False)
  96. extra_query = None
  97. extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
  98. if extra_param_to_segment_url:
  99. extra_query = compat_urlparse.parse_qs(extra_param_to_segment_url)
  100. i = 0
  101. media_sequence = 0
  102. decrypt_info = {'METHOD': 'NONE'}
  103. byte_range = {}
  104. frag_index = 0
  105. ad_frag_next = False
  106. for line in s.splitlines():
  107. line = line.strip()
  108. if line:
  109. if not line.startswith('#'):
  110. if ad_frag_next:
  111. continue
  112. frag_index += 1
  113. if frag_index <= ctx['fragment_index']:
  114. continue
  115. frag_url = (
  116. line
  117. if re.match(r'^https?://', line)
  118. else compat_urlparse.urljoin(man_url, line))
  119. if extra_query:
  120. frag_url = update_url_query(frag_url, extra_query)
  121. count = 0
  122. headers = info_dict.get('http_headers', {})
  123. if byte_range:
  124. headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'])
  125. while count <= fragment_retries:
  126. try:
  127. success, frag_content = self._download_fragment(
  128. ctx, frag_url, info_dict, headers)
  129. if not success:
  130. return False
  131. break
  132. except compat_urllib_error.HTTPError as err:
  133. # Unavailable (possibly temporary) fragments may be served.
  134. # First we try to retry then either skip or abort.
  135. # See https://github.com/ytdl-org/youtube-dl/issues/10165,
  136. # https://github.com/ytdl-org/youtube-dl/issues/10448).
  137. count += 1
  138. if count <= fragment_retries:
  139. self.report_retry_fragment(err, frag_index, count, fragment_retries)
  140. if count > fragment_retries:
  141. if skip_unavailable_fragments:
  142. i += 1
  143. media_sequence += 1
  144. self.report_skip_fragment(frag_index)
  145. continue
  146. self.report_error(
  147. 'giving up after %s fragment retries' % fragment_retries)
  148. return False
  149. if decrypt_info['METHOD'] == 'AES-128':
  150. iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
  151. decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(
  152. self._prepare_url(info_dict, decrypt_info['URI'])).read()
  153. frag_content = AES.new(
  154. decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
  155. self._append_fragment(ctx, frag_content)
  156. # We only download the first fragment during the test
  157. if test:
  158. break
  159. i += 1
  160. media_sequence += 1
  161. elif line.startswith('#EXT-X-KEY'):
  162. decrypt_url = decrypt_info.get('URI')
  163. decrypt_info = parse_m3u8_attributes(line[11:])
  164. if decrypt_info['METHOD'] == 'AES-128':
  165. if 'IV' in decrypt_info:
  166. decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:].zfill(32))
  167. if not re.match(r'^https?://', decrypt_info['URI']):
  168. decrypt_info['URI'] = compat_urlparse.urljoin(
  169. man_url, decrypt_info['URI'])
  170. if extra_query:
  171. decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
  172. if decrypt_url != decrypt_info['URI']:
  173. decrypt_info['KEY'] = None
  174. elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
  175. media_sequence = int(line[22:])
  176. elif line.startswith('#EXT-X-BYTERANGE'):
  177. splitted_byte_range = line[17:].split('@')
  178. sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
  179. byte_range = {
  180. 'start': sub_range_start,
  181. 'end': sub_range_start + int(splitted_byte_range[0]),
  182. }
  183. elif is_ad_fragment_start(line):
  184. ad_frag_next = True
  185. elif is_ad_fragment_end(line):
  186. ad_frag_next = False
  187. self._finish_frag_download(ctx)
  188. return True