You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

232 lines
8.4 KiB

10 years ago
10 years ago
10 years ago
10 years ago
12 years ago
10 years ago
12 years ago
10 years ago
10 years ago
12 years ago
10 years ago
10 years ago
12 years ago
10 years ago
10 years ago
10 years ago
  1. #!/usr/bin/env python
  2. from __future__ import unicode_literals
  3. # Allow direct execution
  4. import os
  5. import sys
  6. import unittest
  7. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  8. from test.helper import (
  9. assertGreaterEqual,
  10. expect_warnings,
  11. get_params,
  12. gettestcases,
  13. expect_info_dict,
  14. try_rm,
  15. report_warning,
  16. )
  17. import hashlib
  18. import io
  19. import json
  20. import socket
  21. import youtube_dl.YoutubeDL
  22. from youtube_dl.compat import (
  23. compat_http_client,
  24. compat_urllib_error,
  25. compat_HTTPError,
  26. )
  27. from youtube_dl.utils import (
  28. DownloadError,
  29. ExtractorError,
  30. format_bytes,
  31. UnavailableVideoError,
  32. )
  33. from youtube_dl.extractor import get_info_extractor
  34. RETRIES = 3
  35. class YoutubeDL(youtube_dl.YoutubeDL):
  36. def __init__(self, *args, **kwargs):
  37. self.to_stderr = self.to_screen
  38. self.processed_info_dicts = []
  39. super(YoutubeDL, self).__init__(*args, **kwargs)
  40. def report_warning(self, message):
  41. # Don't accept warnings during tests
  42. raise ExtractorError(message)
  43. def process_info(self, info_dict):
  44. self.processed_info_dicts.append(info_dict)
  45. return super(YoutubeDL, self).process_info(info_dict)
  46. def _file_md5(fn):
  47. with open(fn, 'rb') as f:
  48. return hashlib.md5(f.read()).hexdigest()
  49. defs = gettestcases()
  50. class TestDownload(unittest.TestCase):
  51. maxDiff = None
  52. def setUp(self):
  53. self.defs = defs
  54. # Dynamically generate tests
  55. def generator(test_case):
  56. def test_template(self):
  57. ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
  58. other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
  59. is_playlist = any(k.startswith('playlist') for k in test_case)
  60. test_cases = test_case.get(
  61. 'playlist', [] if is_playlist else [test_case])
  62. def print_skipping(reason):
  63. print('Skipping %s: %s' % (test_case['name'], reason))
  64. if not ie.working():
  65. print_skipping('IE marked as not _WORKING')
  66. return
  67. for tc in test_cases:
  68. info_dict = tc.get('info_dict', {})
  69. if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
  70. raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
  71. if 'skip' in test_case:
  72. print_skipping(test_case['skip'])
  73. return
  74. for other_ie in other_ies:
  75. if not other_ie.working():
  76. print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
  77. return
  78. params = get_params(test_case.get('params', {}))
  79. if is_playlist and 'playlist' not in test_case:
  80. params.setdefault('extract_flat', True)
  81. params.setdefault('skip_download', True)
  82. ydl = YoutubeDL(params, auto_init=False)
  83. ydl.add_default_info_extractors()
  84. finished_hook_called = set()
  85. def _hook(status):
  86. if status['status'] == 'finished':
  87. finished_hook_called.add(status['filename'])
  88. ydl.add_progress_hook(_hook)
  89. expect_warnings(ydl, test_case.get('expected_warnings', []))
  90. def get_tc_filename(tc):
  91. return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
  92. res_dict = None
  93. def try_rm_tcs_files(tcs=None):
  94. if tcs is None:
  95. tcs = test_cases
  96. for tc in tcs:
  97. tc_filename = get_tc_filename(tc)
  98. try_rm(tc_filename)
  99. try_rm(tc_filename + '.part')
  100. try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
  101. try_rm_tcs_files()
  102. try:
  103. try_num = 1
  104. while True:
  105. try:
  106. # We're not using .download here sine that is just a shim
  107. # for outside error handling, and returns the exit code
  108. # instead of the result dict.
  109. res_dict = ydl.extract_info(test_case['url'])
  110. except (DownloadError, ExtractorError) as err:
  111. # Check if the exception is not a network related one
  112. if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
  113. raise
  114. if try_num == RETRIES:
  115. report_warning('Failed due to network errors, skipping...')
  116. return
  117. print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
  118. try_num += 1
  119. else:
  120. break
  121. if is_playlist:
  122. self.assertEqual(res_dict['_type'], 'playlist')
  123. self.assertTrue('entries' in res_dict)
  124. expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
  125. if 'playlist_mincount' in test_case:
  126. assertGreaterEqual(
  127. self,
  128. len(res_dict['entries']),
  129. test_case['playlist_mincount'],
  130. 'Expected at least %d in playlist %s, but got only %d' % (
  131. test_case['playlist_mincount'], test_case['url'],
  132. len(res_dict['entries'])))
  133. if 'playlist_count' in test_case:
  134. self.assertEqual(
  135. len(res_dict['entries']),
  136. test_case['playlist_count'],
  137. 'Expected %d entries in playlist %s, but got %d.' % (
  138. test_case['playlist_count'],
  139. test_case['url'],
  140. len(res_dict['entries']),
  141. ))
  142. if 'playlist_duration_sum' in test_case:
  143. got_duration = sum(e['duration'] for e in res_dict['entries'])
  144. self.assertEqual(
  145. test_case['playlist_duration_sum'], got_duration)
  146. for tc in test_cases:
  147. tc_filename = get_tc_filename(tc)
  148. if not test_case.get('params', {}).get('skip_download', False):
  149. self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
  150. self.assertTrue(tc_filename in finished_hook_called)
  151. expected_minsize = tc.get('file_minsize', 10000)
  152. if expected_minsize is not None:
  153. if params.get('test'):
  154. expected_minsize = max(expected_minsize, 10000)
  155. got_fsize = os.path.getsize(tc_filename)
  156. assertGreaterEqual(
  157. self, got_fsize, expected_minsize,
  158. 'Expected %s to be at least %s, but it\'s only %s ' %
  159. (tc_filename, format_bytes(expected_minsize),
  160. format_bytes(got_fsize)))
  161. if 'md5' in tc:
  162. md5_for_file = _file_md5(tc_filename)
  163. self.assertEqual(md5_for_file, tc['md5'])
  164. info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
  165. self.assertTrue(
  166. os.path.exists(info_json_fn),
  167. 'Missing info file %s' % info_json_fn)
  168. with io.open(info_json_fn, encoding='utf-8') as infof:
  169. info_dict = json.load(infof)
  170. expect_info_dict(self, info_dict, tc.get('info_dict', {}))
  171. finally:
  172. try_rm_tcs_files()
  173. if is_playlist and res_dict is not None and res_dict.get('entries'):
  174. # Remove all other files that may have been extracted if the
  175. # extractor returns full results even with extract_flat
  176. res_tcs = [{'info_dict': e} for e in res_dict['entries']]
  177. try_rm_tcs_files(res_tcs)
  178. return test_template
  179. # And add them to TestDownload
  180. for n, test_case in enumerate(defs):
  181. test_method = generator(test_case)
  182. tname = 'test_' + str(test_case['name'])
  183. i = 1
  184. while hasattr(TestDownload, tname):
  185. tname = 'test_%s_%d' % (test_case['name'], i)
  186. i += 1
  187. test_method.__name__ = str(tname)
  188. setattr(TestDownload, test_method.__name__, test_method)
  189. del test_method
  190. if __name__ == '__main__':
  191. unittest.main()