You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

169 lines
6.2 KiB

12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
  1. #!/usr/bin/env python
  2. import errno
  3. import hashlib
  4. import io
  5. import os
  6. import json
  7. import unittest
  8. import sys
  9. import socket
  10. import binascii
  11. # Allow direct execution
  12. sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  13. import youtube_dl.YoutubeDL
  14. from youtube_dl.utils import *
  15. PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
  16. RETRIES = 3
  17. # General configuration (from __init__, not very elegant...)
  18. jar = compat_cookiejar.CookieJar()
  19. cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
  20. proxy_handler = compat_urllib_request.ProxyHandler()
  21. opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
  22. compat_urllib_request.install_opener(opener)
  23. socket.setdefaulttimeout(10)
  24. def _try_rm(filename):
  25. """ Remove a file if it exists """
  26. try:
  27. os.remove(filename)
  28. except OSError as ose:
  29. if ose.errno != errno.ENOENT:
  30. raise
  31. md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
  32. class YoutubeDL(youtube_dl.YoutubeDL):
  33. def __init__(self, *args, **kwargs):
  34. self.to_stderr = self.to_screen
  35. self.processed_info_dicts = []
  36. super(YoutubeDL, self).__init__(*args, **kwargs)
  37. def report_warning(self, message):
  38. # Don't accept warnings during tests
  39. raise ExtractorError(message)
  40. def process_info(self, info_dict):
  41. self.processed_info_dicts.append(info_dict)
  42. return super(YoutubeDL, self).process_info(info_dict)
  43. def _file_md5(fn):
  44. with open(fn, 'rb') as f:
  45. return hashlib.md5(f.read()).hexdigest()
  46. from helper import get_testcases
  47. defs = get_testcases()
  48. with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
  49. parameters = json.load(pf)
  50. class TestDownload(unittest.TestCase):
  51. maxDiff = None
  52. def setUp(self):
  53. self.parameters = parameters
  54. self.defs = defs
  55. ### Dynamically generate tests
  56. def generator(test_case):
  57. def test_template(self):
  58. ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
  59. def print_skipping(reason):
  60. print('Skipping %s: %s' % (test_case['name'], reason))
  61. if not ie._WORKING:
  62. print_skipping('IE marked as not _WORKING')
  63. return
  64. if 'playlist' not in test_case and not test_case['file']:
  65. print_skipping('No output file specified')
  66. return
  67. if 'skip' in test_case:
  68. print_skipping(test_case['skip'])
  69. return
  70. params = self.parameters.copy()
  71. params.update(test_case.get('params', {}))
  72. ydl = YoutubeDL(params)
  73. ydl.add_default_info_extractors()
  74. finished_hook_called = set()
  75. def _hook(status):
  76. if status['status'] == 'finished':
  77. finished_hook_called.add(status['filename'])
  78. ydl.fd.add_progress_hook(_hook)
  79. test_cases = test_case.get('playlist', [test_case])
  80. for tc in test_cases:
  81. _try_rm(tc['file'])
  82. _try_rm(tc['file'] + '.part')
  83. _try_rm(tc['file'] + '.info.json')
  84. try:
  85. for retry in range(1, RETRIES + 1):
  86. try:
  87. ydl.download([test_case['url']])
  88. except (DownloadError, ExtractorError) as err:
  89. if retry == RETRIES: raise
  90. # Check if the exception is not a network related one
  91. if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
  92. raise
  93. print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
  94. else:
  95. break
  96. for tc in test_cases:
  97. if not test_case.get('params', {}).get('skip_download', False):
  98. self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file'])
  99. self.assertTrue(tc['file'] in finished_hook_called)
  100. self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
  101. if 'md5' in tc:
  102. md5_for_file = _file_md5(tc['file'])
  103. self.assertEqual(md5_for_file, tc['md5'])
  104. with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
  105. info_dict = json.load(infof)
  106. for (info_field, expected) in tc.get('info_dict', {}).items():
  107. if isinstance(expected, compat_str) and expected.startswith('md5:'):
  108. self.assertEqual(expected, 'md5:' + md5(info_dict.get(info_field)))
  109. else:
  110. got = info_dict.get(info_field)
  111. self.assertEqual(
  112. expected, got,
  113. u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
  114. # If checkable fields are missing from the test case, print the info_dict
  115. test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
  116. for key, value in info_dict.items()
  117. if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location'))
  118. if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()):
  119. sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u'\n')
  120. # Check for the presence of mandatory fields
  121. for key in ('id', 'url', 'title', 'ext'):
  122. self.assertTrue(key in info_dict.keys() and info_dict[key])
  123. finally:
  124. for tc in test_cases:
  125. _try_rm(tc['file'])
  126. _try_rm(tc['file'] + '.part')
  127. _try_rm(tc['file'] + '.info.json')
  128. return test_template
  129. ### And add them to TestDownload
  130. for n, test_case in enumerate(defs):
  131. test_method = generator(test_case)
  132. tname = 'test_' + str(test_case['name'])
  133. i = 1
  134. while hasattr(TestDownload, tname):
  135. tname = 'test_' + str(test_case['name']) + '_' + str(i)
  136. i += 1
  137. test_method.__name__ = tname
  138. setattr(TestDownload, test_method.__name__, test_method)
  139. del test_method
  140. if __name__ == '__main__':
  141. unittest.main()