You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

98 lines
2.8 KiB

  1. from __future__ import unicode_literals, print_function
  2. from inspect import getsource
  3. import os
  4. from os.path import dirname as dirn
  5. import sys
  6. print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
  7. sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
  8. lazy_extractors_filename = sys.argv[1]
  9. if os.path.exists(lazy_extractors_filename):
  10. os.remove(lazy_extractors_filename)
  11. from youtube_dl.extractor import _ALL_CLASSES
  12. from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
  13. with open('devscripts/lazy_load_template.py', 'rt') as f:
  14. module_template = f.read()
  15. module_contents = [
  16. module_template + '\n' + getsource(InfoExtractor.suitable) + '\n',
  17. 'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n']
  18. ie_template = '''
  19. class {name}({bases}):
  20. _VALID_URL = {valid_url!r}
  21. _module = '{module}'
  22. '''
  23. make_valid_template = '''
  24. @classmethod
  25. def _make_valid_url(cls):
  26. return {valid_url!r}
  27. '''
  28. def get_base_name(base):
  29. if base is InfoExtractor:
  30. return 'LazyLoadExtractor'
  31. elif base is SearchInfoExtractor:
  32. return 'LazyLoadSearchExtractor'
  33. else:
  34. return base.__name__
  35. def build_lazy_ie(ie, name):
  36. valid_url = getattr(ie, '_VALID_URL', None)
  37. s = ie_template.format(
  38. name=name,
  39. bases=', '.join(map(get_base_name, ie.__bases__)),
  40. valid_url=valid_url,
  41. module=ie.__module__)
  42. if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
  43. s += '\n' + getsource(ie.suitable)
  44. if hasattr(ie, '_make_valid_url'):
  45. # search extractors
  46. s += make_valid_template.format(valid_url=ie._make_valid_url())
  47. return s
  48. # find the correct sorting and add the required base classes so that sublcasses
  49. # can be correctly created
  50. classes = _ALL_CLASSES[:-1]
  51. ordered_cls = []
  52. while classes:
  53. for c in classes[:]:
  54. bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor))
  55. stop = False
  56. for b in bases:
  57. if b not in classes and b not in ordered_cls:
  58. if b.__name__ == 'GenericIE':
  59. exit()
  60. classes.insert(0, b)
  61. stop = True
  62. if stop:
  63. break
  64. if all(b in ordered_cls for b in bases):
  65. ordered_cls.append(c)
  66. classes.remove(c)
  67. break
  68. ordered_cls.append(_ALL_CLASSES[-1])
  69. names = []
  70. for ie in ordered_cls:
  71. name = ie.__name__
  72. src = build_lazy_ie(ie, name)
  73. module_contents.append(src)
  74. if ie in _ALL_CLASSES:
  75. names.append(name)
  76. module_contents.append(
  77. '_ALL_CLASSES = [{0}]'.format(', '.join(names)))
  78. module_src = '\n'.join(module_contents) + '\n'
  79. with open(lazy_extractors_filename, 'wt') as f:
  80. f.write(module_src)