You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1829 lines
80 KiB

11 years ago
  1. # coding: utf-8
  2. import collections
  3. import errno
  4. import io
  5. import itertools
  6. import json
  7. import os.path
  8. import re
  9. import string
  10. import struct
  11. import traceback
  12. import zlib
  13. from .common import InfoExtractor, SearchInfoExtractor
  14. from .subtitles import SubtitlesInfoExtractor
  15. from ..utils import (
  16. compat_chr,
  17. compat_parse_qs,
  18. compat_urllib_parse,
  19. compat_urllib_request,
  20. compat_urlparse,
  21. compat_str,
  22. clean_html,
  23. get_cachedir,
  24. get_element_by_id,
  25. get_element_by_attribute,
  26. ExtractorError,
  27. int_or_none,
  28. PagedList,
  29. RegexNotFoundError,
  30. unescapeHTML,
  31. unified_strdate,
  32. orderedSet,
  33. write_json_file,
  34. uppercase_escape,
  35. )
  36. class YoutubeBaseInfoExtractor(InfoExtractor):
  37. """Provide base functions for Youtube extractors"""
  38. _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  39. _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
  40. _AGE_URL = 'https://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
  41. _NETRC_MACHINE = 'youtube'
  42. # If True it will raise an error if no login info is provided
  43. _LOGIN_REQUIRED = False
  44. def _set_language(self):
  45. return bool(self._download_webpage(
  46. self._LANG_URL, None,
  47. note=u'Setting language', errnote='unable to set language',
  48. fatal=False))
  49. def _login(self):
  50. (username, password) = self._get_login_info()
  51. # No authentication to be performed
  52. if username is None:
  53. if self._LOGIN_REQUIRED:
  54. raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
  55. return False
  56. login_page = self._download_webpage(
  57. self._LOGIN_URL, None,
  58. note=u'Downloading login page',
  59. errnote=u'unable to fetch login page', fatal=False)
  60. if login_page is False:
  61. return
  62. galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
  63. login_page, u'Login GALX parameter')
  64. # Log in
  65. login_form_strs = {
  66. u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
  67. u'Email': username,
  68. u'GALX': galx,
  69. u'Passwd': password,
  70. u'PersistentCookie': u'yes',
  71. u'_utf8': u'',
  72. u'bgresponse': u'js_disabled',
  73. u'checkConnection': u'',
  74. u'checkedDomains': u'youtube',
  75. u'dnConn': u'',
  76. u'pstMsg': u'0',
  77. u'rmShown': u'1',
  78. u'secTok': u'',
  79. u'signIn': u'Sign in',
  80. u'timeStmp': u'',
  81. u'service': u'youtube',
  82. u'uilel': u'3',
  83. u'hl': u'en_US',
  84. }
  85. # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
  86. # chokes on unicode
  87. login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
  88. login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
  89. req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
  90. login_results = self._download_webpage(
  91. req, None,
  92. note=u'Logging in', errnote=u'unable to log in', fatal=False)
  93. if login_results is False:
  94. return False
  95. if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
  96. self._downloader.report_warning(u'unable to log in: bad username or password')
  97. return False
  98. return True
  99. def _confirm_age(self):
  100. age_form = {
  101. 'next_url': '/',
  102. 'action_confirm': 'Confirm',
  103. }
  104. req = compat_urllib_request.Request(self._AGE_URL,
  105. compat_urllib_parse.urlencode(age_form).encode('ascii'))
  106. self._download_webpage(
  107. req, None,
  108. note=u'Confirming age', errnote=u'Unable to confirm age')
  109. return True
  110. def _real_initialize(self):
  111. if self._downloader is None:
  112. return
  113. if not self._set_language():
  114. return
  115. if not self._login():
  116. return
  117. self._confirm_age()
  118. class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
  119. IE_DESC = u'YouTube.com'
  120. _VALID_URL = r"""(?x)^
  121. (
  122. (?:https?://|//)? # http(s):// or protocol-independent URL (optional)
  123. (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
  124. (?:www\.)?deturl\.com/www\.youtube\.com/|
  125. (?:www\.)?pwnyoutube\.com/|
  126. tube\.majestyc\.net/|
  127. youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
  128. (?:.*?\#/)? # handle anchor (#/) redirect urls
  129. (?: # the various things that can precede the ID:
  130. (?:(?:v|embed|e)/) # v/ or embed/ or e/
  131. |(?: # or the v= param in all its forms
  132. (?:(?:watch|movie)(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
  133. (?:\?|\#!?) # the params delimiter ? or # or #!
  134. (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
  135. v=
  136. )
  137. ))
  138. |youtu\.be/ # just youtu.be/xxxx
  139. )
  140. )? # all until now is optional -> you can pass the naked ID
  141. ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
  142. (?(1).+)? # if we found the ID, everything can follow
  143. $"""
  144. _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
  145. _formats = {
  146. '5': {'ext': 'flv', 'width': 400, 'height': 240},
  147. '6': {'ext': 'flv', 'width': 450, 'height': 270},
  148. '13': {'ext': '3gp'},
  149. '17': {'ext': '3gp', 'width': 176, 'height': 144},
  150. '18': {'ext': 'mp4', 'width': 640, 'height': 360},
  151. '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
  152. '34': {'ext': 'flv', 'width': 640, 'height': 360},
  153. '35': {'ext': 'flv', 'width': 854, 'height': 480},
  154. '36': {'ext': '3gp', 'width': 320, 'height': 240},
  155. '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
  156. '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
  157. '43': {'ext': 'webm', 'width': 640, 'height': 360},
  158. '44': {'ext': 'webm', 'width': 854, 'height': 480},
  159. '45': {'ext': 'webm', 'width': 1280, 'height': 720},
  160. '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
  161. # 3d videos
  162. '82': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': '3D', 'preference': -20},
  163. '83': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': '3D', 'preference': -20},
  164. '84': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': '3D', 'preference': -20},
  165. '85': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': '3D', 'preference': -20},
  166. '100': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': '3D', 'preference': -20},
  167. '101': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': '3D', 'preference': -20},
  168. '102': {'ext': 'webm', 'height': 720, 'resolution': '720p', 'format_note': '3D', 'preference': -20},
  169. # Apple HTTP Live Streaming
  170. '92': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'HLS', 'preference': -10},
  171. '93': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': 'HLS', 'preference': -10},
  172. '94': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'HLS', 'preference': -10},
  173. '95': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'HLS', 'preference': -10},
  174. '96': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'HLS', 'preference': -10},
  175. '132': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'HLS', 'preference': -10},
  176. '151': {'ext': 'mp4', 'height': 72, 'resolution': '72p', 'format_note': 'HLS', 'preference': -10},
  177. # DASH mp4 video
  178. '133': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'DASH video', 'preference': -40},
  179. '134': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': 'DASH video', 'preference': -40},
  180. '135': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'DASH video', 'preference': -40},
  181. '136': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'DASH video', 'preference': -40},
  182. '137': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
  183. '138': {'ext': 'mp4', 'height': 1081, 'resolution': '>1080p', 'format_note': 'DASH video', 'preference': -40},
  184. '160': {'ext': 'mp4', 'height': 192, 'resolution': '192p', 'format_note': 'DASH video', 'preference': -40},
  185. '264': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
  186. # Dash mp4 audio
  187. '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
  188. '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50},
  189. '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50},
  190. # Dash webm
  191. '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40},
  192. '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40},
  193. '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40},
  194. '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40},
  195. '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40},
  196. '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40},
  197. '242': {'ext': 'webm', 'height': 240, 'resolution': '240p', 'format_note': 'DASH webm', 'preference': -40},
  198. '243': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': 'DASH webm', 'preference': -40},
  199. '244': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
  200. '245': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
  201. '246': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
  202. '247': {'ext': 'webm', 'height': 720, 'resolution': '720p', 'format_note': 'DASH webm', 'preference': -40},
  203. '248': {'ext': 'webm', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH webm', 'preference': -40},
  204. # Dash webm audio
  205. '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 48, 'preference': -50},
  206. '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 256, 'preference': -50},
  207. # RTMP (unnamed)
  208. '_rtmp': {'protocol': 'rtmp'},
  209. }
  210. IE_NAME = u'youtube'
  211. _TESTS = [
  212. {
  213. u"url": u"http://www.youtube.com/watch?v=BaW_jenozKc",
  214. u"file": u"BaW_jenozKc.mp4",
  215. u"info_dict": {
  216. u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
  217. u"uploader": u"Philipp Hagemeister",
  218. u"uploader_id": u"phihag",
  219. u"upload_date": u"20121002",
  220. u"description": u"test chars: \"'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
  221. }
  222. },
  223. {
  224. u"url": u"http://www.youtube.com/watch?v=UxxajLWwzqY",
  225. u"file": u"UxxajLWwzqY.mp4",
  226. u"note": u"Test generic use_cipher_signature video (#897)",
  227. u"info_dict": {
  228. u"upload_date": u"20120506",
  229. u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
  230. u"description": u"md5:5b292926389560516e384ac437c0ec07",
  231. u"uploader": u"Icona Pop",
  232. u"uploader_id": u"IconaPop"
  233. }
  234. },
  235. {
  236. u"url": u"https://www.youtube.com/watch?v=07FYdnEawAQ",
  237. u"file": u"07FYdnEawAQ.mp4",
  238. u"note": u"Test VEVO video with age protection (#956)",
  239. u"info_dict": {
  240. u"upload_date": u"20130703",
  241. u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
  242. u"description": u"md5:64249768eec3bc4276236606ea996373",
  243. u"uploader": u"justintimberlakeVEVO",
  244. u"uploader_id": u"justintimberlakeVEVO"
  245. }
  246. },
  247. {
  248. u"url": u"//www.YouTube.com/watch?v=yZIXLfi8CZQ",
  249. u"file": u"yZIXLfi8CZQ.mp4",
  250. u"note": u"Embed-only video (#1746)",
  251. u"info_dict": {
  252. u"upload_date": u"20120608",
  253. u"title": u"Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012",
  254. u"description": u"md5:09b78bd971f1e3e289601dfba15ca4f7",
  255. u"uploader": u"SET India",
  256. u"uploader_id": u"setindia"
  257. }
  258. },
  259. {
  260. u"url": u"http://www.youtube.com/watch?v=a9LDPn-MO4I",
  261. u"file": u"a9LDPn-MO4I.m4a",
  262. u"note": u"256k DASH audio (format 141) via DASH manifest",
  263. u"info_dict": {
  264. u"upload_date": "20121002",
  265. u"uploader_id": "8KVIDEO",
  266. u"description": "No description available.",
  267. u"uploader": "8KVIDEO",
  268. u"title": "UHDTV TEST 8K VIDEO.mp4"
  269. },
  270. u"params": {
  271. u"youtube_include_dash_manifest": True,
  272. u"format": "141",
  273. },
  274. },
  275. ]
  276. @classmethod
  277. def suitable(cls, url):
  278. """Receives a URL and returns True if suitable for this IE."""
  279. if YoutubePlaylistIE.suitable(url): return False
  280. return re.match(cls._VALID_URL, url) is not None
  281. def __init__(self, *args, **kwargs):
  282. super(YoutubeIE, self).__init__(*args, **kwargs)
  283. self._player_cache = {}
  284. def report_video_info_webpage_download(self, video_id):
  285. """Report attempt to download video info webpage."""
  286. self.to_screen(u'%s: Downloading video info webpage' % video_id)
  287. def report_information_extraction(self, video_id):
  288. """Report attempt to extract video information."""
  289. self.to_screen(u'%s: Extracting video information' % video_id)
  290. def report_unavailable_format(self, video_id, format):
  291. """Report extracted video URL."""
  292. self.to_screen(u'%s: Format %s not available' % (video_id, format))
  293. def report_rtmp_download(self):
  294. """Indicate the download will use the RTMP protocol."""
  295. self.to_screen(u'RTMP download detected')
  296. def _extract_signature_function(self, video_id, player_url, slen):
  297. id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$',
  298. player_url)
  299. player_type = id_m.group('ext')
  300. player_id = id_m.group('id')
  301. # Read from filesystem cache
  302. func_id = '%s_%s_%d' % (player_type, player_id, slen)
  303. assert os.path.basename(func_id) == func_id
  304. cache_dir = get_cachedir(self._downloader.params)
  305. cache_enabled = cache_dir is not None
  306. if cache_enabled:
  307. cache_fn = os.path.join(os.path.expanduser(cache_dir),
  308. u'youtube-sigfuncs',
  309. func_id + '.json')
  310. try:
  311. with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
  312. cache_spec = json.load(cachef)
  313. return lambda s: u''.join(s[i] for i in cache_spec)
  314. except IOError:
  315. pass # No cache available
  316. if player_type == 'js':
  317. code = self._download_webpage(
  318. player_url, video_id,
  319. note=u'Downloading %s player %s' % (player_type, player_id),
  320. errnote=u'Download of %s failed' % player_url)
  321. res = self._parse_sig_js(code)
  322. elif player_type == 'swf':
  323. urlh = self._request_webpage(
  324. player_url, video_id,
  325. note=u'Downloading %s player %s' % (player_type, player_id),
  326. errnote=u'Download of %s failed' % player_url)
  327. code = urlh.read()
  328. res = self._parse_sig_swf(code)
  329. else:
  330. assert False, 'Invalid player type %r' % player_type
  331. if cache_enabled:
  332. try:
  333. test_string = u''.join(map(compat_chr, range(slen)))
  334. cache_res = res(test_string)
  335. cache_spec = [ord(c) for c in cache_res]
  336. try:
  337. os.makedirs(os.path.dirname(cache_fn))
  338. except OSError as ose:
  339. if ose.errno != errno.EEXIST:
  340. raise
  341. write_json_file(cache_spec, cache_fn)
  342. except Exception:
  343. tb = traceback.format_exc()
  344. self._downloader.report_warning(
  345. u'Writing cache to %r failed: %s' % (cache_fn, tb))
  346. return res
  347. def _print_sig_code(self, func, slen):
  348. def gen_sig_code(idxs):
  349. def _genslice(start, end, step):
  350. starts = u'' if start == 0 else str(start)
  351. ends = (u':%d' % (end+step)) if end + step >= 0 else u':'
  352. steps = u'' if step == 1 else (u':%d' % step)
  353. return u's[%s%s%s]' % (starts, ends, steps)
  354. step = None
  355. start = '(Never used)' # Quelch pyflakes warnings - start will be
  356. # set as soon as step is set
  357. for i, prev in zip(idxs[1:], idxs[:-1]):
  358. if step is not None:
  359. if i - prev == step:
  360. continue
  361. yield _genslice(start, prev, step)
  362. step = None
  363. continue
  364. if i - prev in [-1, 1]:
  365. step = i - prev
  366. start = prev
  367. continue
  368. else:
  369. yield u's[%d]' % prev
  370. if step is None:
  371. yield u's[%d]' % i
  372. else:
  373. yield _genslice(start, i, step)
  374. test_string = u''.join(map(compat_chr, range(slen)))
  375. cache_res = func(test_string)
  376. cache_spec = [ord(c) for c in cache_res]
  377. expr_code = u' + '.join(gen_sig_code(cache_spec))
  378. code = u'if len(s) == %d:\n return %s\n' % (slen, expr_code)
  379. self.to_screen(u'Extracted signature function:\n' + code)
  380. def _parse_sig_js(self, jscode):
  381. funcname = self._search_regex(
  382. r'signature=([a-zA-Z]+)', jscode,
  383. u'Initial JS player signature function name')
  384. functions = {}
  385. def argidx(varname):
  386. return string.lowercase.index(varname)
  387. def interpret_statement(stmt, local_vars, allow_recursion=20):
  388. if allow_recursion < 0:
  389. raise ExtractorError(u'Recursion limit reached')
  390. if stmt.startswith(u'var '):
  391. stmt = stmt[len(u'var '):]
  392. ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
  393. r'=(?P<expr>.*)$', stmt)
  394. if ass_m:
  395. if ass_m.groupdict().get('index'):
  396. def assign(val):
  397. lvar = local_vars[ass_m.group('out')]
  398. idx = interpret_expression(ass_m.group('index'),
  399. local_vars, allow_recursion)
  400. assert isinstance(idx, int)
  401. lvar[idx] = val
  402. return val
  403. expr = ass_m.group('expr')
  404. else:
  405. def assign(val):
  406. local_vars[ass_m.group('out')] = val
  407. return val
  408. expr = ass_m.group('expr')
  409. elif stmt.startswith(u'return '):
  410. assign = lambda v: v
  411. expr = stmt[len(u'return '):]
  412. else:
  413. raise ExtractorError(
  414. u'Cannot determine left side of statement in %r' % stmt)
  415. v = interpret_expression(expr, local_vars, allow_recursion)
  416. return assign(v)
  417. def interpret_expression(expr, local_vars, allow_recursion):
  418. if expr.isdigit():
  419. return int(expr)
  420. if expr.isalpha():
  421. return local_vars[expr]
  422. m = re.match(r'^(?P<in>[a-z]+)\.(?P<member>.*)$', expr)
  423. if m:
  424. member = m.group('member')
  425. val = local_vars[m.group('in')]
  426. if member == 'split("")':
  427. return list(val)
  428. if member == 'join("")':
  429. return u''.join(val)
  430. if member == 'length':
  431. return len(val)
  432. if member == 'reverse()':
  433. return val[::-1]
  434. slice_m = re.match(r'slice\((?P<idx>.*)\)', member)
  435. if slice_m:
  436. idx = interpret_expression(
  437. slice_m.group('idx'), local_vars, allow_recursion-1)
  438. return val[idx:]
  439. m = re.match(
  440. r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
  441. if m:
  442. val = local_vars[m.group('in')]
  443. idx = interpret_expression(m.group('idx'), local_vars,
  444. allow_recursion-1)
  445. return val[idx]
  446. m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
  447. if m:
  448. a = interpret_expression(m.group('a'),
  449. local_vars, allow_recursion)
  450. b = interpret_expression(m.group('b'),
  451. local_vars, allow_recursion)
  452. return a % b
  453. m = re.match(
  454. r'^(?P<func>[a-zA-Z$]+)\((?P<args>[a-z0-9,]+)\)$', expr)
  455. if m:
  456. fname = m.group('func')
  457. if fname not in functions:
  458. functions[fname] = extract_function(fname)
  459. argvals = [int(v) if v.isdigit() else local_vars[v]
  460. for v in m.group('args').split(',')]
  461. return functions[fname](argvals)
  462. raise ExtractorError(u'Unsupported JS expression %r' % expr)
  463. def extract_function(funcname):
  464. func_m = re.search(
  465. r'function ' + re.escape(funcname) +
  466. r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
  467. jscode)
  468. argnames = func_m.group('args').split(',')
  469. def resf(args):
  470. local_vars = dict(zip(argnames, args))
  471. for stmt in func_m.group('code').split(';'):
  472. res = interpret_statement(stmt, local_vars)
  473. return res
  474. return resf
  475. initial_function = extract_function(funcname)
  476. return lambda s: initial_function([s])
  477. def _parse_sig_swf(self, file_contents):
  478. if file_contents[1:3] != b'WS':
  479. raise ExtractorError(
  480. u'Not an SWF file; header is %r' % file_contents[:3])
  481. if file_contents[:1] == b'C':
  482. content = zlib.decompress(file_contents[8:])
  483. else:
  484. raise NotImplementedError(u'Unsupported compression format %r' %
  485. file_contents[:1])
  486. def extract_tags(content):
  487. pos = 0
  488. while pos < len(content):
  489. header16 = struct.unpack('<H', content[pos:pos+2])[0]
  490. pos += 2
  491. tag_code = header16 >> 6
  492. tag_len = header16 & 0x3f
  493. if tag_len == 0x3f:
  494. tag_len = struct.unpack('<I', content[pos:pos+4])[0]
  495. pos += 4
  496. assert pos+tag_len <= len(content)
  497. yield (tag_code, content[pos:pos+tag_len])
  498. pos += tag_len
  499. code_tag = next(tag
  500. for tag_code, tag in extract_tags(content)
  501. if tag_code == 82)
  502. p = code_tag.index(b'\0', 4) + 1
  503. code_reader = io.BytesIO(code_tag[p:])
  504. # Parse ABC (AVM2 ByteCode)
  505. def read_int(reader=None):
  506. if reader is None:
  507. reader = code_reader
  508. res = 0
  509. shift = 0
  510. for _ in range(5):
  511. buf = reader.read(1)
  512. assert len(buf) == 1
  513. b = struct.unpack('<B', buf)[0]
  514. res = res | ((b & 0x7f) << shift)
  515. if b & 0x80 == 0:
  516. break
  517. shift += 7
  518. return res
  519. def u30(reader=None):
  520. res = read_int(reader)
  521. assert res & 0xf0000000 == 0
  522. return res
  523. u32 = read_int
  524. def s32(reader=None):
  525. v = read_int(reader)
  526. if v & 0x80000000 != 0:
  527. v = - ((v ^ 0xffffffff) + 1)
  528. return v
  529. def read_string(reader=None):
  530. if reader is None:
  531. reader = code_reader
  532. slen = u30(reader)
  533. resb = reader.read(slen)
  534. assert len(resb) == slen
  535. return resb.decode('utf-8')
  536. def read_bytes(count, reader=None):
  537. if reader is None:
  538. reader = code_reader
  539. resb = reader.read(count)
  540. assert len(resb) == count
  541. return resb
  542. def read_byte(reader=None):
  543. resb = read_bytes(1, reader=reader)
  544. res = struct.unpack('<B', resb)[0]
  545. return res
  546. # minor_version + major_version
  547. read_bytes(2 + 2)
  548. # Constant pool
  549. int_count = u30()
  550. for _c in range(1, int_count):
  551. s32()
  552. uint_count = u30()
  553. for _c in range(1, uint_count):
  554. u32()
  555. double_count = u30()
  556. read_bytes((double_count-1) * 8)
  557. string_count = u30()
  558. constant_strings = [u'']
  559. for _c in range(1, string_count):
  560. s = read_string()
  561. constant_strings.append(s)
  562. namespace_count = u30()
  563. for _c in range(1, namespace_count):
  564. read_bytes(1) # kind
  565. u30() # name
  566. ns_set_count = u30()
  567. for _c in range(1, ns_set_count):
  568. count = u30()
  569. for _c2 in range(count):
  570. u30()
  571. multiname_count = u30()
  572. MULTINAME_SIZES = {
  573. 0x07: 2, # QName
  574. 0x0d: 2, # QNameA
  575. 0x0f: 1, # RTQName
  576. 0x10: 1, # RTQNameA
  577. 0x11: 0, # RTQNameL
  578. 0x12: 0, # RTQNameLA
  579. 0x09: 2, # Multiname
  580. 0x0e: 2, # MultinameA
  581. 0x1b: 1, # MultinameL
  582. 0x1c: 1, # MultinameLA
  583. }
  584. multinames = [u'']
  585. for _c in range(1, multiname_count):
  586. kind = u30()
  587. assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind
  588. if kind == 0x07:
  589. u30() # namespace_idx
  590. name_idx = u30()
  591. multinames.append(constant_strings[name_idx])
  592. else:
  593. multinames.append('[MULTINAME kind: %d]' % kind)
  594. for _c2 in range(MULTINAME_SIZES[kind]):
  595. u30()
  596. # Methods
  597. method_count = u30()
  598. MethodInfo = collections.namedtuple(
  599. 'MethodInfo',
  600. ['NEED_ARGUMENTS', 'NEED_REST'])
  601. method_infos = []
  602. for method_id in range(method_count):
  603. param_count = u30()
  604. u30() # return type
  605. for _ in range(param_count):
  606. u30() # param type
  607. u30() # name index (always 0 for youtube)
  608. flags = read_byte()
  609. if flags & 0x08 != 0:
  610. # Options present
  611. option_count = u30()
  612. for c in range(option_count):
  613. u30() # val
  614. read_bytes(1) # kind
  615. if flags & 0x80 != 0:
  616. # Param names present
  617. for _ in range(param_count):
  618. u30() # param name
  619. mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
  620. method_infos.append(mi)
  621. # Metadata
  622. metadata_count = u30()
  623. for _c in range(metadata_count):
  624. u30() # name
  625. item_count = u30()
  626. for _c2 in range(item_count):
  627. u30() # key
  628. u30() # value
  629. def parse_traits_info():
  630. trait_name_idx = u30()
  631. kind_full = read_byte()
  632. kind = kind_full & 0x0f
  633. attrs = kind_full >> 4
  634. methods = {}
  635. if kind in [0x00, 0x06]: # Slot or Const
  636. u30() # Slot id
  637. u30() # type_name_idx
  638. vindex = u30()
  639. if vindex != 0:
  640. read_byte() # vkind
  641. elif kind in [0x01, 0x02, 0x03]: # Method / Getter / Setter
  642. u30() # disp_id
  643. method_idx = u30()
  644. methods[multinames[trait_name_idx]] = method_idx
  645. elif kind == 0x04: # Class
  646. u30() # slot_id
  647. u30() # classi
  648. elif kind == 0x05: # Function
  649. u30() # slot_id
  650. function_idx = u30()
  651. methods[function_idx] = multinames[trait_name_idx]
  652. else:
  653. raise ExtractorError(u'Unsupported trait kind %d' % kind)
  654. if attrs & 0x4 != 0: # Metadata present
  655. metadata_count = u30()
  656. for _c3 in range(metadata_count):
  657. u30() # metadata index
  658. return methods
  659. # Classes
  660. TARGET_CLASSNAME = u'SignatureDecipher'
  661. searched_idx = multinames.index(TARGET_CLASSNAME)
  662. searched_class_id = None
  663. class_count = u30()
  664. for class_id in range(class_count):
  665. name_idx = u30()
  666. if name_idx == searched_idx:
  667. # We found the class we're looking for!
  668. searched_class_id = class_id
  669. u30() # super_name idx
  670. flags = read_byte()
  671. if flags & 0x08 != 0: # Protected namespace is present
  672. u30() # protected_ns_idx
  673. intrf_count = u30()
  674. for _c2 in range(intrf_count):
  675. u30()
  676. u30() # iinit
  677. trait_count = u30()
  678. for _c2 in range(trait_count):
  679. parse_traits_info()
  680. if searched_class_id is None:
  681. raise ExtractorError(u'Target class %r not found' %
  682. TARGET_CLASSNAME)
  683. method_names = {}
  684. method_idxs = {}
  685. for class_id in range(class_count):
  686. u30() # cinit
  687. trait_count = u30()
  688. for _c2 in range(trait_count):
  689. trait_methods = parse_traits_info()
  690. if class_id == searched_class_id:
  691. method_names.update(trait_methods.items())
  692. method_idxs.update(dict(
  693. (idx, name)
  694. for name, idx in trait_methods.items()))
  695. # Scripts
  696. script_count = u30()
  697. for _c in range(script_count):
  698. u30() # init
  699. trait_count = u30()
  700. for _c2 in range(trait_count):
  701. parse_traits_info()
  702. # Method bodies
  703. method_body_count = u30()
  704. Method = collections.namedtuple('Method', ['code', 'local_count'])
  705. methods = {}
  706. for _c in range(method_body_count):
  707. method_idx = u30()
  708. u30() # max_stack
  709. local_count = u30()
  710. u30() # init_scope_depth
  711. u30() # max_scope_depth
  712. code_length = u30()
  713. code = read_bytes(code_length)
  714. if method_idx in method_idxs:
  715. m = Method(code, local_count)
  716. methods[method_idxs[method_idx]] = m
  717. exception_count = u30()
  718. for _c2 in range(exception_count):
  719. u30() # from
  720. u30() # to
  721. u30() # target
  722. u30() # exc_type
  723. u30() # var_name
  724. trait_count = u30()
  725. for _c2 in range(trait_count):
  726. parse_traits_info()
  727. assert p + code_reader.tell() == len(code_tag)
  728. assert len(methods) == len(method_idxs)
  729. method_pyfunctions = {}
  730. def extract_function(func_name):
  731. if func_name in method_pyfunctions:
  732. return method_pyfunctions[func_name]
  733. if func_name not in methods:
  734. raise ExtractorError(u'Cannot find function %r' % func_name)
  735. m = methods[func_name]
  736. def resfunc(args):
  737. registers = ['(this)'] + list(args) + [None] * m.local_count
  738. stack = []
  739. coder = io.BytesIO(m.code)
  740. while True:
  741. opcode = struct.unpack('!B', coder.read(1))[0]
  742. if opcode == 36: # pushbyte
  743. v = struct.unpack('!B', coder.read(1))[0]
  744. stack.append(v)
  745. elif opcode == 44: # pushstring
  746. idx = u30(coder)
  747. stack.append(constant_strings[idx])
  748. elif opcode == 48: # pushscope
  749. # We don't implement the scope register, so we'll just
  750. # ignore the popped value
  751. stack.pop()
  752. elif opcode == 70: # callproperty
  753. index = u30(coder)
  754. mname = multinames[index]
  755. arg_count = u30(coder)
  756. args = list(reversed(
  757. [stack.pop() for _ in range(arg_count)]))
  758. obj = stack.pop()
  759. if mname == u'split':
  760. assert len(args) == 1
  761. assert isinstance(args[0], compat_str)
  762. assert isinstance(obj, compat_str)
  763. if args[0] == u'':
  764. res = list(obj)
  765. else:
  766. res = obj.split(args[0])
  767. stack.append(res)
  768. elif mname == u'slice':
  769. assert len(args) == 1
  770. assert isinstance(args[0], int)
  771. assert isinstance(obj, list)
  772. res = obj[args[0]:]
  773. stack.append(res)
  774. elif mname == u'join':
  775. assert len(args) == 1
  776. assert isinstance(args[0], compat_str)
  777. assert isinstance(obj, list)
  778. res = args[0].join(obj)
  779. stack.append(res)
  780. elif mname in method_pyfunctions:
  781. stack.append(method_pyfunctions[mname](args))
  782. else:
  783. raise NotImplementedError(
  784. u'Unsupported property %r on %r'
  785. % (mname, obj))
  786. elif opcode == 72: # returnvalue
  787. res = stack.pop()
  788. return res
  789. elif opcode == 79: # callpropvoid
  790. index = u30(coder)
  791. mname = multinames[index]
  792. arg_count = u30(coder)
  793. args = list(reversed(
  794. [stack.pop() for _ in range(arg_count)]))
  795. obj = stack.pop()
  796. if mname == u'reverse':
  797. assert isinstance(obj, list)
  798. obj.reverse()
  799. else:
  800. raise NotImplementedError(
  801. u'Unsupported (void) property %r on %r'
  802. % (mname, obj))
  803. elif opcode == 93: # findpropstrict
  804. index = u30(coder)
  805. mname = multinames[index]
  806. res = extract_function(mname)
  807. stack.append(res)
  808. elif opcode == 97: # setproperty
  809. index = u30(coder)
  810. value = stack.pop()
  811. idx = stack.pop()
  812. obj = stack.pop()
  813. assert isinstance(obj, list)
  814. assert isinstance(idx, int)
  815. obj[idx] = value
  816. elif opcode == 98: # getlocal
  817. index = u30(coder)
  818. stack.append(registers[index])
  819. elif opcode == 99: # setlocal
  820. index = u30(coder)
  821. value = stack.pop()
  822. registers[index] = value
  823. elif opcode == 102: # getproperty
  824. index = u30(coder)
  825. pname = multinames[index]
  826. if pname == u'length':
  827. obj = stack.pop()
  828. assert isinstance(obj, list)
  829. stack.append(len(obj))
  830. else: # Assume attribute access
  831. idx = stack.pop()
  832. assert isinstance(idx, int)
  833. obj = stack.pop()
  834. assert isinstance(obj, list)
  835. stack.append(obj[idx])
  836. elif opcode == 128: # coerce
  837. u30(coder)
  838. elif opcode == 133: # coerce_s
  839. assert isinstance(stack[-1], (type(None), compat_str))
  840. elif opcode == 164: # modulo
  841. value2 = stack.pop()
  842. value1 = stack.pop()
  843. res = value1 % value2
  844. stack.append(res)
  845. elif opcode == 208: # getlocal_0
  846. stack.append(registers[0])
  847. elif opcode == 209: # getlocal_1
  848. stack.append(registers[1])
  849. elif opcode == 210: # getlocal_2
  850. stack.append(registers[2])
  851. elif opcode == 211: # getlocal_3
  852. stack.append(registers[3])
  853. elif opcode == 214: # setlocal_2
  854. registers[2] = stack.pop()
  855. elif opcode == 215: # setlocal_3
  856. registers[3] = stack.pop()
  857. else:
  858. raise NotImplementedError(
  859. u'Unsupported opcode %d' % opcode)
  860. method_pyfunctions[func_name] = resfunc
  861. return resfunc
  862. initial_function = extract_function(u'decipher')
  863. return lambda s: initial_function([s])
  864. def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
  865. """Turn the encrypted s field into a working signature"""
  866. if player_url is not None:
  867. if player_url.startswith(u'//'):
  868. player_url = u'https:' + player_url
  869. try:
  870. player_id = (player_url, len(s))
  871. if player_id not in self._player_cache:
  872. func = self._extract_signature_function(
  873. video_id, player_url, len(s)
  874. )
  875. self._player_cache[player_id] = func
  876. func = self._player_cache[player_id]
  877. if self._downloader.params.get('youtube_print_sig_code'):
  878. self._print_sig_code(func, len(s))
  879. return func(s)
  880. except Exception:
  881. tb = traceback.format_exc()
  882. self._downloader.report_warning(
  883. u'Automatic signature extraction failed: ' + tb)
  884. self._downloader.report_warning(
  885. u'Warning: Falling back to static signature algorithm')
  886. return self._static_decrypt_signature(
  887. s, video_id, player_url, age_gate)
  888. def _static_decrypt_signature(self, s, video_id, player_url, age_gate):
  889. if age_gate:
  890. # The videos with age protection use another player, so the
  891. # algorithms can be different.
  892. if len(s) == 86:
  893. return s[2:63] + s[82] + s[64:82] + s[63]
  894. if len(s) == 93:
  895. return s[86:29:-1] + s[88] + s[28:5:-1]
  896. elif len(s) == 92:
  897. return s[25] + s[3:25] + s[0] + s[26:42] + s[79] + s[43:79] + s[91] + s[80:83]
  898. elif len(s) == 91:
  899. return s[84:27:-1] + s[86] + s[26:5:-1]
  900. elif len(s) == 90:
  901. return s[25] + s[3:25] + s[2] + s[26:40] + s[77] + s[41:77] + s[89] + s[78:81]
  902. elif len(s) == 89:
  903. return s[84:78:-1] + s[87] + s[77:60:-1] + s[0] + s[59:3:-1]
  904. elif len(s) == 88:
  905. return s[7:28] + s[87] + s[29:45] + s[55] + s[46:55] + s[2] + s[56:87] + s[28]
  906. elif len(s) == 87:
  907. return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:]
  908. elif len(s) == 86:
  909. return s[80:72:-1] + s[16] + s[71:39:-1] + s[72] + s[38:16:-1] + s[82] + s[15::-1]
  910. elif len(s) == 85:
  911. return s[3:11] + s[0] + s[12:55] + s[84] + s[56:84]
  912. elif len(s) == 84:
  913. return s[78:70:-1] + s[14] + s[69:37:-1] + s[70] + s[36:14:-1] + s[80] + s[:14][::-1]
  914. elif len(s) == 83:
  915. return s[80:63:-1] + s[0] + s[62:0:-1] + s[63]
  916. elif len(s) == 82:
  917. return s[80:37:-1] + s[7] + s[36:7:-1] + s[0] + s[6:0:-1] + s[37]
  918. elif len(s) == 81:
  919. return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
  920. elif len(s) == 80:
  921. return s[1:19] + s[0] + s[20:68] + s[19] + s[69:80]
  922. elif len(s) == 79:
  923. return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
  924. else:
  925. raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
  926. def _get_available_subtitles(self, video_id, webpage):
  927. try:
  928. sub_list = self._download_webpage(
  929. 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
  930. video_id, note=False)
  931. except ExtractorError as err:
  932. self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
  933. return {}
  934. lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
  935. sub_lang_list = {}
  936. for l in lang_list:
  937. lang = l[1]
  938. params = compat_urllib_parse.urlencode({
  939. 'lang': lang,
  940. 'v': video_id,
  941. 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
  942. 'name': unescapeHTML(l[0]).encode('utf-8'),
  943. })
  944. url = u'https://www.youtube.com/api/timedtext?' + params
  945. sub_lang_list[lang] = url
  946. if not sub_lang_list:
  947. self._downloader.report_warning(u'video doesn\'t have subtitles')
  948. return {}
  949. return sub_lang_list
  950. def _get_available_automatic_caption(self, video_id, webpage):
  951. """We need the webpage for getting the captions url, pass it as an
  952. argument to speed up the process."""
  953. sub_format = self._downloader.params.get('subtitlesformat', 'srt')
  954. self.to_screen(u'%s: Looking for automatic captions' % video_id)
  955. mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
  956. err_msg = u'Couldn\'t find automatic captions for %s' % video_id
  957. if mobj is None:
  958. self._downloader.report_warning(err_msg)
  959. return {}
  960. player_config = json.loads(mobj.group(1))
  961. try:
  962. args = player_config[u'args']
  963. caption_url = args[u'ttsurl']
  964. timestamp = args[u'timestamp']
  965. # We get the available subtitles
  966. list_params = compat_urllib_parse.urlencode({
  967. 'type': 'list',
  968. 'tlangs': 1,
  969. 'asrs': 1,
  970. })
  971. list_url = caption_url + '&' + list_params
  972. caption_list = self._download_xml(list_url, video_id)
  973. original_lang_node = caption_list.find('track')
  974. if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' :
  975. self._downloader.report_warning(u'Video doesn\'t have automatic captions')
  976. return {}
  977. original_lang = original_lang_node.attrib['lang_code']
  978. sub_lang_list = {}
  979. for lang_node in caption_list.findall('target'):
  980. sub_lang = lang_node.attrib['lang_code']
  981. params = compat_urllib_parse.urlencode({
  982. 'lang': original_lang,
  983. 'tlang': sub_lang,
  984. 'fmt': sub_format,
  985. 'ts': timestamp,
  986. 'kind': 'asr',
  987. })
  988. sub_lang_list[sub_lang] = caption_url + '&' + params
  989. return sub_lang_list
  990. # An extractor error can be raise by the download process if there are
  991. # no automatic captions but there are subtitles
  992. except (KeyError, ExtractorError):
  993. self._downloader.report_warning(err_msg)
  994. return {}
  995. @classmethod
  996. def extract_id(cls, url):
  997. mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
  998. if mobj is None:
  999. raise ExtractorError(u'Invalid URL: %s' % url)
  1000. video_id = mobj.group(2)
  1001. return video_id
  1002. def _extract_from_m3u8(self, manifest_url, video_id):
  1003. url_map = {}
  1004. def _get_urls(_manifest):
  1005. lines = _manifest.split('\n')
  1006. urls = filter(lambda l: l and not l.startswith('#'),
  1007. lines)
  1008. return urls
  1009. manifest = self._download_webpage(manifest_url, video_id, u'Downloading formats manifest')
  1010. formats_urls = _get_urls(manifest)
  1011. for format_url in formats_urls:
  1012. itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
  1013. url_map[itag] = format_url
  1014. return url_map
  1015. def _extract_annotations(self, video_id):
  1016. url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
  1017. return self._download_webpage(url, video_id, note=u'Searching for annotations.', errnote=u'Unable to download video annotations.')
  1018. def _real_extract(self, url):
  1019. # Extract original video URL from URL with redirection, like age verification, using next_url parameter
  1020. mobj = re.search(self._NEXT_URL_RE, url)
  1021. if mobj:
  1022. url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
  1023. video_id = self.extract_id(url)
  1024. # Get video webpage
  1025. url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
  1026. video_webpage = self._download_webpage(url, video_id)
  1027. # Attempt to extract SWF player URL
  1028. mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
  1029. if mobj is not None:
  1030. player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
  1031. else:
  1032. player_url = None
  1033. # Get video info
  1034. self.report_video_info_webpage_download(video_id)
  1035. if re.search(r'player-age-gate-content">', video_webpage) is not None:
  1036. self.report_age_confirmation()
  1037. age_gate = True
  1038. # We simulate the access to the video from www.youtube.com/v/{video_id}
  1039. # this can be viewed without login into Youtube
  1040. data = compat_urllib_parse.urlencode({'video_id': video_id,
  1041. 'el': 'player_embedded',
  1042. 'gl': 'US',
  1043. 'hl': 'en',
  1044. 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
  1045. 'asv': 3,
  1046. 'sts':'1588',
  1047. })
  1048. video_info_url = 'https://www.youtube.com/get_video_info?' + data
  1049. video_info_webpage = self._download_webpage(video_info_url, video_id,
  1050. note=False,
  1051. errnote='unable to download video info webpage')
  1052. video_info = compat_parse_qs(video_info_webpage)
  1053. else:
  1054. age_gate = False
  1055. for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
  1056. video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
  1057. % (video_id, el_type))
  1058. video_info_webpage = self._download_webpage(video_info_url, video_id,
  1059. note=False,
  1060. errnote='unable to download video info webpage')
  1061. video_info = compat_parse_qs(video_info_webpage)
  1062. if 'token' in video_info:
  1063. break
  1064. if 'token' not in video_info:
  1065. if 'reason' in video_info:
  1066. raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0], expected=True)
  1067. else:
  1068. raise ExtractorError(u'"token" parameter not in video info for unknown reason')
  1069. if 'view_count' in video_info:
  1070. view_count = int(video_info['view_count'][0])
  1071. else:
  1072. view_count = None
  1073. # Check for "rental" videos
  1074. if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
  1075. raise ExtractorError(u'"rental" videos not supported')
  1076. # Start extracting information
  1077. self.report_information_extraction(video_id)
  1078. # uploader
  1079. if 'author' not in video_info:
  1080. raise ExtractorError(u'Unable to extract uploader name')
  1081. video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
  1082. # uploader_id
  1083. video_uploader_id = None
  1084. mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
  1085. if mobj is not None:
  1086. video_uploader_id = mobj.group(1)
  1087. else:
  1088. self._downloader.report_warning(u'unable to extract uploader nickname')
  1089. # title
  1090. if 'title' in video_info:
  1091. video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
  1092. else:
  1093. self._downloader.report_warning(u'Unable to extract video title')
  1094. video_title = u'_'
  1095. # thumbnail image
  1096. # We try first to get a high quality image:
  1097. m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
  1098. video_webpage, re.DOTALL)
  1099. if m_thumb is not None:
  1100. video_thumbnail = m_thumb.group(1)
  1101. elif 'thumbnail_url' not in video_info:
  1102. self._downloader.report_warning(u'unable to extract video thumbnail')
  1103. video_thumbnail = None
  1104. else: # don't panic if we can't find it
  1105. video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
  1106. # upload date
  1107. upload_date = None
  1108. mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
  1109. if mobj is not None:
  1110. upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
  1111. upload_date = unified_strdate(upload_date)
  1112. # description
  1113. video_description = get_element_by_id("eow-description", video_webpage)
  1114. if video_description:
  1115. video_description = re.sub(r'''(?x)
  1116. <a\s+
  1117. (?:[a-zA-Z-]+="[^"]+"\s+)*?
  1118. title="([^"]+)"\s+
  1119. (?:[a-zA-Z-]+="[^"]+"\s+)*?
  1120. class="yt-uix-redirect-link"\s*>
  1121. [^<]+
  1122. </a>
  1123. ''', r'\1', video_description)
  1124. video_description = clean_html(video_description)
  1125. else:
  1126. fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
  1127. if fd_mobj:
  1128. video_description = unescapeHTML(fd_mobj.group(1))
  1129. else:
  1130. video_description = u''
  1131. def _extract_count(klass):
  1132. count = self._search_regex(
  1133. r'class="%s">([\d,]+)</span>' % re.escape(klass),
  1134. video_webpage, klass, default=None)
  1135. if count is not None:
  1136. return int(count.replace(',', ''))
  1137. return None
  1138. like_count = _extract_count(u'likes-count')
  1139. dislike_count = _extract_count(u'dislikes-count')
  1140. # subtitles
  1141. video_subtitles = self.extract_subtitles(video_id, video_webpage)
  1142. if self._downloader.params.get('listsubtitles', False):
  1143. self._list_available_subtitles(video_id, video_webpage)
  1144. return
  1145. if 'length_seconds' not in video_info:
  1146. self._downloader.report_warning(u'unable to extract video duration')
  1147. video_duration = None
  1148. else:
  1149. video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
  1150. # annotations
  1151. video_annotations = None
  1152. if self._downloader.params.get('writeannotations', False):
  1153. video_annotations = self._extract_annotations(video_id)
  1154. # Decide which formats to download
  1155. try:
  1156. mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
  1157. if not mobj:
  1158. raise ValueError('Could not find vevo ID')
  1159. info = json.loads(mobj.group(1))
  1160. args = info['args']
  1161. # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
  1162. # this signatures are encrypted
  1163. if 'url_encoded_fmt_stream_map' not in args:
  1164. raise ValueError(u'No stream_map present') # caught below
  1165. re_signature = re.compile(r'[&,]s=')
  1166. m_s = re_signature.search(args['url_encoded_fmt_stream_map'])
  1167. if m_s is not None:
  1168. self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
  1169. video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
  1170. m_s = re_signature.search(args.get('adaptive_fmts', u''))
  1171. if m_s is not None:
  1172. if 'adaptive_fmts' in video_info:
  1173. video_info['adaptive_fmts'][0] += ',' + args['adaptive_fmts']
  1174. else:
  1175. video_info['adaptive_fmts'] = [args['adaptive_fmts']]
  1176. except ValueError:
  1177. pass
  1178. def _map_to_format_list(urlmap):
  1179. formats = []
  1180. for itag, video_real_url in urlmap.items():
  1181. dct = {
  1182. 'format_id': itag,
  1183. 'url': video_real_url,
  1184. 'player_url': player_url,
  1185. }
  1186. if itag in self._formats:
  1187. dct.update(self._formats[itag])
  1188. formats.append(dct)
  1189. return formats
  1190. if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
  1191. self.report_rtmp_download()
  1192. formats = [{
  1193. 'format_id': '_rtmp',
  1194. 'protocol': 'rtmp',
  1195. 'url': video_info['conn'][0],
  1196. 'player_url': player_url,
  1197. }]
  1198. elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
  1199. encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0]
  1200. if 'rtmpe%3Dyes' in encoded_url_map:
  1201. raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
  1202. url_map = {}
  1203. for url_data_str in encoded_url_map.split(','):
  1204. url_data = compat_parse_qs(url_data_str)
  1205. if 'itag' in url_data and 'url' in url_data:
  1206. url = url_data['url'][0]
  1207. if 'sig' in url_data:
  1208. url += '&signature=' + url_data['sig'][0]
  1209. elif 's' in url_data:
  1210. encrypted_sig = url_data['s'][0]
  1211. if self._downloader.params.get('verbose'):
  1212. if age_gate:
  1213. if player_url is None:
  1214. player_version = 'unknown'
  1215. else:
  1216. player_version = self._search_regex(
  1217. r'-(.+)\.swf$', player_url,
  1218. u'flash player', fatal=False)
  1219. player_desc = 'flash player %s' % player_version
  1220. else:
  1221. player_version = self._search_regex(
  1222. r'html5player-(.+?)\.js', video_webpage,
  1223. 'html5 player', fatal=False)
  1224. player_desc = u'html5 player %s' % player_version
  1225. parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.'))
  1226. self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' %
  1227. (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc))
  1228. if not age_gate:
  1229. jsplayer_url_json = self._search_regex(
  1230. r'"assets":.+?"js":\s*("[^"]+")',
  1231. video_webpage, u'JS player URL')
  1232. player_url = json.loads(jsplayer_url_json)
  1233. signature = self._decrypt_signature(
  1234. encrypted_sig, video_id, player_url, age_gate)
  1235. url += '&signature=' + signature
  1236. if 'ratebypass' not in url:
  1237. url += '&ratebypass=yes'
  1238. url_map[url_data['itag'][0]] = url
  1239. formats = _map_to_format_list(url_map)
  1240. elif video_info.get('hlsvp'):
  1241. manifest_url = video_info['hlsvp'][0]
  1242. url_map = self._extract_from_m3u8(manifest_url, video_id)
  1243. formats = _map_to_format_list(url_map)
  1244. else:
  1245. raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
  1246. # Look for the DASH manifest
  1247. dash_manifest_url_lst = video_info.get('dashmpd')
  1248. if (dash_manifest_url_lst and dash_manifest_url_lst[0] and
  1249. self._downloader.params.get('youtube_include_dash_manifest', False)):
  1250. try:
  1251. dash_doc = self._download_xml(
  1252. dash_manifest_url_lst[0], video_id,
  1253. note=u'Downloading DASH manifest',
  1254. errnote=u'Could not download DASH manifest')
  1255. for r in dash_doc.findall(u'.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
  1256. url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
  1257. if url_el is None:
  1258. continue
  1259. format_id = r.attrib['id']
  1260. video_url = url_el.text
  1261. filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
  1262. f = {
  1263. 'format_id': format_id,
  1264. 'url': video_url,
  1265. 'width': int_or_none(r.attrib.get('width')),
  1266. 'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
  1267. 'asr': int_or_none(r.attrib.get('audioSamplingRate')),
  1268. 'filesize': filesize,
  1269. }
  1270. try:
  1271. existing_format = next(
  1272. fo for fo in formats
  1273. if fo['format_id'] == format_id)
  1274. except StopIteration:
  1275. f.update(self._formats.get(format_id, {}))
  1276. formats.append(f)
  1277. else:
  1278. existing_format.update(f)
  1279. except (ExtractorError, KeyError) as e:
  1280. self.report_warning(u'Skipping DASH manifest: %s' % e, video_id)
  1281. self._sort_formats(formats)
  1282. return {
  1283. 'id': video_id,
  1284. 'uploader': video_uploader,
  1285. 'uploader_id': video_uploader_id,
  1286. 'upload_date': upload_date,
  1287. 'title': video_title,
  1288. 'thumbnail': video_thumbnail,
  1289. 'description': video_description,
  1290. 'subtitles': video_subtitles,
  1291. 'duration': video_duration,
  1292. 'age_limit': 18 if age_gate else 0,
  1293. 'annotations': video_annotations,
  1294. 'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id,
  1295. 'view_count': view_count,
  1296. 'like_count': like_count,
  1297. 'dislike_count': dislike_count,
  1298. 'formats': formats,
  1299. }
  1300. class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
  1301. IE_DESC = u'YouTube.com playlists'
  1302. _VALID_URL = r"""(?x)(?:
  1303. (?:https?://)?
  1304. (?:\w+\.)?
  1305. youtube\.com/
  1306. (?:
  1307. (?:course|view_play_list|my_playlists|artist|playlist|watch)
  1308. \? (?:.*?&)*? (?:p|a|list)=
  1309. | p/
  1310. )
  1311. (
  1312. (?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}
  1313. # Top tracks, they can also include dots
  1314. |(?:MC)[\w\.]*
  1315. )
  1316. .*
  1317. |
  1318. ((?:PL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
  1319. )"""
  1320. _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&page=%s'
  1321. _MORE_PAGES_INDICATOR = r'data-link-type="next"'
  1322. _VIDEO_RE = r'href="/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
  1323. IE_NAME = u'youtube:playlist'
  1324. def _real_initialize(self):
  1325. self._login()
  1326. def _ids_to_results(self, ids):
  1327. return [self.url_result(vid_id, 'Youtube', video_id=vid_id)
  1328. for vid_id in ids]
  1329. def _extract_mix(self, playlist_id):
  1330. # The mixes are generated from a a single video
  1331. # the id of the playlist is just 'RD' + video_id
  1332. url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
  1333. webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix')
  1334. title_span = (get_element_by_attribute('class', 'title long-title', webpage) or
  1335. get_element_by_attribute('class', 'title ', webpage))
  1336. title = clean_html(title_span)
  1337. video_re = r'data-index="\d+".*?href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s' % re.escape(playlist_id)
  1338. ids = orderedSet(re.findall(video_re, webpage))
  1339. url_results = self._ids_to_results(ids)
  1340. return self.playlist_result(url_results, playlist_id, title)
  1341. def _real_extract(self, url):
  1342. # Extract playlist id
  1343. mobj = re.match(self._VALID_URL, url)
  1344. if mobj is None:
  1345. raise ExtractorError(u'Invalid URL: %s' % url)
  1346. playlist_id = mobj.group(1) or mobj.group(2)
  1347. # Check if it's a video-specific URL
  1348. query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  1349. if 'v' in query_dict:
  1350. video_id = query_dict['v'][0]
  1351. if self._downloader.params.get('noplaylist'):
  1352. self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
  1353. return self.url_result(video_id, 'Youtube', video_id=video_id)
  1354. else:
  1355. self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id))
  1356. if playlist_id.startswith('RD'):
  1357. # Mixes require a custom extraction process
  1358. return self._extract_mix(playlist_id)
  1359. if playlist_id.startswith('TL'):
  1360. raise ExtractorError(u'For downloading YouTube.com top lists, use '
  1361. u'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
  1362. # Extract the video ids from the playlist pages
  1363. ids = []
  1364. for page_num in itertools.count(1):
  1365. url = self._TEMPLATE_URL % (playlist_id, page_num)
  1366. page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
  1367. matches = re.finditer(self._VIDEO_RE, page)
  1368. # We remove the duplicates and the link with index 0
  1369. # (it's not the first video of the playlist)
  1370. new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
  1371. ids.extend(new_ids)
  1372. if re.search(self._MORE_PAGES_INDICATOR, page) is None:
  1373. break
  1374. try:
  1375. playlist_title = self._og_search_title(page)
  1376. except RegexNotFoundError:
  1377. self.report_warning(
  1378. u'Playlist page is missing OpenGraph title, falling back ...',
  1379. playlist_id)
  1380. playlist_title = self._html_search_regex(
  1381. r'<h1 class="pl-header-title">(.*?)</h1>', page, u'title')
  1382. url_results = self._ids_to_results(ids)
  1383. return self.playlist_result(url_results, playlist_id, playlist_title)
  1384. class YoutubeTopListIE(YoutubePlaylistIE):
  1385. IE_NAME = u'youtube:toplist'
  1386. IE_DESC = (u'YouTube.com top lists, "yttoplist:{channel}:{list title}"'
  1387. u' (Example: "yttoplist:music:Top Tracks")')
  1388. _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
  1389. def _real_extract(self, url):
  1390. mobj = re.match(self._VALID_URL, url)
  1391. channel = mobj.group('chann')
  1392. title = mobj.group('title')
  1393. query = compat_urllib_parse.urlencode({'title': title})
  1394. playlist_re = 'href="([^"]+?%s.*?)"' % re.escape(query)
  1395. channel_page = self._download_webpage('https://www.youtube.com/%s' % channel, title)
  1396. link = self._html_search_regex(playlist_re, channel_page, u'list')
  1397. url = compat_urlparse.urljoin('https://www.youtube.com/', link)
  1398. video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
  1399. ids = []
  1400. # sometimes the webpage doesn't contain the videos
  1401. # retry until we get them
  1402. for i in itertools.count(0):
  1403. msg = u'Downloading Youtube mix'
  1404. if i > 0:
  1405. msg += ', retry #%d' % i
  1406. webpage = self._download_webpage(url, title, msg)
  1407. ids = orderedSet(re.findall(video_re, webpage))
  1408. if ids:
  1409. break
  1410. url_results = self._ids_to_results(ids)
  1411. return self.playlist_result(url_results, playlist_title=title)
  1412. class YoutubeChannelIE(InfoExtractor):
  1413. IE_DESC = u'YouTube.com channels'
  1414. _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
  1415. _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
  1416. _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
  1417. IE_NAME = u'youtube:channel'
  1418. def extract_videos_from_page(self, page):
  1419. ids_in_page = []
  1420. for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
  1421. if mobj.group(1) not in ids_in_page:
  1422. ids_in_page.append(mobj.group(1))
  1423. return ids_in_page
  1424. def _real_extract(self, url):
  1425. # Extract channel id
  1426. mobj = re.match(self._VALID_URL, url)
  1427. if mobj is None:
  1428. raise ExtractorError(u'Invalid URL: %s' % url)
  1429. # Download channel page
  1430. channel_id = mobj.group(1)
  1431. video_ids = []
  1432. url = 'https://www.youtube.com/channel/%s/videos' % channel_id
  1433. channel_page = self._download_webpage(url, channel_id)
  1434. autogenerated = re.search(r'''(?x)
  1435. class="[^"]*?(?:
  1436. channel-header-autogenerated-label|
  1437. yt-channel-title-autogenerated
  1438. )[^"]*"''', channel_page) is not None
  1439. if autogenerated:
  1440. # The videos are contained in a single page
  1441. # the ajax pages can't be used, they are empty
  1442. video_ids = self.extract_videos_from_page(channel_page)
  1443. else:
  1444. # Download all channel pages using the json-based channel_ajax query
  1445. for pagenum in itertools.count(1):
  1446. url = self._MORE_PAGES_URL % (pagenum, channel_id)
  1447. page = self._download_json(
  1448. url, channel_id, note=u'Downloading page #%s' % pagenum,
  1449. transform_source=uppercase_escape)
  1450. ids_in_page = self.extract_videos_from_page(page['content_html'])
  1451. video_ids.extend(ids_in_page)
  1452. if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
  1453. break
  1454. self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
  1455. url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id)
  1456. for video_id in video_ids]
  1457. return self.playlist_result(url_entries, channel_id)
  1458. class YoutubeUserIE(InfoExtractor):
  1459. IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
  1460. _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
  1461. _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s'
  1462. _GDATA_PAGE_SIZE = 50
  1463. _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
  1464. IE_NAME = u'youtube:user'
  1465. @classmethod
  1466. def suitable(cls, url):
  1467. # Don't return True if the url can be extracted with other youtube
  1468. # extractor, the regex would is too permissive and it would match.
  1469. other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
  1470. if any(ie.suitable(url) for ie in other_ies): return False
  1471. else: return super(YoutubeUserIE, cls).suitable(url)
  1472. def _real_extract(self, url):
  1473. # Extract username
  1474. mobj = re.match(self._VALID_URL, url)
  1475. if mobj is None:
  1476. raise ExtractorError(u'Invalid URL: %s' % url)
  1477. username = mobj.group(1)
  1478. # Download video ids using YouTube Data API. Result size per
  1479. # query is limited (currently to 50 videos) so we need to query
  1480. # page by page until there are no video ids - it means we got
  1481. # all of them.
  1482. def download_page(pagenum):
  1483. start_index = pagenum * self._GDATA_PAGE_SIZE + 1
  1484. gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
  1485. page = self._download_webpage(
  1486. gdata_url, username,
  1487. u'Downloading video ids from %d to %d' % (
  1488. start_index, start_index + self._GDATA_PAGE_SIZE))
  1489. try:
  1490. response = json.loads(page)
  1491. except ValueError as err:
  1492. raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
  1493. if 'entry' not in response['feed']:
  1494. return
  1495. # Extract video identifiers
  1496. entries = response['feed']['entry']
  1497. for entry in entries:
  1498. title = entry['title']['$t']
  1499. video_id = entry['id']['$t'].split('/')[-1]
  1500. yield {
  1501. '_type': 'url',
  1502. 'url': video_id,
  1503. 'ie_key': 'Youtube',
  1504. 'id': video_id,
  1505. 'title': title,
  1506. }
  1507. url_results = PagedList(download_page, self._GDATA_PAGE_SIZE)
  1508. return self.playlist_result(url_results, playlist_title=username)
  1509. class YoutubeSearchIE(SearchInfoExtractor):
  1510. IE_DESC = u'YouTube.com searches'
  1511. _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
  1512. _MAX_RESULTS = 1000
  1513. IE_NAME = u'youtube:search'
  1514. _SEARCH_KEY = 'ytsearch'
  1515. def _get_n_results(self, query, n):
  1516. """Get a specified number of results for a query"""
  1517. video_ids = []
  1518. pagenum = 0
  1519. limit = n
  1520. while (50 * pagenum) < limit:
  1521. result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
  1522. data_json = self._download_webpage(
  1523. result_url, video_id=u'query "%s"' % query,
  1524. note=u'Downloading page %s' % (pagenum + 1),
  1525. errnote=u'Unable to download API page')
  1526. data = json.loads(data_json)
  1527. api_response = data['data']
  1528. if 'items' not in api_response:
  1529. raise ExtractorError(
  1530. u'[youtube] No video results', expected=True)
  1531. new_ids = list(video['id'] for video in api_response['items'])
  1532. video_ids += new_ids
  1533. limit = min(n, api_response['totalItems'])
  1534. pagenum += 1
  1535. if len(video_ids) > n:
  1536. video_ids = video_ids[:n]
  1537. videos = [self.url_result(video_id, 'Youtube', video_id=video_id)
  1538. for video_id in video_ids]
  1539. return self.playlist_result(videos, query)
  1540. class YoutubeSearchDateIE(YoutubeSearchIE):
  1541. IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
  1542. _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
  1543. _SEARCH_KEY = 'ytsearchdate'
  1544. IE_DESC = u'YouTube.com searches, newest videos first'
  1545. class YoutubeShowIE(InfoExtractor):
  1546. IE_DESC = u'YouTube.com (multi-season) shows'
  1547. _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
  1548. IE_NAME = u'youtube:show'
  1549. def _real_extract(self, url):
  1550. mobj = re.match(self._VALID_URL, url)
  1551. show_name = mobj.group(1)
  1552. webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
  1553. # There's one playlist for each season of the show
  1554. m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
  1555. self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
  1556. return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
  1557. class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
  1558. """
  1559. Base class for extractors that fetch info from
  1560. http://www.youtube.com/feed_ajax
  1561. Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
  1562. """
  1563. _LOGIN_REQUIRED = True
  1564. # use action_load_personal_feed instead of action_load_system_feed
  1565. _PERSONAL_FEED = False
  1566. @property
  1567. def _FEED_TEMPLATE(self):
  1568. action = 'action_load_system_feed'
  1569. if self._PERSONAL_FEED:
  1570. action = 'action_load_personal_feed'
  1571. return 'https://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
  1572. @property
  1573. def IE_NAME(self):
  1574. return u'youtube:%s' % self._FEED_NAME
  1575. def _real_initialize(self):
  1576. self._login()
  1577. def _real_extract(self, url):
  1578. feed_entries = []
  1579. paging = 0
  1580. for i in itertools.count(1):
  1581. info = self._download_webpage(self._FEED_TEMPLATE % paging,
  1582. u'%s feed' % self._FEED_NAME,
  1583. u'Downloading page %s' % i)
  1584. info = json.loads(info)
  1585. feed_html = info['feed_html']
  1586. m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
  1587. ids = orderedSet(m.group(1) for m in m_ids)
  1588. feed_entries.extend(
  1589. self.url_result(video_id, 'Youtube', video_id=video_id)
  1590. for video_id in ids)
  1591. if info['paging'] is None:
  1592. break
  1593. paging = info['paging']
  1594. return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
  1595. class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
  1596. IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)'
  1597. _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
  1598. _FEED_NAME = 'subscriptions'
  1599. _PLAYLIST_TITLE = u'Youtube Subscriptions'
  1600. class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
  1601. IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
  1602. _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
  1603. _FEED_NAME = 'recommended'
  1604. _PLAYLIST_TITLE = u'Youtube Recommended videos'
  1605. class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
  1606. IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
  1607. _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
  1608. _FEED_NAME = 'watch_later'
  1609. _PLAYLIST_TITLE = u'Youtube Watch Later'
  1610. _PERSONAL_FEED = True
  1611. class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
  1612. IE_DESC = u'Youtube watch history, "ythistory" keyword (requires authentication)'
  1613. _VALID_URL = u'https?://www\.youtube\.com/feed/history|:ythistory'
  1614. _FEED_NAME = 'history'
  1615. _PERSONAL_FEED = True
  1616. _PLAYLIST_TITLE = u'Youtube Watch History'
  1617. class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
  1618. IE_NAME = u'youtube:favorites'
  1619. IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
  1620. _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
  1621. _LOGIN_REQUIRED = True
  1622. def _real_extract(self, url):
  1623. webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
  1624. playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, u'favourites playlist id')
  1625. return self.url_result(playlist_id, 'YoutubePlaylist')
  1626. class YoutubeTruncatedURLIE(InfoExtractor):
  1627. IE_NAME = 'youtube:truncated_url'
  1628. IE_DESC = False # Do not list
  1629. _VALID_URL = r'''(?x)
  1630. (?:https?://)?[^/]+/watch\?feature=[a-z_]+$|
  1631. (?:https?://)?(?:www\.)?youtube\.com/attribution_link\?a=[^&]+$
  1632. '''
  1633. def _real_extract(self, url):
  1634. raise ExtractorError(
  1635. u'Did you forget to quote the URL? Remember that & is a meta '
  1636. u'character in most shells, so you want to put the URL in quotes, '
  1637. u'like youtube-dl '
  1638. u'"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
  1639. u' or simply youtube-dl BaW_jenozKc .',
  1640. expected=True)