You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

4079 lines
162 KiB

13 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
13 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. from __future__ import absolute_import
  4. import base64
  5. import datetime
  6. import itertools
  7. import netrc
  8. import os
  9. import re
  10. import socket
  11. import time
  12. import email.utils
  13. import xml.etree.ElementTree
  14. import random
  15. import math
  16. from .utils import *
  17. class InfoExtractor(object):
  18. """Information Extractor class.
  19. Information extractors are the classes that, given a URL, extract
  20. information about the video (or videos) the URL refers to. This
  21. information includes the real video URL, the video title, author and
  22. others. The information is stored in a dictionary which is then
  23. passed to the FileDownloader. The FileDownloader processes this
  24. information possibly downloading the video to the file system, among
  25. other possible outcomes.
  26. The dictionaries must include the following fields:
  27. id: Video identifier.
  28. url: Final video URL.
  29. title: Video title, unescaped.
  30. ext: Video filename extension.
  31. The following fields are optional:
  32. format: The video format, defaults to ext (used for --get-format)
  33. thumbnail: Full URL to a video thumbnail image.
  34. description: One-line video description.
  35. uploader: Full name of the video uploader.
  36. upload_date: Video upload date (YYYYMMDD).
  37. uploader_id: Nickname or id of the video uploader.
  38. location: Physical location of the video.
  39. player_url: SWF Player URL (used for rtmpdump).
  40. subtitles: The .srt file contents.
  41. urlhandle: [internal] The urlHandle to be used to download the file,
  42. like returned by urllib.request.urlopen
  43. The fields should all be Unicode strings.
  44. Subclasses of this one should re-define the _real_initialize() and
  45. _real_extract() methods and define a _VALID_URL regexp.
  46. Probably, they should also be added to the list of extractors.
  47. _real_extract() must return a *list* of information dictionaries as
  48. described above.
  49. Finally, the _WORKING attribute should be set to False for broken IEs
  50. in order to warn the users and skip the tests.
  51. """
  52. _ready = False
  53. _downloader = None
  54. _WORKING = True
  55. def __init__(self, downloader=None):
  56. """Constructor. Receives an optional downloader."""
  57. self._ready = False
  58. self.set_downloader(downloader)
  59. def suitable(self, url):
  60. """Receives a URL and returns True if suitable for this IE."""
  61. return re.match(self._VALID_URL, url) is not None
  62. def working(self):
  63. """Getter method for _WORKING."""
  64. return self._WORKING
  65. def initialize(self):
  66. """Initializes an instance (authentication, etc)."""
  67. if not self._ready:
  68. self._real_initialize()
  69. self._ready = True
  70. def extract(self, url):
  71. """Extracts URL information and returns it in list of dicts."""
  72. self.initialize()
  73. return self._real_extract(url)
  74. def set_downloader(self, downloader):
  75. """Sets the downloader for this IE."""
  76. self._downloader = downloader
  77. def _real_initialize(self):
  78. """Real initialization process. Redefine in subclasses."""
  79. pass
  80. def _real_extract(self, url):
  81. """Real extraction process. Redefine in subclasses."""
  82. pass
  83. @property
  84. def IE_NAME(self):
  85. return type(self).__name__[:-2]
  86. def _request_webpage(self, url_or_request, video_id, note=None, errnote=None):
  87. """ Returns the response handle """
  88. if note is None:
  89. note = u'Downloading video webpage'
  90. self._downloader.to_screen(u'[%s] %s: %s' % (self.IE_NAME, video_id, note))
  91. try:
  92. return compat_urllib_request.urlopen(url_or_request)
  93. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  94. if errnote is None:
  95. errnote = u'Unable to download webpage'
  96. raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2])
  97. def _download_webpage(self, url_or_request, video_id, note=None, errnote=None):
  98. """ Returns the data of the page as a string """
  99. urlh = self._request_webpage(url_or_request, video_id, note, errnote)
  100. webpage_bytes = urlh.read()
  101. return webpage_bytes.decode('utf-8', 'replace')
  102. class YoutubeIE(InfoExtractor):
  103. """Information extractor for youtube.com."""
  104. _VALID_URL = r"""^
  105. (
  106. (?:https?://)? # http(s):// (optional)
  107. (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
  108. tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains
  109. (?:.*?\#/)? # handle anchor (#/) redirect urls
  110. (?!view_play_list|my_playlists|artist|playlist) # ignore playlist URLs
  111. (?: # the various things that can precede the ID:
  112. (?:(?:v|embed|e)/) # v/ or embed/ or e/
  113. |(?: # or the v= param in all its forms
  114. (?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
  115. (?:\?|\#!?) # the params delimiter ? or # or #!
  116. (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
  117. v=
  118. )
  119. )? # optional -> youtube.com/xxxx is OK
  120. )? # all until now is optional -> you can pass the naked ID
  121. ([0-9A-Za-z_-]+) # here is it! the YouTube video ID
  122. (?(1).+)? # if we found the ID, everything can follow
  123. $"""
  124. _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
  125. _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  126. _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
  127. _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
  128. _NETRC_MACHINE = 'youtube'
  129. # Listed in order of quality
  130. _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
  131. _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
  132. _video_extensions = {
  133. '13': '3gp',
  134. '17': 'mp4',
  135. '18': 'mp4',
  136. '22': 'mp4',
  137. '37': 'mp4',
  138. '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
  139. '43': 'webm',
  140. '44': 'webm',
  141. '45': 'webm',
  142. '46': 'webm',
  143. }
  144. _video_dimensions = {
  145. '5': '240x400',
  146. '6': '???',
  147. '13': '???',
  148. '17': '144x176',
  149. '18': '360x640',
  150. '22': '720x1280',
  151. '34': '360x640',
  152. '35': '480x854',
  153. '37': '1080x1920',
  154. '38': '3072x4096',
  155. '43': '360x640',
  156. '44': '480x854',
  157. '45': '720x1280',
  158. '46': '1080x1920',
  159. }
  160. IE_NAME = u'youtube'
  161. def suitable(self, url):
  162. """Receives a URL and returns True if suitable for this IE."""
  163. return re.match(self._VALID_URL, url, re.VERBOSE) is not None
  164. def report_lang(self):
  165. """Report attempt to set language."""
  166. self._downloader.to_screen(u'[youtube] Setting language')
  167. def report_login(self):
  168. """Report attempt to log in."""
  169. self._downloader.to_screen(u'[youtube] Logging in')
  170. def report_age_confirmation(self):
  171. """Report attempt to confirm age."""
  172. self._downloader.to_screen(u'[youtube] Confirming age')
  173. def report_video_webpage_download(self, video_id):
  174. """Report attempt to download video webpage."""
  175. self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
  176. def report_video_info_webpage_download(self, video_id):
  177. """Report attempt to download video info webpage."""
  178. self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
  179. def report_video_subtitles_download(self, video_id):
  180. """Report attempt to download video info webpage."""
  181. self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id)
  182. def report_information_extraction(self, video_id):
  183. """Report attempt to extract video information."""
  184. self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
  185. def report_unavailable_format(self, video_id, format):
  186. """Report extracted video URL."""
  187. self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
  188. def report_rtmp_download(self):
  189. """Indicate the download will use the RTMP protocol."""
  190. self._downloader.to_screen(u'[youtube] RTMP download detected')
  191. def _closed_captions_xml_to_srt(self, xml_string):
  192. srt = ''
  193. texts = re.findall(r'<text start="([\d\.]+)"( dur="([\d\.]+)")?>([^<]+)</text>', xml_string, re.MULTILINE)
  194. # TODO parse xml instead of regex
  195. for n, (start, dur_tag, dur, caption) in enumerate(texts):
  196. if not dur: dur = '4'
  197. start = float(start)
  198. end = start + float(dur)
  199. start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
  200. end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
  201. caption = unescapeHTML(caption)
  202. caption = unescapeHTML(caption) # double cycle, intentional
  203. srt += str(n+1) + '\n'
  204. srt += start + ' --> ' + end + '\n'
  205. srt += caption + '\n\n'
  206. return srt
  207. def _extract_subtitles(self, video_id):
  208. self.report_video_subtitles_download(video_id)
  209. request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
  210. try:
  211. srt_list = compat_urllib_request.urlopen(request).read().decode('utf-8')
  212. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  213. return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None)
  214. srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
  215. srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
  216. if not srt_lang_list:
  217. return (u'WARNING: video has no closed captions', None)
  218. if self._downloader.params.get('subtitleslang', False):
  219. srt_lang = self._downloader.params.get('subtitleslang')
  220. elif 'en' in srt_lang_list:
  221. srt_lang = 'en'
  222. else:
  223. srt_lang = list(srt_lang_list.keys())[0]
  224. if not srt_lang in srt_lang_list:
  225. return (u'WARNING: no closed captions found in the specified language', None)
  226. params = compat_urllib_parse.urlencode({
  227. 'lang': srt_lang,
  228. 'name': srt_lang_list[srt_lang].encode('utf-8'),
  229. 'v': video_id,
  230. })
  231. url = 'http://www.youtube.com/api/timedtext?' + params
  232. try:
  233. srt_xml = compat_urllib_request.urlopen(url).read().decode('utf-8')
  234. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  235. return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None)
  236. if not srt_xml:
  237. return (u'WARNING: Did not fetch video subtitles', None)
  238. return (None, self._closed_captions_xml_to_srt(srt_xml))
  239. def _print_formats(self, formats):
  240. print('Available formats:')
  241. for x in formats:
  242. print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
  243. def _real_initialize(self):
  244. if self._downloader is None:
  245. return
  246. username = None
  247. password = None
  248. downloader_params = self._downloader.params
  249. # Attempt to use provided username and password or .netrc data
  250. if downloader_params.get('username', None) is not None:
  251. username = downloader_params['username']
  252. password = downloader_params['password']
  253. elif downloader_params.get('usenetrc', False):
  254. try:
  255. info = netrc.netrc().authenticators(self._NETRC_MACHINE)
  256. if info is not None:
  257. username = info[0]
  258. password = info[2]
  259. else:
  260. raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
  261. except (IOError, netrc.NetrcParseError) as err:
  262. self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
  263. return
  264. # Set language
  265. request = compat_urllib_request.Request(self._LANG_URL)
  266. try:
  267. self.report_lang()
  268. compat_urllib_request.urlopen(request).read()
  269. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  270. self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
  271. return
  272. # No authentication to be performed
  273. if username is None:
  274. return
  275. request = compat_urllib_request.Request(self._LOGIN_URL)
  276. try:
  277. login_page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  278. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  279. self._downloader.to_stderr(u'WARNING: unable to fetch login page: %s' % compat_str(err))
  280. return
  281. galx = None
  282. dsh = None
  283. match = re.search(re.compile(r'<input.+?name="GALX".+?value="(.+?)"', re.DOTALL), login_page)
  284. if match:
  285. galx = match.group(1)
  286. match = re.search(re.compile(r'<input.+?name="dsh".+?value="(.+?)"', re.DOTALL), login_page)
  287. if match:
  288. dsh = match.group(1)
  289. # Log in
  290. login_form_strs = {
  291. u'continue': u'http://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
  292. u'Email': username,
  293. u'GALX': galx,
  294. u'Passwd': password,
  295. u'PersistentCookie': u'yes',
  296. u'_utf8': u'',
  297. u'bgresponse': u'js_disabled',
  298. u'checkConnection': u'',
  299. u'checkedDomains': u'youtube',
  300. u'dnConn': u'',
  301. u'dsh': dsh,
  302. u'pstMsg': u'0',
  303. u'rmShown': u'1',
  304. u'secTok': u'',
  305. u'signIn': u'Sign in',
  306. u'timeStmp': u'',
  307. u'service': u'youtube',
  308. u'uilel': u'3',
  309. u'hl': u'en_US',
  310. }
  311. # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
  312. # chokes on unicode
  313. login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
  314. login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
  315. request = compat_urllib_request.Request(self._LOGIN_URL, login_data)
  316. try:
  317. self.report_login()
  318. login_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
  319. if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
  320. self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
  321. return
  322. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  323. self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
  324. return
  325. # Confirm age
  326. age_form = {
  327. 'next_url': '/',
  328. 'action_confirm': 'Confirm',
  329. }
  330. request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
  331. try:
  332. self.report_age_confirmation()
  333. age_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
  334. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  335. self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
  336. return
  337. def _extract_id(self, url):
  338. mobj = re.match(self._VALID_URL, url, re.VERBOSE)
  339. if mobj is None:
  340. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  341. return
  342. video_id = mobj.group(2)
  343. return video_id
  344. def _real_extract(self, url):
  345. # Extract original video URL from URL with redirection, like age verification, using next_url parameter
  346. mobj = re.search(self._NEXT_URL_RE, url)
  347. if mobj:
  348. url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
  349. video_id = self._extract_id(url)
  350. # Get video webpage
  351. self.report_video_webpage_download(video_id)
  352. url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
  353. request = compat_urllib_request.Request(url)
  354. try:
  355. video_webpage_bytes = compat_urllib_request.urlopen(request).read()
  356. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  357. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
  358. return
  359. video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
  360. # Attempt to extract SWF player URL
  361. mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
  362. if mobj is not None:
  363. player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
  364. else:
  365. player_url = None
  366. # Get video info
  367. self.report_video_info_webpage_download(video_id)
  368. for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
  369. video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
  370. % (video_id, el_type))
  371. request = compat_urllib_request.Request(video_info_url)
  372. try:
  373. video_info_webpage_bytes = compat_urllib_request.urlopen(request).read()
  374. video_info_webpage = video_info_webpage_bytes.decode('utf-8', 'ignore')
  375. video_info = compat_parse_qs(video_info_webpage)
  376. if 'token' in video_info:
  377. break
  378. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  379. self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
  380. return
  381. if 'token' not in video_info:
  382. if 'reason' in video_info:
  383. self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0])
  384. else:
  385. self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
  386. return
  387. # Check for "rental" videos
  388. if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
  389. self._downloader.trouble(u'ERROR: "rental" videos not supported')
  390. return
  391. # Start extracting information
  392. self.report_information_extraction(video_id)
  393. # uploader
  394. if 'author' not in video_info:
  395. self._downloader.trouble(u'ERROR: unable to extract uploader name')
  396. return
  397. video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
  398. # uploader_id
  399. video_uploader_id = None
  400. mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
  401. if mobj is not None:
  402. video_uploader_id = mobj.group(1)
  403. else:
  404. self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
  405. # title
  406. if 'title' not in video_info:
  407. self._downloader.trouble(u'ERROR: unable to extract video title')
  408. return
  409. video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
  410. # thumbnail image
  411. if 'thumbnail_url' not in video_info:
  412. self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
  413. video_thumbnail = ''
  414. else: # don't panic if we can't find it
  415. video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
  416. # upload date
  417. upload_date = None
  418. mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
  419. if mobj is not None:
  420. upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
  421. format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
  422. for expression in format_expressions:
  423. try:
  424. upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
  425. except:
  426. pass
  427. # description
  428. video_description = get_element_by_id("eow-description", video_webpage)
  429. if video_description:
  430. video_description = clean_html(video_description)
  431. else:
  432. video_description = ''
  433. # closed captions
  434. video_subtitles = None
  435. if self._downloader.params.get('writesubtitles', False):
  436. (srt_error, video_subtitles) = self._extract_subtitles(video_id)
  437. if srt_error:
  438. self._downloader.trouble(srt_error)
  439. if 'length_seconds' not in video_info:
  440. self._downloader.trouble(u'WARNING: unable to extract video duration')
  441. video_duration = ''
  442. else:
  443. video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
  444. # token
  445. video_token = compat_urllib_parse.unquote_plus(video_info['token'][0])
  446. # Decide which formats to download
  447. req_format = self._downloader.params.get('format', None)
  448. if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
  449. self.report_rtmp_download()
  450. video_url_list = [(None, video_info['conn'][0])]
  451. elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
  452. url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
  453. url_data = [compat_parse_qs(uds) for uds in url_data_strs]
  454. url_data = [ud for ud in url_data if 'itag' in ud and 'url' in ud]
  455. url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
  456. format_limit = self._downloader.params.get('format_limit', None)
  457. available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
  458. if format_limit is not None and format_limit in available_formats:
  459. format_list = available_formats[available_formats.index(format_limit):]
  460. else:
  461. format_list = available_formats
  462. existing_formats = [x for x in format_list if x in url_map]
  463. if len(existing_formats) == 0:
  464. self._downloader.trouble(u'ERROR: no known formats available for video')
  465. return
  466. if self._downloader.params.get('listformats', None):
  467. self._print_formats(existing_formats)
  468. return
  469. if req_format is None or req_format == 'best':
  470. video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
  471. elif req_format == 'worst':
  472. video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
  473. elif req_format in ('-1', 'all'):
  474. video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
  475. else:
  476. # Specific formats. We pick the first in a slash-delimeted sequence.
  477. # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
  478. req_formats = req_format.split('/')
  479. video_url_list = None
  480. for rf in req_formats:
  481. if rf in url_map:
  482. video_url_list = [(rf, url_map[rf])]
  483. break
  484. if video_url_list is None:
  485. self._downloader.trouble(u'ERROR: requested format not available')
  486. return
  487. else:
  488. self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
  489. return
  490. results = []
  491. for format_param, video_real_url in video_url_list:
  492. # Extension
  493. video_extension = self._video_extensions.get(format_param, 'flv')
  494. video_format = '{0} - {1}'.format(format_param if format_param else video_extension,
  495. self._video_dimensions.get(format_param, '???'))
  496. results.append({
  497. 'id': video_id,
  498. 'url': video_real_url,
  499. 'uploader': video_uploader,
  500. 'uploader_id': video_uploader_id,
  501. 'upload_date': upload_date,
  502. 'title': video_title,
  503. 'ext': video_extension,
  504. 'format': video_format,
  505. 'thumbnail': video_thumbnail,
  506. 'description': video_description,
  507. 'player_url': player_url,
  508. 'subtitles': video_subtitles,
  509. 'duration': video_duration
  510. })
  511. return results
  512. class MetacafeIE(InfoExtractor):
  513. """Information Extractor for metacafe.com."""
  514. _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
  515. _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
  516. _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
  517. IE_NAME = u'metacafe'
  518. def __init__(self, downloader=None):
  519. InfoExtractor.__init__(self, downloader)
  520. def report_disclaimer(self):
  521. """Report disclaimer retrieval."""
  522. self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
  523. def report_age_confirmation(self):
  524. """Report attempt to confirm age."""
  525. self._downloader.to_screen(u'[metacafe] Confirming age')
  526. def report_download_webpage(self, video_id):
  527. """Report webpage download."""
  528. self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
  529. def report_extraction(self, video_id):
  530. """Report information extraction."""
  531. self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
  532. def _real_initialize(self):
  533. # Retrieve disclaimer
  534. request = compat_urllib_request.Request(self._DISCLAIMER)
  535. try:
  536. self.report_disclaimer()
  537. disclaimer = compat_urllib_request.urlopen(request).read()
  538. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  539. self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
  540. return
  541. # Confirm age
  542. disclaimer_form = {
  543. 'filters': '0',
  544. 'submit': "Continue - I'm over 18",
  545. }
  546. request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
  547. try:
  548. self.report_age_confirmation()
  549. disclaimer = compat_urllib_request.urlopen(request).read()
  550. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  551. self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
  552. return
  553. def _real_extract(self, url):
  554. # Extract id and simplified title from URL
  555. mobj = re.match(self._VALID_URL, url)
  556. if mobj is None:
  557. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  558. return
  559. video_id = mobj.group(1)
  560. # Check if video comes from YouTube
  561. mobj2 = re.match(r'^yt-(.*)$', video_id)
  562. if mobj2 is not None:
  563. self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
  564. return
  565. # Retrieve video webpage to extract further information
  566. request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
  567. try:
  568. self.report_download_webpage(video_id)
  569. webpage = compat_urllib_request.urlopen(request).read()
  570. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  571. self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
  572. return
  573. # Extract URL, uploader and title from webpage
  574. self.report_extraction(video_id)
  575. mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
  576. if mobj is not None:
  577. mediaURL = compat_urllib_parse.unquote(mobj.group(1))
  578. video_extension = mediaURL[-3:]
  579. # Extract gdaKey if available
  580. mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
  581. if mobj is None:
  582. video_url = mediaURL
  583. else:
  584. gdaKey = mobj.group(1)
  585. video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
  586. else:
  587. mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
  588. if mobj is None:
  589. self._downloader.trouble(u'ERROR: unable to extract media URL')
  590. return
  591. vardict = compat_parse_qs(mobj.group(1))
  592. if 'mediaData' not in vardict:
  593. self._downloader.trouble(u'ERROR: unable to extract media URL')
  594. return
  595. mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
  596. if mobj is None:
  597. self._downloader.trouble(u'ERROR: unable to extract media URL')
  598. return
  599. mediaURL = mobj.group(1).replace('\\/', '/')
  600. video_extension = mediaURL[-3:]
  601. video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
  602. mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
  603. if mobj is None:
  604. self._downloader.trouble(u'ERROR: unable to extract title')
  605. return
  606. video_title = mobj.group(1).decode('utf-8')
  607. mobj = re.search(r'submitter=(.*?);', webpage)
  608. if mobj is None:
  609. self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
  610. return
  611. video_uploader = mobj.group(1)
  612. return [{
  613. 'id': video_id.decode('utf-8'),
  614. 'url': video_url.decode('utf-8'),
  615. 'uploader': video_uploader.decode('utf-8'),
  616. 'upload_date': None,
  617. 'title': video_title,
  618. 'ext': video_extension.decode('utf-8'),
  619. }]
  620. class DailymotionIE(InfoExtractor):
  621. """Information Extractor for Dailymotion"""
  622. _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
  623. IE_NAME = u'dailymotion'
  624. def __init__(self, downloader=None):
  625. InfoExtractor.__init__(self, downloader)
  626. def report_extraction(self, video_id):
  627. """Report information extraction."""
  628. self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
  629. def _real_extract(self, url):
  630. # Extract id and simplified title from URL
  631. mobj = re.match(self._VALID_URL, url)
  632. if mobj is None:
  633. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  634. return
  635. video_id = mobj.group(1).split('_')[0].split('?')[0]
  636. video_extension = 'mp4'
  637. # Retrieve video webpage to extract further information
  638. request = compat_urllib_request.Request(url)
  639. request.add_header('Cookie', 'family_filter=off')
  640. webpage = self._download_webpage(request, video_id)
  641. # Extract URL, uploader and title from webpage
  642. self.report_extraction(video_id)
  643. mobj = re.search(r'\s*var flashvars = (.*)', webpage)
  644. if mobj is None:
  645. self._downloader.trouble(u'ERROR: unable to extract media URL')
  646. return
  647. flashvars = compat_urllib_parse.unquote(mobj.group(1))
  648. for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
  649. if key in flashvars:
  650. max_quality = key
  651. self._downloader.to_screen(u'[dailymotion] Using %s' % key)
  652. break
  653. else:
  654. self._downloader.trouble(u'ERROR: unable to extract video URL')
  655. return
  656. mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
  657. if mobj is None:
  658. self._downloader.trouble(u'ERROR: unable to extract video URL')
  659. return
  660. video_url = compat_urllib_parse.unquote(mobj.group(1)).replace('\\/', '/')
  661. # TODO: support choosing qualities
  662. mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
  663. if mobj is None:
  664. self._downloader.trouble(u'ERROR: unable to extract title')
  665. return
  666. video_title = unescapeHTML(mobj.group('title'))
  667. video_uploader = None
  668. mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
  669. if mobj is None:
  670. # lookin for official user
  671. mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage)
  672. if mobj_official is None:
  673. self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
  674. else:
  675. video_uploader = mobj_official.group(1)
  676. else:
  677. video_uploader = mobj.group(1)
  678. video_upload_date = None
  679. mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
  680. if mobj is not None:
  681. video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
  682. return [{
  683. 'id': video_id,
  684. 'url': video_url,
  685. 'uploader': video_uploader,
  686. 'upload_date': video_upload_date,
  687. 'title': video_title,
  688. 'ext': video_extension,
  689. }]
  690. class PhotobucketIE(InfoExtractor):
  691. """Information extractor for photobucket.com."""
  692. _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
  693. IE_NAME = u'photobucket'
  694. def __init__(self, downloader=None):
  695. InfoExtractor.__init__(self, downloader)
  696. def report_download_webpage(self, video_id):
  697. """Report webpage download."""
  698. self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
  699. def report_extraction(self, video_id):
  700. """Report information extraction."""
  701. self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
  702. def _real_extract(self, url):
  703. # Extract id from URL
  704. mobj = re.match(self._VALID_URL, url)
  705. if mobj is None:
  706. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  707. return
  708. video_id = mobj.group(1)
  709. video_extension = 'flv'
  710. # Retrieve video webpage to extract further information
  711. request = compat_urllib_request.Request(url)
  712. try:
  713. self.report_download_webpage(video_id)
  714. webpage = compat_urllib_request.urlopen(request).read()
  715. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  716. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  717. return
  718. # Extract URL, uploader, and title from webpage
  719. self.report_extraction(video_id)
  720. mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
  721. if mobj is None:
  722. self._downloader.trouble(u'ERROR: unable to extract media URL')
  723. return
  724. mediaURL = compat_urllib_parse.unquote(mobj.group(1))
  725. video_url = mediaURL
  726. mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
  727. if mobj is None:
  728. self._downloader.trouble(u'ERROR: unable to extract title')
  729. return
  730. video_title = mobj.group(1).decode('utf-8')
  731. video_uploader = mobj.group(2).decode('utf-8')
  732. return [{
  733. 'id': video_id.decode('utf-8'),
  734. 'url': video_url.decode('utf-8'),
  735. 'uploader': video_uploader,
  736. 'upload_date': None,
  737. 'title': video_title,
  738. 'ext': video_extension.decode('utf-8'),
  739. }]
  740. class YahooIE(InfoExtractor):
  741. """Information extractor for video.yahoo.com."""
  742. _WORKING = False
  743. # _VALID_URL matches all Yahoo! Video URLs
  744. # _VPAGE_URL matches only the extractable '/watch/' URLs
  745. _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
  746. _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
  747. IE_NAME = u'video.yahoo'
  748. def __init__(self, downloader=None):
  749. InfoExtractor.__init__(self, downloader)
  750. def report_download_webpage(self, video_id):
  751. """Report webpage download."""
  752. self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
  753. def report_extraction(self, video_id):
  754. """Report information extraction."""
  755. self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
  756. def _real_extract(self, url, new_video=True):
  757. # Extract ID from URL
  758. mobj = re.match(self._VALID_URL, url)
  759. if mobj is None:
  760. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  761. return
  762. video_id = mobj.group(2)
  763. video_extension = 'flv'
  764. # Rewrite valid but non-extractable URLs as
  765. # extractable English language /watch/ URLs
  766. if re.match(self._VPAGE_URL, url) is None:
  767. request = compat_urllib_request.Request(url)
  768. try:
  769. webpage = compat_urllib_request.urlopen(request).read()
  770. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  771. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  772. return
  773. mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
  774. if mobj is None:
  775. self._downloader.trouble(u'ERROR: Unable to extract id field')
  776. return
  777. yahoo_id = mobj.group(1)
  778. mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
  779. if mobj is None:
  780. self._downloader.trouble(u'ERROR: Unable to extract vid field')
  781. return
  782. yahoo_vid = mobj.group(1)
  783. url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
  784. return self._real_extract(url, new_video=False)
  785. # Retrieve video webpage to extract further information
  786. request = compat_urllib_request.Request(url)
  787. try:
  788. self.report_download_webpage(video_id)
  789. webpage = compat_urllib_request.urlopen(request).read()
  790. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  791. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  792. return
  793. # Extract uploader and title from webpage
  794. self.report_extraction(video_id)
  795. mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
  796. if mobj is None:
  797. self._downloader.trouble(u'ERROR: unable to extract video title')
  798. return
  799. video_title = mobj.group(1).decode('utf-8')
  800. mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
  801. if mobj is None:
  802. self._downloader.trouble(u'ERROR: unable to extract video uploader')
  803. return
  804. video_uploader = mobj.group(1).decode('utf-8')
  805. # Extract video thumbnail
  806. mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
  807. if mobj is None:
  808. self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
  809. return
  810. video_thumbnail = mobj.group(1).decode('utf-8')
  811. # Extract video description
  812. mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
  813. if mobj is None:
  814. self._downloader.trouble(u'ERROR: unable to extract video description')
  815. return
  816. video_description = mobj.group(1).decode('utf-8')
  817. if not video_description:
  818. video_description = 'No description available.'
  819. # Extract video height and width
  820. mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
  821. if mobj is None:
  822. self._downloader.trouble(u'ERROR: unable to extract video height')
  823. return
  824. yv_video_height = mobj.group(1)
  825. mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
  826. if mobj is None:
  827. self._downloader.trouble(u'ERROR: unable to extract video width')
  828. return
  829. yv_video_width = mobj.group(1)
  830. # Retrieve video playlist to extract media URL
  831. # I'm not completely sure what all these options are, but we
  832. # seem to need most of them, otherwise the server sends a 401.
  833. yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
  834. yv_bitrate = '700' # according to Wikipedia this is hard-coded
  835. request = compat_urllib_request.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
  836. '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
  837. '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
  838. try:
  839. self.report_download_webpage(video_id)
  840. webpage = compat_urllib_request.urlopen(request).read()
  841. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  842. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  843. return
  844. # Extract media URL from playlist XML
  845. mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
  846. if mobj is None:
  847. self._downloader.trouble(u'ERROR: Unable to extract media URL')
  848. return
  849. video_url = compat_urllib_parse.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
  850. video_url = unescapeHTML(video_url)
  851. return [{
  852. 'id': video_id.decode('utf-8'),
  853. 'url': video_url,
  854. 'uploader': video_uploader,
  855. 'upload_date': None,
  856. 'title': video_title,
  857. 'ext': video_extension.decode('utf-8'),
  858. 'thumbnail': video_thumbnail.decode('utf-8'),
  859. 'description': video_description,
  860. }]
  861. class VimeoIE(InfoExtractor):
  862. """Information extractor for vimeo.com."""
  863. # _VALID_URL matches Vimeo URLs
  864. _VALID_URL = r'(?P<proto>https?://)?(?:(?:www|player)\.)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)'
  865. IE_NAME = u'vimeo'
  866. def __init__(self, downloader=None):
  867. InfoExtractor.__init__(self, downloader)
  868. def report_download_webpage(self, video_id):
  869. """Report webpage download."""
  870. self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
  871. def report_extraction(self, video_id):
  872. """Report information extraction."""
  873. self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
  874. def _real_extract(self, url, new_video=True):
  875. # Extract ID from URL
  876. mobj = re.match(self._VALID_URL, url)
  877. if mobj is None:
  878. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  879. return
  880. video_id = mobj.group('id')
  881. if not mobj.group('proto'):
  882. url = 'https://' + url
  883. if mobj.group('direct_link'):
  884. url = 'https://vimeo.com/' + video_id
  885. # Retrieve video webpage to extract further information
  886. request = compat_urllib_request.Request(url, None, std_headers)
  887. try:
  888. self.report_download_webpage(video_id)
  889. webpage_bytes = compat_urllib_request.urlopen(request).read()
  890. webpage = webpage_bytes.decode('utf-8')
  891. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  892. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  893. return
  894. # Now we begin extracting as much information as we can from what we
  895. # retrieved. First we extract the information common to all extractors,
  896. # and latter we extract those that are Vimeo specific.
  897. self.report_extraction(video_id)
  898. # Extract the config JSON
  899. try:
  900. config = webpage.split(' = {config:')[1].split(',assets:')[0]
  901. config = json.loads(config)
  902. except:
  903. self._downloader.trouble(u'ERROR: unable to extract info section')
  904. return
  905. # Extract title
  906. video_title = config["video"]["title"]
  907. # Extract uploader and uploader_id
  908. video_uploader = config["video"]["owner"]["name"]
  909. video_uploader_id = config["video"]["owner"]["url"].split('/')[-1]
  910. # Extract video thumbnail
  911. video_thumbnail = config["video"]["thumbnail"]
  912. # Extract video description
  913. video_description = get_element_by_attribute("itemprop", "description", webpage)
  914. if video_description: video_description = clean_html(video_description)
  915. else: video_description = ''
  916. # Extract upload date
  917. video_upload_date = None
  918. mobj = re.search(r'<meta itemprop="dateCreated" content="(\d{4})-(\d{2})-(\d{2})T', webpage)
  919. if mobj is not None:
  920. video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
  921. # Vimeo specific: extract request signature and timestamp
  922. sig = config['request']['signature']
  923. timestamp = config['request']['timestamp']
  924. # Vimeo specific: extract video codec and quality information
  925. # First consider quality, then codecs, then take everything
  926. # TODO bind to format param
  927. codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
  928. files = { 'hd': [], 'sd': [], 'other': []}
  929. for codec_name, codec_extension in codecs:
  930. if codec_name in config["video"]["files"]:
  931. if 'hd' in config["video"]["files"][codec_name]:
  932. files['hd'].append((codec_name, codec_extension, 'hd'))
  933. elif 'sd' in config["video"]["files"][codec_name]:
  934. files['sd'].append((codec_name, codec_extension, 'sd'))
  935. else:
  936. files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0]))
  937. for quality in ('hd', 'sd', 'other'):
  938. if len(files[quality]) > 0:
  939. video_quality = files[quality][0][2]
  940. video_codec = files[quality][0][0]
  941. video_extension = files[quality][0][1]
  942. self._downloader.to_screen(u'[vimeo] %s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality))
  943. break
  944. else:
  945. self._downloader.trouble(u'ERROR: no known codec found')
  946. return
  947. video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
  948. %(video_id, sig, timestamp, video_quality, video_codec.upper())
  949. return [{
  950. 'id': video_id,
  951. 'url': video_url,
  952. 'uploader': video_uploader,
  953. 'uploader_id': video_uploader_id,
  954. 'upload_date': video_upload_date,
  955. 'title': video_title,
  956. 'ext': video_extension,
  957. 'thumbnail': video_thumbnail,
  958. 'description': video_description,
  959. }]
  960. class ArteTvIE(InfoExtractor):
  961. """arte.tv information extractor."""
  962. _VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*'
  963. _LIVE_URL = r'index-[0-9]+\.html$'
  964. IE_NAME = u'arte.tv'
  965. def __init__(self, downloader=None):
  966. InfoExtractor.__init__(self, downloader)
  967. def report_download_webpage(self, video_id):
  968. """Report webpage download."""
  969. self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id)
  970. def report_extraction(self, video_id):
  971. """Report information extraction."""
  972. self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id)
  973. def fetch_webpage(self, url):
  974. request = compat_urllib_request.Request(url)
  975. try:
  976. self.report_download_webpage(url)
  977. webpage = compat_urllib_request.urlopen(request).read()
  978. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  979. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  980. return
  981. except ValueError as err:
  982. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  983. return
  984. return webpage
  985. def grep_webpage(self, url, regex, regexFlags, matchTuples):
  986. page = self.fetch_webpage(url)
  987. mobj = re.search(regex, page, regexFlags)
  988. info = {}
  989. if mobj is None:
  990. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  991. return
  992. for (i, key, err) in matchTuples:
  993. if mobj.group(i) is None:
  994. self._downloader.trouble(err)
  995. return
  996. else:
  997. info[key] = mobj.group(i)
  998. return info
  999. def extractLiveStream(self, url):
  1000. video_lang = url.split('/')[-4]
  1001. info = self.grep_webpage(
  1002. url,
  1003. r'src="(.*?/videothek_js.*?\.js)',
  1004. 0,
  1005. [
  1006. (1, 'url', u'ERROR: Invalid URL: %s' % url)
  1007. ]
  1008. )
  1009. http_host = url.split('/')[2]
  1010. next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url')))
  1011. info = self.grep_webpage(
  1012. next_url,
  1013. r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
  1014. '(http://.*?\.swf).*?' +
  1015. '(rtmp://.*?)\'',
  1016. re.DOTALL,
  1017. [
  1018. (1, 'path', u'ERROR: could not extract video path: %s' % url),
  1019. (2, 'player', u'ERROR: could not extract video player: %s' % url),
  1020. (3, 'url', u'ERROR: could not extract video url: %s' % url)
  1021. ]
  1022. )
  1023. video_url = u'%s/%s' % (info.get('url'), info.get('path'))
  1024. def extractPlus7Stream(self, url):
  1025. video_lang = url.split('/')[-3]
  1026. info = self.grep_webpage(
  1027. url,
  1028. r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
  1029. 0,
  1030. [
  1031. (1, 'url', u'ERROR: Invalid URL: %s' % url)
  1032. ]
  1033. )
  1034. next_url = compat_urllib_parse.unquote(info.get('url'))
  1035. info = self.grep_webpage(
  1036. next_url,
  1037. r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
  1038. 0,
  1039. [
  1040. (1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
  1041. ]
  1042. )
  1043. next_url = compat_urllib_parse.unquote(info.get('url'))
  1044. info = self.grep_webpage(
  1045. next_url,
  1046. r'<video id="(.*?)".*?>.*?' +
  1047. '<name>(.*?)</name>.*?' +
  1048. '<dateVideo>(.*?)</dateVideo>.*?' +
  1049. '<url quality="hd">(.*?)</url>',
  1050. re.DOTALL,
  1051. [
  1052. (1, 'id', u'ERROR: could not extract video id: %s' % url),
  1053. (2, 'title', u'ERROR: could not extract video title: %s' % url),
  1054. (3, 'date', u'ERROR: could not extract video date: %s' % url),
  1055. (4, 'url', u'ERROR: could not extract video url: %s' % url)
  1056. ]
  1057. )
  1058. return {
  1059. 'id': info.get('id'),
  1060. 'url': compat_urllib_parse.unquote(info.get('url')),
  1061. 'uploader': u'arte.tv',
  1062. 'upload_date': info.get('date'),
  1063. 'title': info.get('title').decode('utf-8'),
  1064. 'ext': u'mp4',
  1065. 'format': u'NA',
  1066. 'player_url': None,
  1067. }
  1068. def _real_extract(self, url):
  1069. video_id = url.split('/')[-1]
  1070. self.report_extraction(video_id)
  1071. if re.search(self._LIVE_URL, video_id) is not None:
  1072. self.extractLiveStream(url)
  1073. return
  1074. else:
  1075. info = self.extractPlus7Stream(url)
  1076. return [info]
  1077. class GenericIE(InfoExtractor):
  1078. """Generic last-resort information extractor."""
  1079. _VALID_URL = r'.*'
  1080. IE_NAME = u'generic'
  1081. def __init__(self, downloader=None):
  1082. InfoExtractor.__init__(self, downloader)
  1083. def report_download_webpage(self, video_id):
  1084. """Report webpage download."""
  1085. self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
  1086. self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
  1087. def report_extraction(self, video_id):
  1088. """Report information extraction."""
  1089. self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
  1090. def report_following_redirect(self, new_url):
  1091. """Report information extraction."""
  1092. self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
  1093. def _test_redirect(self, url):
  1094. """Check if it is a redirect, like url shorteners, in case restart chain."""
  1095. class HeadRequest(compat_urllib_request.Request):
  1096. def get_method(self):
  1097. return "HEAD"
  1098. class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
  1099. """
  1100. Subclass the HTTPRedirectHandler to make it use our
  1101. HeadRequest also on the redirected URL
  1102. """
  1103. def redirect_request(self, req, fp, code, msg, headers, newurl):
  1104. if code in (301, 302, 303, 307):
  1105. newurl = newurl.replace(' ', '%20')
  1106. newheaders = dict((k,v) for k,v in req.headers.items()
  1107. if k.lower() not in ("content-length", "content-type"))
  1108. return HeadRequest(newurl,
  1109. headers=newheaders,
  1110. origin_req_host=req.get_origin_req_host(),
  1111. unverifiable=True)
  1112. else:
  1113. raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp)
  1114. class HTTPMethodFallback(compat_urllib_request.BaseHandler):
  1115. """
  1116. Fallback to GET if HEAD is not allowed (405 HTTP error)
  1117. """
  1118. def http_error_405(self, req, fp, code, msg, headers):
  1119. fp.read()
  1120. fp.close()
  1121. newheaders = dict((k,v) for k,v in req.headers.items()
  1122. if k.lower() not in ("content-length", "content-type"))
  1123. return self.parent.open(compat_urllib_request.Request(req.get_full_url(),
  1124. headers=newheaders,
  1125. origin_req_host=req.get_origin_req_host(),
  1126. unverifiable=True))
  1127. # Build our opener
  1128. opener = compat_urllib_request.OpenerDirector()
  1129. for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler,
  1130. HTTPMethodFallback, HEADRedirectHandler,
  1131. compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
  1132. opener.add_handler(handler())
  1133. response = opener.open(HeadRequest(url))
  1134. new_url = response.geturl()
  1135. if url == new_url:
  1136. return False
  1137. self.report_following_redirect(new_url)
  1138. self._downloader.download([new_url])
  1139. return True
  1140. def _real_extract(self, url):
  1141. if self._test_redirect(url): return
  1142. video_id = url.split('/')[-1]
  1143. request = compat_urllib_request.Request(url)
  1144. try:
  1145. self.report_download_webpage(video_id)
  1146. webpage = compat_urllib_request.urlopen(request).read()
  1147. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1148. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  1149. return
  1150. except ValueError as err:
  1151. # since this is the last-resort InfoExtractor, if
  1152. # this error is thrown, it'll be thrown here
  1153. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  1154. return
  1155. self.report_extraction(video_id)
  1156. # Start with something easy: JW Player in SWFObject
  1157. mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
  1158. if mobj is None:
  1159. # Broaden the search a little bit
  1160. mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
  1161. if mobj is None:
  1162. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  1163. return
  1164. # It's possible that one of the regexes
  1165. # matched, but returned an empty group:
  1166. if mobj.group(1) is None:
  1167. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  1168. return
  1169. video_url = compat_urllib_parse.unquote(mobj.group(1))
  1170. video_id = os.path.basename(video_url)
  1171. # here's a fun little line of code for you:
  1172. video_extension = os.path.splitext(video_id)[1][1:]
  1173. video_id = os.path.splitext(video_id)[0]
  1174. # it's tempting to parse this further, but you would
  1175. # have to take into account all the variations like
  1176. # Video Title - Site Name
  1177. # Site Name | Video Title
  1178. # Video Title - Tagline | Site Name
  1179. # and so on and so forth; it's just not practical
  1180. mobj = re.search(r'<title>(.*)</title>', webpage)
  1181. if mobj is None:
  1182. self._downloader.trouble(u'ERROR: unable to extract title')
  1183. return
  1184. video_title = mobj.group(1)
  1185. # video uploader is domain name
  1186. mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
  1187. if mobj is None:
  1188. self._downloader.trouble(u'ERROR: unable to extract title')
  1189. return
  1190. video_uploader = mobj.group(1)
  1191. return [{
  1192. 'id': video_id,
  1193. 'url': video_url,
  1194. 'uploader': video_uploader,
  1195. 'upload_date': None,
  1196. 'title': video_title,
  1197. 'ext': video_extension,
  1198. }]
  1199. class YoutubeSearchIE(InfoExtractor):
  1200. """Information Extractor for YouTube search queries."""
  1201. _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
  1202. _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
  1203. _max_youtube_results = 1000
  1204. IE_NAME = u'youtube:search'
  1205. def __init__(self, downloader=None):
  1206. InfoExtractor.__init__(self, downloader)
  1207. def report_download_page(self, query, pagenum):
  1208. """Report attempt to download search page with given number."""
  1209. query = query.decode(preferredencoding())
  1210. self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
  1211. def _real_extract(self, query):
  1212. mobj = re.match(self._VALID_URL, query)
  1213. if mobj is None:
  1214. self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
  1215. return
  1216. prefix, query = query.split(':')
  1217. prefix = prefix[8:]
  1218. query = query.encode('utf-8')
  1219. if prefix == '':
  1220. self._download_n_results(query, 1)
  1221. return
  1222. elif prefix == 'all':
  1223. self._download_n_results(query, self._max_youtube_results)
  1224. return
  1225. else:
  1226. try:
  1227. n = int(prefix)
  1228. if n <= 0:
  1229. self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
  1230. return
  1231. elif n > self._max_youtube_results:
  1232. self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
  1233. n = self._max_youtube_results
  1234. self._download_n_results(query, n)
  1235. return
  1236. except ValueError: # parsing prefix as integer fails
  1237. self._download_n_results(query, 1)
  1238. return
  1239. def _download_n_results(self, query, n):
  1240. """Downloads a specified number of results for a query"""
  1241. video_ids = []
  1242. pagenum = 0
  1243. limit = n
  1244. while (50 * pagenum) < limit:
  1245. self.report_download_page(query, pagenum+1)
  1246. result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
  1247. request = compat_urllib_request.Request(result_url)
  1248. try:
  1249. data = compat_urllib_request.urlopen(request).read()
  1250. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1251. self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
  1252. return
  1253. api_response = json.loads(data)['data']
  1254. new_ids = list(video['id'] for video in api_response['items'])
  1255. video_ids += new_ids
  1256. limit = min(n, api_response['totalItems'])
  1257. pagenum += 1
  1258. if len(video_ids) > n:
  1259. video_ids = video_ids[:n]
  1260. for id in video_ids:
  1261. self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
  1262. return
  1263. class GoogleSearchIE(InfoExtractor):
  1264. """Information Extractor for Google Video search queries."""
  1265. _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
  1266. _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
  1267. _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
  1268. _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
  1269. _max_google_results = 1000
  1270. IE_NAME = u'video.google:search'
  1271. def __init__(self, downloader=None):
  1272. InfoExtractor.__init__(self, downloader)
  1273. def report_download_page(self, query, pagenum):
  1274. """Report attempt to download playlist page with given number."""
  1275. query = query.decode(preferredencoding())
  1276. self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
  1277. def _real_extract(self, query):
  1278. mobj = re.match(self._VALID_URL, query)
  1279. if mobj is None:
  1280. self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
  1281. return
  1282. prefix, query = query.split(':')
  1283. prefix = prefix[8:]
  1284. query = query.encode('utf-8')
  1285. if prefix == '':
  1286. self._download_n_results(query, 1)
  1287. return
  1288. elif prefix == 'all':
  1289. self._download_n_results(query, self._max_google_results)
  1290. return
  1291. else:
  1292. try:
  1293. n = int(prefix)
  1294. if n <= 0:
  1295. self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
  1296. return
  1297. elif n > self._max_google_results:
  1298. self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
  1299. n = self._max_google_results
  1300. self._download_n_results(query, n)
  1301. return
  1302. except ValueError: # parsing prefix as integer fails
  1303. self._download_n_results(query, 1)
  1304. return
  1305. def _download_n_results(self, query, n):
  1306. """Downloads a specified number of results for a query"""
  1307. video_ids = []
  1308. pagenum = 0
  1309. while True:
  1310. self.report_download_page(query, pagenum)
  1311. result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum*10)
  1312. request = compat_urllib_request.Request(result_url)
  1313. try:
  1314. page = compat_urllib_request.urlopen(request).read()
  1315. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1316. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1317. return
  1318. # Extract video identifiers
  1319. for mobj in re.finditer(self._VIDEO_INDICATOR, page):
  1320. video_id = mobj.group(1)
  1321. if video_id not in video_ids:
  1322. video_ids.append(video_id)
  1323. if len(video_ids) == n:
  1324. # Specified n videos reached
  1325. for id in video_ids:
  1326. self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
  1327. return
  1328. if re.search(self._MORE_PAGES_INDICATOR, page) is None:
  1329. for id in video_ids:
  1330. self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
  1331. return
  1332. pagenum = pagenum + 1
  1333. class YahooSearchIE(InfoExtractor):
  1334. """Information Extractor for Yahoo! Video search queries."""
  1335. _WORKING = False
  1336. _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
  1337. _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
  1338. _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
  1339. _MORE_PAGES_INDICATOR = r'\s*Next'
  1340. _max_yahoo_results = 1000
  1341. IE_NAME = u'video.yahoo:search'
  1342. def __init__(self, downloader=None):
  1343. InfoExtractor.__init__(self, downloader)
  1344. def report_download_page(self, query, pagenum):
  1345. """Report attempt to download playlist page with given number."""
  1346. query = query.decode(preferredencoding())
  1347. self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
  1348. def _real_extract(self, query):
  1349. mobj = re.match(self._VALID_URL, query)
  1350. if mobj is None:
  1351. self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
  1352. return
  1353. prefix, query = query.split(':')
  1354. prefix = prefix[8:]
  1355. query = query.encode('utf-8')
  1356. if prefix == '':
  1357. self._download_n_results(query, 1)
  1358. return
  1359. elif prefix == 'all':
  1360. self._download_n_results(query, self._max_yahoo_results)
  1361. return
  1362. else:
  1363. try:
  1364. n = int(prefix)
  1365. if n <= 0:
  1366. self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
  1367. return
  1368. elif n > self._max_yahoo_results:
  1369. self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
  1370. n = self._max_yahoo_results
  1371. self._download_n_results(query, n)
  1372. return
  1373. except ValueError: # parsing prefix as integer fails
  1374. self._download_n_results(query, 1)
  1375. return
  1376. def _download_n_results(self, query, n):
  1377. """Downloads a specified number of results for a query"""
  1378. video_ids = []
  1379. already_seen = set()
  1380. pagenum = 1
  1381. while True:
  1382. self.report_download_page(query, pagenum)
  1383. result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum)
  1384. request = compat_urllib_request.Request(result_url)
  1385. try:
  1386. page = compat_urllib_request.urlopen(request).read()
  1387. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1388. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1389. return
  1390. # Extract video identifiers
  1391. for mobj in re.finditer(self._VIDEO_INDICATOR, page):
  1392. video_id = mobj.group(1)
  1393. if video_id not in already_seen:
  1394. video_ids.append(video_id)
  1395. already_seen.add(video_id)
  1396. if len(video_ids) == n:
  1397. # Specified n videos reached
  1398. for id in video_ids:
  1399. self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
  1400. return
  1401. if re.search(self._MORE_PAGES_INDICATOR, page) is None:
  1402. for id in video_ids:
  1403. self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
  1404. return
  1405. pagenum = pagenum + 1
  1406. class YoutubePlaylistIE(InfoExtractor):
  1407. """Information Extractor for YouTube playlists."""
  1408. _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
  1409. _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
  1410. _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;([^&"]+&amp;)*list=.*?%s'
  1411. _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
  1412. IE_NAME = u'youtube:playlist'
  1413. def __init__(self, downloader=None):
  1414. InfoExtractor.__init__(self, downloader)
  1415. def report_download_page(self, playlist_id, pagenum):
  1416. """Report attempt to download playlist page with given number."""
  1417. self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
  1418. def _real_extract(self, url):
  1419. # Extract playlist id
  1420. mobj = re.match(self._VALID_URL, url)
  1421. if mobj is None:
  1422. self._downloader.trouble(u'ERROR: invalid url: %s' % url)
  1423. return
  1424. # Single video case
  1425. if mobj.group(3) is not None:
  1426. self._downloader.download([mobj.group(3)])
  1427. return
  1428. # Download playlist pages
  1429. # prefix is 'p' as default for playlists but there are other types that need extra care
  1430. playlist_prefix = mobj.group(1)
  1431. if playlist_prefix == 'a':
  1432. playlist_access = 'artist'
  1433. else:
  1434. playlist_prefix = 'p'
  1435. playlist_access = 'view_play_list'
  1436. playlist_id = mobj.group(2)
  1437. video_ids = []
  1438. pagenum = 1
  1439. while True:
  1440. self.report_download_page(playlist_id, pagenum)
  1441. url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
  1442. request = compat_urllib_request.Request(url)
  1443. try:
  1444. page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  1445. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1446. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1447. return
  1448. # Extract video identifiers
  1449. ids_in_page = []
  1450. for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page):
  1451. if mobj.group(1) not in ids_in_page:
  1452. ids_in_page.append(mobj.group(1))
  1453. video_ids.extend(ids_in_page)
  1454. if self._MORE_PAGES_INDICATOR not in page:
  1455. break
  1456. pagenum = pagenum + 1
  1457. total = len(video_ids)
  1458. playliststart = self._downloader.params.get('playliststart', 1) - 1
  1459. playlistend = self._downloader.params.get('playlistend', -1)
  1460. if playlistend == -1:
  1461. video_ids = video_ids[playliststart:]
  1462. else:
  1463. video_ids = video_ids[playliststart:playlistend]
  1464. if len(video_ids) == total:
  1465. self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total))
  1466. else:
  1467. self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids)))
  1468. for id in video_ids:
  1469. self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
  1470. return
  1471. class YoutubeChannelIE(InfoExtractor):
  1472. """Information Extractor for YouTube channels."""
  1473. _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
  1474. _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
  1475. _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
  1476. IE_NAME = u'youtube:channel'
  1477. def report_download_page(self, channel_id, pagenum):
  1478. """Report attempt to download channel page with given number."""
  1479. self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
  1480. def _real_extract(self, url):
  1481. # Extract channel id
  1482. mobj = re.match(self._VALID_URL, url)
  1483. if mobj is None:
  1484. self._downloader.trouble(u'ERROR: invalid url: %s' % url)
  1485. return
  1486. # Download channel pages
  1487. channel_id = mobj.group(1)
  1488. video_ids = []
  1489. pagenum = 1
  1490. while True:
  1491. self.report_download_page(channel_id, pagenum)
  1492. url = self._TEMPLATE_URL % (channel_id, pagenum)
  1493. request = compat_urllib_request.Request(url)
  1494. try:
  1495. page = compat_urllib_request.urlopen(request).read().decode('utf8')
  1496. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1497. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1498. return
  1499. # Extract video identifiers
  1500. ids_in_page = []
  1501. for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
  1502. if mobj.group(1) not in ids_in_page:
  1503. ids_in_page.append(mobj.group(1))
  1504. video_ids.extend(ids_in_page)
  1505. if self._MORE_PAGES_INDICATOR not in page:
  1506. break
  1507. pagenum = pagenum + 1
  1508. self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
  1509. for id in video_ids:
  1510. self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
  1511. return
  1512. class YoutubeUserIE(InfoExtractor):
  1513. """Information Extractor for YouTube users."""
  1514. _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
  1515. _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
  1516. _GDATA_PAGE_SIZE = 50
  1517. _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
  1518. _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
  1519. IE_NAME = u'youtube:user'
  1520. def __init__(self, downloader=None):
  1521. InfoExtractor.__init__(self, downloader)
  1522. def report_download_page(self, username, start_index):
  1523. """Report attempt to download user page."""
  1524. self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
  1525. (username, start_index, start_index + self._GDATA_PAGE_SIZE))
  1526. def _real_extract(self, url):
  1527. # Extract username
  1528. mobj = re.match(self._VALID_URL, url)
  1529. if mobj is None:
  1530. self._downloader.trouble(u'ERROR: invalid url: %s' % url)
  1531. return
  1532. username = mobj.group(1)
  1533. # Download video ids using YouTube Data API. Result size per
  1534. # query is limited (currently to 50 videos) so we need to query
  1535. # page by page until there are no video ids - it means we got
  1536. # all of them.
  1537. video_ids = []
  1538. pagenum = 0
  1539. while True:
  1540. start_index = pagenum * self._GDATA_PAGE_SIZE + 1
  1541. self.report_download_page(username, start_index)
  1542. request = compat_urllib_request.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
  1543. try:
  1544. page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  1545. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1546. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1547. return
  1548. # Extract video identifiers
  1549. ids_in_page = []
  1550. for mobj in re.finditer(self._VIDEO_INDICATOR, page):
  1551. if mobj.group(1) not in ids_in_page:
  1552. ids_in_page.append(mobj.group(1))
  1553. video_ids.extend(ids_in_page)
  1554. # A little optimization - if current page is not
  1555. # "full", ie. does not contain PAGE_SIZE video ids then
  1556. # we can assume that this page is the last one - there
  1557. # are no more ids on further pages - no need to query
  1558. # again.
  1559. if len(ids_in_page) < self._GDATA_PAGE_SIZE:
  1560. break
  1561. pagenum += 1
  1562. all_ids_count = len(video_ids)
  1563. playliststart = self._downloader.params.get('playliststart', 1) - 1
  1564. playlistend = self._downloader.params.get('playlistend', -1)
  1565. if playlistend == -1:
  1566. video_ids = video_ids[playliststart:]
  1567. else:
  1568. video_ids = video_ids[playliststart:playlistend]
  1569. self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
  1570. (username, all_ids_count, len(video_ids)))
  1571. for video_id in video_ids:
  1572. self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
  1573. class BlipTVUserIE(InfoExtractor):
  1574. """Information Extractor for blip.tv users."""
  1575. _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
  1576. _PAGE_SIZE = 12
  1577. IE_NAME = u'blip.tv:user'
  1578. def __init__(self, downloader=None):
  1579. InfoExtractor.__init__(self, downloader)
  1580. def report_download_page(self, username, pagenum):
  1581. """Report attempt to download user page."""
  1582. self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
  1583. (self.IE_NAME, username, pagenum))
  1584. def _real_extract(self, url):
  1585. # Extract username
  1586. mobj = re.match(self._VALID_URL, url)
  1587. if mobj is None:
  1588. self._downloader.trouble(u'ERROR: invalid url: %s' % url)
  1589. return
  1590. username = mobj.group(1)
  1591. page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
  1592. request = compat_urllib_request.Request(url)
  1593. try:
  1594. page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  1595. mobj = re.search(r'data-users-id="([^"]+)"', page)
  1596. page_base = page_base % mobj.group(1)
  1597. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1598. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1599. return
  1600. # Download video ids using BlipTV Ajax calls. Result size per
  1601. # query is limited (currently to 12 videos) so we need to query
  1602. # page by page until there are no video ids - it means we got
  1603. # all of them.
  1604. video_ids = []
  1605. pagenum = 1
  1606. while True:
  1607. self.report_download_page(username, pagenum)
  1608. request = compat_urllib_request.Request( page_base + "&page=" + str(pagenum) )
  1609. try:
  1610. page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  1611. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1612. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
  1613. return
  1614. # Extract video identifiers
  1615. ids_in_page = []
  1616. for mobj in re.finditer(r'href="/([^"]+)"', page):
  1617. if mobj.group(1) not in ids_in_page:
  1618. ids_in_page.append(unescapeHTML(mobj.group(1)))
  1619. video_ids.extend(ids_in_page)
  1620. # A little optimization - if current page is not
  1621. # "full", ie. does not contain PAGE_SIZE video ids then
  1622. # we can assume that this page is the last one - there
  1623. # are no more ids on further pages - no need to query
  1624. # again.
  1625. if len(ids_in_page) < self._PAGE_SIZE:
  1626. break
  1627. pagenum += 1
  1628. all_ids_count = len(video_ids)
  1629. playliststart = self._downloader.params.get('playliststart', 1) - 1
  1630. playlistend = self._downloader.params.get('playlistend', -1)
  1631. if playlistend == -1:
  1632. video_ids = video_ids[playliststart:]
  1633. else:
  1634. video_ids = video_ids[playliststart:playlistend]
  1635. self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
  1636. (self.IE_NAME, username, all_ids_count, len(video_ids)))
  1637. for video_id in video_ids:
  1638. self._downloader.download([u'http://blip.tv/'+video_id])
  1639. class DepositFilesIE(InfoExtractor):
  1640. """Information extractor for depositfiles.com"""
  1641. _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
  1642. def report_download_webpage(self, file_id):
  1643. """Report webpage download."""
  1644. self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
  1645. def report_extraction(self, file_id):
  1646. """Report information extraction."""
  1647. self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
  1648. def _real_extract(self, url):
  1649. file_id = url.split('/')[-1]
  1650. # Rebuild url in english locale
  1651. url = 'http://depositfiles.com/en/files/' + file_id
  1652. # Retrieve file webpage with 'Free download' button pressed
  1653. free_download_indication = { 'gateway_result' : '1' }
  1654. request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication))
  1655. try:
  1656. self.report_download_webpage(file_id)
  1657. webpage = compat_urllib_request.urlopen(request).read()
  1658. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1659. self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
  1660. return
  1661. # Search for the real file URL
  1662. mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
  1663. if (mobj is None) or (mobj.group(1) is None):
  1664. # Try to figure out reason of the error.
  1665. mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
  1666. if (mobj is not None) and (mobj.group(1) is not None):
  1667. restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
  1668. self._downloader.trouble(u'ERROR: %s' % restriction_message)
  1669. else:
  1670. self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
  1671. return
  1672. file_url = mobj.group(1)
  1673. file_extension = os.path.splitext(file_url)[1][1:]
  1674. # Search for file title
  1675. mobj = re.search(r'<b title="(.*?)">', webpage)
  1676. if mobj is None:
  1677. self._downloader.trouble(u'ERROR: unable to extract title')
  1678. return
  1679. file_title = mobj.group(1).decode('utf-8')
  1680. return [{
  1681. 'id': file_id.decode('utf-8'),
  1682. 'url': file_url.decode('utf-8'),
  1683. 'uploader': None,
  1684. 'upload_date': None,
  1685. 'title': file_title,
  1686. 'ext': file_extension.decode('utf-8'),
  1687. }]
  1688. class FacebookIE(InfoExtractor):
  1689. """Information Extractor for Facebook"""
  1690. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
  1691. _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
  1692. _NETRC_MACHINE = 'facebook'
  1693. IE_NAME = u'facebook'
  1694. def report_login(self):
  1695. """Report attempt to log in."""
  1696. self._downloader.to_screen(u'[%s] Logging in' % self.IE_NAME)
  1697. def _real_initialize(self):
  1698. if self._downloader is None:
  1699. return
  1700. useremail = None
  1701. password = None
  1702. downloader_params = self._downloader.params
  1703. # Attempt to use provided username and password or .netrc data
  1704. if downloader_params.get('username', None) is not None:
  1705. useremail = downloader_params['username']
  1706. password = downloader_params['password']
  1707. elif downloader_params.get('usenetrc', False):
  1708. try:
  1709. info = netrc.netrc().authenticators(self._NETRC_MACHINE)
  1710. if info is not None:
  1711. useremail = info[0]
  1712. password = info[2]
  1713. else:
  1714. raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
  1715. except (IOError, netrc.NetrcParseError) as err:
  1716. self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
  1717. return
  1718. if useremail is None:
  1719. return
  1720. # Log in
  1721. login_form = {
  1722. 'email': useremail,
  1723. 'pass': password,
  1724. 'login': 'Log+In'
  1725. }
  1726. request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
  1727. try:
  1728. self.report_login()
  1729. login_results = compat_urllib_request.urlopen(request).read()
  1730. if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
  1731. self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
  1732. return
  1733. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1734. self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
  1735. return
  1736. def _real_extract(self, url):
  1737. mobj = re.match(self._VALID_URL, url)
  1738. if mobj is None:
  1739. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  1740. return
  1741. video_id = mobj.group('ID')
  1742. url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
  1743. webpage = self._download_webpage(url, video_id)
  1744. BEFORE = '[["allowFullScreen","true"],["allowScriptAccess","always"],["salign","tl"],["scale","noscale"],["wmode","opaque"]].forEach(function(param) {swf.addParam(param[0], param[1]);});\n'
  1745. AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
  1746. m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
  1747. if not m:
  1748. raise ExtractorError(u'Cannot parse data')
  1749. data = dict(json.loads(m.group(1)))
  1750. params_raw = compat_urllib_parse.unquote(data['params'])
  1751. params = json.loads(params_raw)
  1752. video_url = params['hd_src']
  1753. video_duration = int(params['video_duration'])
  1754. m = re.search('<h2 class="uiHeaderTitle">([^<]+)</h2>', webpage)
  1755. if not m:
  1756. raise ExtractorError(u'Cannot find title in webpage')
  1757. video_title = unescapeHTML(m.group(1))
  1758. info = {
  1759. 'id': video_id,
  1760. 'title': video_title,
  1761. 'url': video_url,
  1762. 'ext': 'mp4',
  1763. 'duration': video_duration,
  1764. 'thumbnail': params['thumbnail_src'],
  1765. }
  1766. return [info]
  1767. class BlipTVIE(InfoExtractor):
  1768. """Information extractor for blip.tv"""
  1769. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
  1770. _URL_EXT = r'^.*\.([a-z0-9]+)$'
  1771. IE_NAME = u'blip.tv'
  1772. def report_extraction(self, file_id):
  1773. """Report information extraction."""
  1774. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
  1775. def report_direct_download(self, title):
  1776. """Report information extraction."""
  1777. self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
  1778. def _real_extract(self, url):
  1779. mobj = re.match(self._VALID_URL, url)
  1780. if mobj is None:
  1781. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  1782. return
  1783. if '?' in url:
  1784. cchar = '&'
  1785. else:
  1786. cchar = '?'
  1787. json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
  1788. request = compat_urllib_request.Request(json_url)
  1789. request.add_header('User-Agent', 'iTunes/10.6.1')
  1790. self.report_extraction(mobj.group(1))
  1791. info = None
  1792. try:
  1793. urlh = compat_urllib_request.urlopen(request)
  1794. if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
  1795. basename = url.split('/')[-1]
  1796. title,ext = os.path.splitext(basename)
  1797. title = title.decode('UTF-8')
  1798. ext = ext.replace('.', '')
  1799. self.report_direct_download(title)
  1800. info = {
  1801. 'id': title,
  1802. 'url': url,
  1803. 'uploader': None,
  1804. 'upload_date': None,
  1805. 'title': title,
  1806. 'ext': ext,
  1807. 'urlhandle': urlh
  1808. }
  1809. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1810. raise ExtractorError(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
  1811. if info is None: # Regular URL
  1812. try:
  1813. json_code_bytes = urlh.read()
  1814. json_code = json_code_bytes.decode('utf-8')
  1815. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1816. self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
  1817. return
  1818. try:
  1819. json_data = json.loads(json_code)
  1820. if 'Post' in json_data:
  1821. data = json_data['Post']
  1822. else:
  1823. data = json_data
  1824. upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
  1825. video_url = data['media']['url']
  1826. umobj = re.match(self._URL_EXT, video_url)
  1827. if umobj is None:
  1828. raise ValueError('Can not determine filename extension')
  1829. ext = umobj.group(1)
  1830. info = {
  1831. 'id': data['item_id'],
  1832. 'url': video_url,
  1833. 'uploader': data['display_name'],
  1834. 'upload_date': upload_date,
  1835. 'title': data['title'],
  1836. 'ext': ext,
  1837. 'format': data['media']['mimeType'],
  1838. 'thumbnail': data['thumbnailUrl'],
  1839. 'description': data['description'],
  1840. 'player_url': data['embedUrl'],
  1841. 'user_agent': 'iTunes/10.6.1',
  1842. }
  1843. except (ValueError,KeyError) as err:
  1844. self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
  1845. return
  1846. return [info]
  1847. class MyVideoIE(InfoExtractor):
  1848. """Information Extractor for myvideo.de."""
  1849. _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
  1850. IE_NAME = u'myvideo'
  1851. def __init__(self, downloader=None):
  1852. InfoExtractor.__init__(self, downloader)
  1853. def report_extraction(self, video_id):
  1854. """Report information extraction."""
  1855. self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
  1856. def _real_extract(self,url):
  1857. mobj = re.match(self._VALID_URL, url)
  1858. if mobj is None:
  1859. self._download.trouble(u'ERROR: invalid URL: %s' % url)
  1860. return
  1861. video_id = mobj.group(1)
  1862. # Get video webpage
  1863. webpage_url = 'http://www.myvideo.de/watch/%s' % video_id
  1864. webpage = self._download_webpage(webpage_url, video_id)
  1865. self.report_extraction(video_id)
  1866. mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
  1867. webpage)
  1868. if mobj is None:
  1869. self._downloader.trouble(u'ERROR: unable to extract media URL')
  1870. return
  1871. video_url = mobj.group(1) + ('/%s.flv' % video_id)
  1872. mobj = re.search('<title>([^<]+)</title>', webpage)
  1873. if mobj is None:
  1874. self._downloader.trouble(u'ERROR: unable to extract title')
  1875. return
  1876. video_title = mobj.group(1)
  1877. return [{
  1878. 'id': video_id,
  1879. 'url': video_url,
  1880. 'uploader': None,
  1881. 'upload_date': None,
  1882. 'title': video_title,
  1883. 'ext': u'flv',
  1884. }]
  1885. class ComedyCentralIE(InfoExtractor):
  1886. """Information extractor for The Daily Show and Colbert Report """
  1887. # urls can be abbreviations like :thedailyshow or :colbert
  1888. # urls for episodes like:
  1889. # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
  1890. # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
  1891. # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
  1892. _VALID_URL = r"""^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport)
  1893. |(https?://)?(www\.)?
  1894. (?P<showname>thedailyshow|colbertnation)\.com/
  1895. (full-episodes/(?P<episode>.*)|
  1896. (?P<clip>
  1897. (the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
  1898. |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*)))))
  1899. $"""
  1900. _available_formats = ['3500', '2200', '1700', '1200', '750', '400']
  1901. _video_extensions = {
  1902. '3500': 'mp4',
  1903. '2200': 'mp4',
  1904. '1700': 'mp4',
  1905. '1200': 'mp4',
  1906. '750': 'mp4',
  1907. '400': 'mp4',
  1908. }
  1909. _video_dimensions = {
  1910. '3500': '1280x720',
  1911. '2200': '960x540',
  1912. '1700': '768x432',
  1913. '1200': '640x360',
  1914. '750': '512x288',
  1915. '400': '384x216',
  1916. }
  1917. def suitable(self, url):
  1918. """Receives a URL and returns True if suitable for this IE."""
  1919. return re.match(self._VALID_URL, url, re.VERBOSE) is not None
  1920. def report_extraction(self, episode_id):
  1921. self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
  1922. def report_config_download(self, episode_id, media_id):
  1923. self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration for %s' % (episode_id, media_id))
  1924. def report_index_download(self, episode_id):
  1925. self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
  1926. def _print_formats(self, formats):
  1927. print('Available formats:')
  1928. for x in formats:
  1929. print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
  1930. def _real_extract(self, url):
  1931. mobj = re.match(self._VALID_URL, url, re.VERBOSE)
  1932. if mobj is None:
  1933. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  1934. return
  1935. if mobj.group('shortname'):
  1936. if mobj.group('shortname') in ('tds', 'thedailyshow'):
  1937. url = u'http://www.thedailyshow.com/full-episodes/'
  1938. else:
  1939. url = u'http://www.colbertnation.com/full-episodes/'
  1940. mobj = re.match(self._VALID_URL, url, re.VERBOSE)
  1941. assert mobj is not None
  1942. if mobj.group('clip'):
  1943. if mobj.group('showname') == 'thedailyshow':
  1944. epTitle = mobj.group('tdstitle')
  1945. else:
  1946. epTitle = mobj.group('cntitle')
  1947. dlNewest = False
  1948. else:
  1949. dlNewest = not mobj.group('episode')
  1950. if dlNewest:
  1951. epTitle = mobj.group('showname')
  1952. else:
  1953. epTitle = mobj.group('episode')
  1954. req = compat_urllib_request.Request(url)
  1955. self.report_extraction(epTitle)
  1956. try:
  1957. htmlHandle = compat_urllib_request.urlopen(req)
  1958. html = htmlHandle.read()
  1959. webpage = html.decode('utf-8')
  1960. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1961. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1962. return
  1963. if dlNewest:
  1964. url = htmlHandle.geturl()
  1965. mobj = re.match(self._VALID_URL, url, re.VERBOSE)
  1966. if mobj is None:
  1967. self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
  1968. return
  1969. if mobj.group('episode') == '':
  1970. self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
  1971. return
  1972. epTitle = mobj.group('episode')
  1973. mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', webpage)
  1974. if len(mMovieParams) == 0:
  1975. # The Colbert Report embeds the information in a without
  1976. # a URL prefix; so extract the alternate reference
  1977. # and then add the URL prefix manually.
  1978. altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video).*?:.*?)"', webpage)
  1979. if len(altMovieParams) == 0:
  1980. self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
  1981. return
  1982. else:
  1983. mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
  1984. uri = mMovieParams[0][1]
  1985. indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri})
  1986. self.report_index_download(epTitle)
  1987. try:
  1988. indexXml = compat_urllib_request.urlopen(indexUrl).read()
  1989. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1990. self._downloader.trouble(u'ERROR: unable to download episode index: ' + compat_str(err))
  1991. return
  1992. results = []
  1993. idoc = xml.etree.ElementTree.fromstring(indexXml)
  1994. itemEls = idoc.findall('.//item')
  1995. for partNum,itemEl in enumerate(itemEls):
  1996. mediaId = itemEl.findall('./guid')[0].text
  1997. shortMediaId = mediaId.split(':')[-1]
  1998. showId = mediaId.split(':')[-2].replace('.com', '')
  1999. officialTitle = itemEl.findall('./title')[0].text
  2000. officialDate = itemEl.findall('./pubDate')[0].text
  2001. configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
  2002. compat_urllib_parse.urlencode({'uri': mediaId}))
  2003. configReq = compat_urllib_request.Request(configUrl)
  2004. self.report_config_download(epTitle, shortMediaId)
  2005. try:
  2006. configXml = compat_urllib_request.urlopen(configReq).read()
  2007. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2008. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  2009. return
  2010. cdoc = xml.etree.ElementTree.fromstring(configXml)
  2011. turls = []
  2012. for rendition in cdoc.findall('.//rendition'):
  2013. finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
  2014. turls.append(finfo)
  2015. if len(turls) == 0:
  2016. self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
  2017. continue
  2018. if self._downloader.params.get('listformats', None):
  2019. self._print_formats([i[0] for i in turls])
  2020. return
  2021. # For now, just pick the highest bitrate
  2022. format,rtmp_video_url = turls[-1]
  2023. # Get the format arg from the arg stream
  2024. req_format = self._downloader.params.get('format', None)
  2025. # Select format if we can find one
  2026. for f,v in turls:
  2027. if f == req_format:
  2028. format, rtmp_video_url = f, v
  2029. break
  2030. m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url)
  2031. if not m:
  2032. raise ExtractorError(u'Cannot transform RTMP url')
  2033. base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
  2034. video_url = base + m.group('finalid')
  2035. effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1)
  2036. info = {
  2037. 'id': shortMediaId,
  2038. 'url': video_url,
  2039. 'uploader': showId,
  2040. 'upload_date': officialDate,
  2041. 'title': effTitle,
  2042. 'ext': 'mp4',
  2043. 'format': format,
  2044. 'thumbnail': None,
  2045. 'description': officialTitle,
  2046. }
  2047. results.append(info)
  2048. return results
  2049. class EscapistIE(InfoExtractor):
  2050. """Information extractor for The Escapist """
  2051. _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
  2052. IE_NAME = u'escapist'
  2053. def report_extraction(self, showName):
  2054. self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
  2055. def report_config_download(self, showName):
  2056. self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
  2057. def _real_extract(self, url):
  2058. mobj = re.match(self._VALID_URL, url)
  2059. if mobj is None:
  2060. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2061. return
  2062. showName = mobj.group('showname')
  2063. videoId = mobj.group('episode')
  2064. self.report_extraction(showName)
  2065. try:
  2066. webPage = compat_urllib_request.urlopen(url)
  2067. webPageBytes = webPage.read()
  2068. m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
  2069. webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
  2070. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2071. self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err))
  2072. return
  2073. descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
  2074. description = unescapeHTML(descMatch.group(1))
  2075. imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
  2076. imgUrl = unescapeHTML(imgMatch.group(1))
  2077. playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
  2078. playerUrl = unescapeHTML(playerUrlMatch.group(1))
  2079. configUrlMatch = re.search('config=(.*)$', playerUrl)
  2080. configUrl = compat_urllib_parse.unquote(configUrlMatch.group(1))
  2081. self.report_config_download(showName)
  2082. try:
  2083. configJSON = compat_urllib_request.urlopen(configUrl)
  2084. m = re.match(r'text/html; charset="?([^"]+)"?', configJSON.headers['Content-Type'])
  2085. configJSON = configJSON.read().decode(m.group(1) if m else 'utf-8')
  2086. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2087. self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err))
  2088. return
  2089. # Technically, it's JavaScript, not JSON
  2090. configJSON = configJSON.replace("'", '"')
  2091. try:
  2092. config = json.loads(configJSON)
  2093. except (ValueError,) as err:
  2094. self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + compat_str(err))
  2095. return
  2096. playlist = config['playlist']
  2097. videoUrl = playlist[1]['url']
  2098. info = {
  2099. 'id': videoId,
  2100. 'url': videoUrl,
  2101. 'uploader': showName,
  2102. 'upload_date': None,
  2103. 'title': showName,
  2104. 'ext': 'flv',
  2105. 'thumbnail': imgUrl,
  2106. 'description': description,
  2107. 'player_url': playerUrl,
  2108. }
  2109. return [info]
  2110. class CollegeHumorIE(InfoExtractor):
  2111. """Information extractor for collegehumor.com"""
  2112. _WORKING = False
  2113. _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
  2114. IE_NAME = u'collegehumor'
  2115. def report_manifest(self, video_id):
  2116. """Report information extraction."""
  2117. self._downloader.to_screen(u'[%s] %s: Downloading XML manifest' % (self.IE_NAME, video_id))
  2118. def report_extraction(self, video_id):
  2119. """Report information extraction."""
  2120. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2121. def _real_extract(self, url):
  2122. mobj = re.match(self._VALID_URL, url)
  2123. if mobj is None:
  2124. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2125. return
  2126. video_id = mobj.group('videoid')
  2127. info = {
  2128. 'id': video_id,
  2129. 'uploader': None,
  2130. 'upload_date': None,
  2131. }
  2132. self.report_extraction(video_id)
  2133. xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id
  2134. try:
  2135. metaXml = compat_urllib_request.urlopen(xmlUrl).read()
  2136. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2137. self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
  2138. return
  2139. mdoc = xml.etree.ElementTree.fromstring(metaXml)
  2140. try:
  2141. videoNode = mdoc.findall('./video')[0]
  2142. info['description'] = videoNode.findall('./description')[0].text
  2143. info['title'] = videoNode.findall('./caption')[0].text
  2144. info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
  2145. manifest_url = videoNode.findall('./file')[0].text
  2146. except IndexError:
  2147. self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
  2148. return
  2149. manifest_url += '?hdcore=2.10.3'
  2150. self.report_manifest(video_id)
  2151. try:
  2152. manifestXml = compat_urllib_request.urlopen(manifest_url).read()
  2153. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2154. self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
  2155. return
  2156. adoc = xml.etree.ElementTree.fromstring(manifestXml)
  2157. try:
  2158. media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0]
  2159. node_id = media_node.attrib['url']
  2160. video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
  2161. except IndexError as err:
  2162. self._downloader.trouble(u'\nERROR: Invalid manifest file')
  2163. return
  2164. url_pr = compat_urllib_parse_urlparse(manifest_url)
  2165. url = url_pr.scheme + '://' + url_pr.netloc + '/z' + video_id[:-2] + '/' + node_id + 'Seg1-Frag1'
  2166. info['url'] = url
  2167. info['ext'] = 'f4f'
  2168. return [info]
  2169. class XVideosIE(InfoExtractor):
  2170. """Information extractor for xvideos.com"""
  2171. _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
  2172. IE_NAME = u'xvideos'
  2173. def report_extraction(self, video_id):
  2174. """Report information extraction."""
  2175. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2176. def _real_extract(self, url):
  2177. mobj = re.match(self._VALID_URL, url)
  2178. if mobj is None:
  2179. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2180. return
  2181. video_id = mobj.group(1)
  2182. webpage = self._download_webpage(url, video_id)
  2183. self.report_extraction(video_id)
  2184. # Extract video URL
  2185. mobj = re.search(r'flv_url=(.+?)&', webpage)
  2186. if mobj is None:
  2187. self._downloader.trouble(u'ERROR: unable to extract video url')
  2188. return
  2189. video_url = compat_urllib_parse.unquote(mobj.group(1))
  2190. # Extract title
  2191. mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
  2192. if mobj is None:
  2193. self._downloader.trouble(u'ERROR: unable to extract video title')
  2194. return
  2195. video_title = mobj.group(1)
  2196. # Extract video thumbnail
  2197. mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
  2198. if mobj is None:
  2199. self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
  2200. return
  2201. video_thumbnail = mobj.group(0)
  2202. info = {
  2203. 'id': video_id,
  2204. 'url': video_url,
  2205. 'uploader': None,
  2206. 'upload_date': None,
  2207. 'title': video_title,
  2208. 'ext': 'flv',
  2209. 'thumbnail': video_thumbnail,
  2210. 'description': None,
  2211. }
  2212. return [info]
  2213. class SoundcloudIE(InfoExtractor):
  2214. """Information extractor for soundcloud.com
  2215. To access the media, the uid of the song and a stream token
  2216. must be extracted from the page source and the script must make
  2217. a request to media.soundcloud.com/crossdomain.xml. Then
  2218. the media can be grabbed by requesting from an url composed
  2219. of the stream token and uid
  2220. """
  2221. _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
  2222. IE_NAME = u'soundcloud'
  2223. def __init__(self, downloader=None):
  2224. InfoExtractor.__init__(self, downloader)
  2225. def report_resolve(self, video_id):
  2226. """Report information extraction."""
  2227. self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
  2228. def report_extraction(self, video_id):
  2229. """Report information extraction."""
  2230. self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
  2231. def _real_extract(self, url):
  2232. mobj = re.match(self._VALID_URL, url)
  2233. if mobj is None:
  2234. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2235. return
  2236. # extract uploader (which is in the url)
  2237. uploader = mobj.group(1)
  2238. # extract simple title (uploader + slug of song title)
  2239. slug_title = mobj.group(2)
  2240. simple_title = uploader + u'-' + slug_title
  2241. self.report_resolve('%s/%s' % (uploader, slug_title))
  2242. url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
  2243. resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
  2244. request = compat_urllib_request.Request(resolv_url)
  2245. try:
  2246. info_json_bytes = compat_urllib_request.urlopen(request).read()
  2247. info_json = info_json_bytes.decode('utf-8')
  2248. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2249. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
  2250. return
  2251. info = json.loads(info_json)
  2252. video_id = info['id']
  2253. self.report_extraction('%s/%s' % (uploader, slug_title))
  2254. streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
  2255. request = compat_urllib_request.Request(streams_url)
  2256. try:
  2257. stream_json_bytes = compat_urllib_request.urlopen(request).read()
  2258. stream_json = stream_json_bytes.decode('utf-8')
  2259. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2260. self._downloader.trouble(u'ERROR: unable to download stream definitions: %s' % compat_str(err))
  2261. return
  2262. streams = json.loads(stream_json)
  2263. mediaURL = streams['http_mp3_128_url']
  2264. return [{
  2265. 'id': info['id'],
  2266. 'url': mediaURL,
  2267. 'uploader': info['user']['username'],
  2268. 'upload_date': info['created_at'],
  2269. 'title': info['title'],
  2270. 'ext': u'mp3',
  2271. 'description': info['description'],
  2272. }]
  2273. class InfoQIE(InfoExtractor):
  2274. """Information extractor for infoq.com"""
  2275. _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
  2276. def report_extraction(self, video_id):
  2277. """Report information extraction."""
  2278. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2279. def _real_extract(self, url):
  2280. mobj = re.match(self._VALID_URL, url)
  2281. if mobj is None:
  2282. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2283. return
  2284. webpage = self._download_webpage(url, video_id=url)
  2285. self.report_extraction(url)
  2286. # Extract video URL
  2287. mobj = re.search(r"jsclassref='([^']*)'", webpage)
  2288. if mobj is None:
  2289. self._downloader.trouble(u'ERROR: unable to extract video url')
  2290. return
  2291. real_id = compat_urllib_parse.unquote(base64.b64decode(mobj.group(1).encode('ascii')).decode('utf-8'))
  2292. video_url = 'rtmpe://video.infoq.com/cfx/st/' + real_id
  2293. # Extract title
  2294. mobj = re.search(r'contentTitle = "(.*?)";', webpage)
  2295. if mobj is None:
  2296. self._downloader.trouble(u'ERROR: unable to extract video title')
  2297. return
  2298. video_title = mobj.group(1)
  2299. # Extract description
  2300. video_description = u'No description available.'
  2301. mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
  2302. if mobj is not None:
  2303. video_description = mobj.group(1)
  2304. video_filename = video_url.split('/')[-1]
  2305. video_id, extension = video_filename.split('.')
  2306. info = {
  2307. 'id': video_id,
  2308. 'url': video_url,
  2309. 'uploader': None,
  2310. 'upload_date': None,
  2311. 'title': video_title,
  2312. 'ext': extension, # Extension is always(?) mp4, but seems to be flv
  2313. 'thumbnail': None,
  2314. 'description': video_description,
  2315. }
  2316. return [info]
  2317. class MixcloudIE(InfoExtractor):
  2318. """Information extractor for www.mixcloud.com"""
  2319. _WORKING = False # New API, but it seems good http://www.mixcloud.com/developers/documentation/
  2320. _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
  2321. IE_NAME = u'mixcloud'
  2322. def __init__(self, downloader=None):
  2323. InfoExtractor.__init__(self, downloader)
  2324. def report_download_json(self, file_id):
  2325. """Report JSON download."""
  2326. self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
  2327. def report_extraction(self, file_id):
  2328. """Report information extraction."""
  2329. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
  2330. def get_urls(self, jsonData, fmt, bitrate='best'):
  2331. """Get urls from 'audio_formats' section in json"""
  2332. file_url = None
  2333. try:
  2334. bitrate_list = jsonData[fmt]
  2335. if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
  2336. bitrate = max(bitrate_list) # select highest
  2337. url_list = jsonData[fmt][bitrate]
  2338. except TypeError: # we have no bitrate info.
  2339. url_list = jsonData[fmt]
  2340. return url_list
  2341. def check_urls(self, url_list):
  2342. """Returns 1st active url from list"""
  2343. for url in url_list:
  2344. try:
  2345. compat_urllib_request.urlopen(url)
  2346. return url
  2347. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2348. url = None
  2349. return None
  2350. def _print_formats(self, formats):
  2351. print('Available formats:')
  2352. for fmt in formats.keys():
  2353. for b in formats[fmt]:
  2354. try:
  2355. ext = formats[fmt][b][0]
  2356. print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
  2357. except TypeError: # we have no bitrate info
  2358. ext = formats[fmt][0]
  2359. print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
  2360. break
  2361. def _real_extract(self, url):
  2362. mobj = re.match(self._VALID_URL, url)
  2363. if mobj is None:
  2364. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2365. return
  2366. # extract uploader & filename from url
  2367. uploader = mobj.group(1).decode('utf-8')
  2368. file_id = uploader + "-" + mobj.group(2).decode('utf-8')
  2369. # construct API request
  2370. file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
  2371. # retrieve .json file with links to files
  2372. request = compat_urllib_request.Request(file_url)
  2373. try:
  2374. self.report_download_json(file_url)
  2375. jsonData = compat_urllib_request.urlopen(request).read()
  2376. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2377. self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
  2378. return
  2379. # parse JSON
  2380. json_data = json.loads(jsonData)
  2381. player_url = json_data['player_swf_url']
  2382. formats = dict(json_data['audio_formats'])
  2383. req_format = self._downloader.params.get('format', None)
  2384. bitrate = None
  2385. if self._downloader.params.get('listformats', None):
  2386. self._print_formats(formats)
  2387. return
  2388. if req_format is None or req_format == 'best':
  2389. for format_param in formats.keys():
  2390. url_list = self.get_urls(formats, format_param)
  2391. # check urls
  2392. file_url = self.check_urls(url_list)
  2393. if file_url is not None:
  2394. break # got it!
  2395. else:
  2396. if req_format not in formats:
  2397. self._downloader.trouble(u'ERROR: format is not available')
  2398. return
  2399. url_list = self.get_urls(formats, req_format)
  2400. file_url = self.check_urls(url_list)
  2401. format_param = req_format
  2402. return [{
  2403. 'id': file_id.decode('utf-8'),
  2404. 'url': file_url.decode('utf-8'),
  2405. 'uploader': uploader.decode('utf-8'),
  2406. 'upload_date': None,
  2407. 'title': json_data['name'],
  2408. 'ext': file_url.split('.')[-1].decode('utf-8'),
  2409. 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
  2410. 'thumbnail': json_data['thumbnail_url'],
  2411. 'description': json_data['description'],
  2412. 'player_url': player_url.decode('utf-8'),
  2413. }]
  2414. class StanfordOpenClassroomIE(InfoExtractor):
  2415. """Information extractor for Stanford's Open ClassRoom"""
  2416. _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
  2417. IE_NAME = u'stanfordoc'
  2418. def report_download_webpage(self, objid):
  2419. """Report information extraction."""
  2420. self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
  2421. def report_extraction(self, video_id):
  2422. """Report information extraction."""
  2423. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2424. def _real_extract(self, url):
  2425. mobj = re.match(self._VALID_URL, url)
  2426. if mobj is None:
  2427. raise ExtractorError(u'Invalid URL: %s' % url)
  2428. if mobj.group('course') and mobj.group('video'): # A specific video
  2429. course = mobj.group('course')
  2430. video = mobj.group('video')
  2431. info = {
  2432. 'id': course + '_' + video,
  2433. 'uploader': None,
  2434. 'upload_date': None,
  2435. }
  2436. self.report_extraction(info['id'])
  2437. baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
  2438. xmlUrl = baseUrl + video + '.xml'
  2439. try:
  2440. metaXml = compat_urllib_request.urlopen(xmlUrl).read()
  2441. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2442. self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
  2443. return
  2444. mdoc = xml.etree.ElementTree.fromstring(metaXml)
  2445. try:
  2446. info['title'] = mdoc.findall('./title')[0].text
  2447. info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
  2448. except IndexError:
  2449. self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
  2450. return
  2451. info['ext'] = info['url'].rpartition('.')[2]
  2452. return [info]
  2453. elif mobj.group('course'): # A course page
  2454. course = mobj.group('course')
  2455. info = {
  2456. 'id': course,
  2457. 'type': 'playlist',
  2458. 'uploader': None,
  2459. 'upload_date': None,
  2460. }
  2461. coursepage = self._download_webpage(url, info['id'],
  2462. note='Downloading course info page',
  2463. errnote='Unable to download course info page')
  2464. m = re.search('<h1>([^<]+)</h1>', coursepage)
  2465. if m:
  2466. info['title'] = unescapeHTML(m.group(1))
  2467. else:
  2468. info['title'] = info['id']
  2469. m = re.search('<description>([^<]+)</description>', coursepage)
  2470. if m:
  2471. info['description'] = unescapeHTML(m.group(1))
  2472. links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
  2473. info['list'] = [
  2474. {
  2475. 'type': 'reference',
  2476. 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
  2477. }
  2478. for vpage in links]
  2479. results = []
  2480. for entry in info['list']:
  2481. assert entry['type'] == 'reference'
  2482. results += self.extract(entry['url'])
  2483. return results
  2484. else: # Root page
  2485. info = {
  2486. 'id': 'Stanford OpenClassroom',
  2487. 'type': 'playlist',
  2488. 'uploader': None,
  2489. 'upload_date': None,
  2490. }
  2491. self.report_download_webpage(info['id'])
  2492. rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
  2493. try:
  2494. rootpage = compat_urllib_request.urlopen(rootURL).read()
  2495. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2496. self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
  2497. return
  2498. info['title'] = info['id']
  2499. links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
  2500. info['list'] = [
  2501. {
  2502. 'type': 'reference',
  2503. 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
  2504. }
  2505. for cpage in links]
  2506. results = []
  2507. for entry in info['list']:
  2508. assert entry['type'] == 'reference'
  2509. results += self.extract(entry['url'])
  2510. return results
  2511. class MTVIE(InfoExtractor):
  2512. """Information extractor for MTV.com"""
  2513. _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
  2514. IE_NAME = u'mtv'
  2515. def report_extraction(self, video_id):
  2516. """Report information extraction."""
  2517. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2518. def _real_extract(self, url):
  2519. mobj = re.match(self._VALID_URL, url)
  2520. if mobj is None:
  2521. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2522. return
  2523. if not mobj.group('proto'):
  2524. url = 'http://' + url
  2525. video_id = mobj.group('videoid')
  2526. webpage = self._download_webpage(url, video_id)
  2527. mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
  2528. if mobj is None:
  2529. self._downloader.trouble(u'ERROR: unable to extract song name')
  2530. return
  2531. song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
  2532. mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
  2533. if mobj is None:
  2534. self._downloader.trouble(u'ERROR: unable to extract performer')
  2535. return
  2536. performer = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
  2537. video_title = performer + ' - ' + song_name
  2538. mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
  2539. if mobj is None:
  2540. self._downloader.trouble(u'ERROR: unable to mtvn_uri')
  2541. return
  2542. mtvn_uri = mobj.group(1)
  2543. mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
  2544. if mobj is None:
  2545. self._downloader.trouble(u'ERROR: unable to extract content id')
  2546. return
  2547. content_id = mobj.group(1)
  2548. videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
  2549. self.report_extraction(video_id)
  2550. request = compat_urllib_request.Request(videogen_url)
  2551. try:
  2552. metadataXml = compat_urllib_request.urlopen(request).read()
  2553. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2554. self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
  2555. return
  2556. mdoc = xml.etree.ElementTree.fromstring(metadataXml)
  2557. renditions = mdoc.findall('.//rendition')
  2558. # For now, always pick the highest quality.
  2559. rendition = renditions[-1]
  2560. try:
  2561. _,_,ext = rendition.attrib['type'].partition('/')
  2562. format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
  2563. video_url = rendition.find('./src').text
  2564. except KeyError:
  2565. self._downloader.trouble('Invalid rendition field.')
  2566. return
  2567. info = {
  2568. 'id': video_id,
  2569. 'url': video_url,
  2570. 'uploader': performer,
  2571. 'upload_date': None,
  2572. 'title': video_title,
  2573. 'ext': ext,
  2574. 'format': format,
  2575. }
  2576. return [info]
  2577. class YoukuIE(InfoExtractor):
  2578. _VALID_URL = r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
  2579. def report_download_webpage(self, file_id):
  2580. """Report webpage download."""
  2581. self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, file_id))
  2582. def report_extraction(self, file_id):
  2583. """Report information extraction."""
  2584. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
  2585. def _gen_sid(self):
  2586. nowTime = int(time.time() * 1000)
  2587. random1 = random.randint(1000,1998)
  2588. random2 = random.randint(1000,9999)
  2589. return "%d%d%d" %(nowTime,random1,random2)
  2590. def _get_file_ID_mix_string(self, seed):
  2591. mixed = []
  2592. source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
  2593. seed = float(seed)
  2594. for i in range(len(source)):
  2595. seed = (seed * 211 + 30031 ) % 65536
  2596. index = math.floor(seed / 65536 * len(source) )
  2597. mixed.append(source[int(index)])
  2598. source.remove(source[int(index)])
  2599. #return ''.join(mixed)
  2600. return mixed
  2601. def _get_file_id(self, fileId, seed):
  2602. mixed = self._get_file_ID_mix_string(seed)
  2603. ids = fileId.split('*')
  2604. realId = []
  2605. for ch in ids:
  2606. if ch:
  2607. realId.append(mixed[int(ch)])
  2608. return ''.join(realId)
  2609. def _real_extract(self, url):
  2610. mobj = re.match(self._VALID_URL, url)
  2611. if mobj is None:
  2612. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2613. return
  2614. video_id = mobj.group('ID')
  2615. info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
  2616. request = compat_urllib_request.Request(info_url, None, std_headers)
  2617. try:
  2618. self.report_download_webpage(video_id)
  2619. jsondata = compat_urllib_request.urlopen(request).read()
  2620. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2621. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  2622. return
  2623. self.report_extraction(video_id)
  2624. try:
  2625. jsonstr = jsondata.decode('utf-8')
  2626. config = json.loads(jsonstr)
  2627. video_title = config['data'][0]['title']
  2628. seed = config['data'][0]['seed']
  2629. format = self._downloader.params.get('format', None)
  2630. supported_format = list(config['data'][0]['streamfileids'].keys())
  2631. if format is None or format == 'best':
  2632. if 'hd2' in supported_format:
  2633. format = 'hd2'
  2634. else:
  2635. format = 'flv'
  2636. ext = u'flv'
  2637. elif format == 'worst':
  2638. format = 'mp4'
  2639. ext = u'mp4'
  2640. else:
  2641. format = 'flv'
  2642. ext = u'flv'
  2643. fileid = config['data'][0]['streamfileids'][format]
  2644. keys = [s['k'] for s in config['data'][0]['segs'][format]]
  2645. except (UnicodeDecodeError, ValueError, KeyError):
  2646. self._downloader.trouble(u'ERROR: unable to extract info section')
  2647. return
  2648. files_info=[]
  2649. sid = self._gen_sid()
  2650. fileid = self._get_file_id(fileid, seed)
  2651. #column 8,9 of fileid represent the segment number
  2652. #fileid[7:9] should be changed
  2653. for index, key in enumerate(keys):
  2654. temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
  2655. download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
  2656. info = {
  2657. 'id': '%s_part%02d' % (video_id, index),
  2658. 'url': download_url,
  2659. 'uploader': None,
  2660. 'upload_date': None,
  2661. 'title': video_title,
  2662. 'ext': ext,
  2663. }
  2664. files_info.append(info)
  2665. return files_info
  2666. class XNXXIE(InfoExtractor):
  2667. """Information extractor for xnxx.com"""
  2668. _VALID_URL = r'^(?:https?://)?video\.xnxx\.com/video([0-9]+)/(.*)'
  2669. IE_NAME = u'xnxx'
  2670. VIDEO_URL_RE = r'flv_url=(.*?)&amp;'
  2671. VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
  2672. VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&amp;'
  2673. def report_webpage(self, video_id):
  2674. """Report information extraction"""
  2675. self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
  2676. def report_extraction(self, video_id):
  2677. """Report information extraction"""
  2678. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2679. def _real_extract(self, url):
  2680. mobj = re.match(self._VALID_URL, url)
  2681. if mobj is None:
  2682. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2683. return
  2684. video_id = mobj.group(1)
  2685. self.report_webpage(video_id)
  2686. # Get webpage content
  2687. try:
  2688. webpage_bytes = compat_urllib_request.urlopen(url).read()
  2689. webpage = webpage_bytes.decode('utf-8')
  2690. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2691. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
  2692. return
  2693. result = re.search(self.VIDEO_URL_RE, webpage)
  2694. if result is None:
  2695. self._downloader.trouble(u'ERROR: unable to extract video url')
  2696. return
  2697. video_url = compat_urllib_parse.unquote(result.group(1))
  2698. result = re.search(self.VIDEO_TITLE_RE, webpage)
  2699. if result is None:
  2700. self._downloader.trouble(u'ERROR: unable to extract video title')
  2701. return
  2702. video_title = result.group(1)
  2703. result = re.search(self.VIDEO_THUMB_RE, webpage)
  2704. if result is None:
  2705. self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
  2706. return
  2707. video_thumbnail = result.group(1)
  2708. return [{
  2709. 'id': video_id,
  2710. 'url': video_url,
  2711. 'uploader': None,
  2712. 'upload_date': None,
  2713. 'title': video_title,
  2714. 'ext': 'flv',
  2715. 'thumbnail': video_thumbnail,
  2716. 'description': None,
  2717. }]
  2718. class GooglePlusIE(InfoExtractor):
  2719. """Information extractor for plus.google.com."""
  2720. _VALID_URL = r'(?:https://)?plus\.google\.com/(?:[^/]+/)*?posts/(\w+)'
  2721. IE_NAME = u'plus.google'
  2722. def __init__(self, downloader=None):
  2723. InfoExtractor.__init__(self, downloader)
  2724. def report_extract_entry(self, url):
  2725. """Report downloading extry"""
  2726. self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url)
  2727. def report_date(self, upload_date):
  2728. """Report downloading extry"""
  2729. self._downloader.to_screen(u'[plus.google] Entry date: %s' % upload_date)
  2730. def report_uploader(self, uploader):
  2731. """Report downloading extry"""
  2732. self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader)
  2733. def report_title(self, video_title):
  2734. """Report downloading extry"""
  2735. self._downloader.to_screen(u'[plus.google] Title: %s' % video_title)
  2736. def report_extract_vid_page(self, video_page):
  2737. """Report information extraction."""
  2738. self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page)
  2739. def _real_extract(self, url):
  2740. # Extract id from URL
  2741. mobj = re.match(self._VALID_URL, url)
  2742. if mobj is None:
  2743. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  2744. return
  2745. post_url = mobj.group(0)
  2746. video_id = mobj.group(1)
  2747. video_extension = 'flv'
  2748. # Step 1, Retrieve post webpage to extract further information
  2749. self.report_extract_entry(post_url)
  2750. request = compat_urllib_request.Request(post_url)
  2751. try:
  2752. webpage = compat_urllib_request.urlopen(request).read().decode('utf-8')
  2753. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2754. self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
  2755. return
  2756. # Extract update date
  2757. upload_date = None
  2758. pattern = 'title="Timestamp">(.*?)</a>'
  2759. mobj = re.search(pattern, webpage)
  2760. if mobj:
  2761. upload_date = mobj.group(1)
  2762. # Convert timestring to a format suitable for filename
  2763. upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
  2764. upload_date = upload_date.strftime('%Y%m%d')
  2765. self.report_date(upload_date)
  2766. # Extract uploader
  2767. uploader = None
  2768. pattern = r'rel\="author".*?>(.*?)</a>'
  2769. mobj = re.search(pattern, webpage)
  2770. if mobj:
  2771. uploader = mobj.group(1)
  2772. self.report_uploader(uploader)
  2773. # Extract title
  2774. # Get the first line for title
  2775. video_title = u'NA'
  2776. pattern = r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]'
  2777. mobj = re.search(pattern, webpage)
  2778. if mobj:
  2779. video_title = mobj.group(1)
  2780. self.report_title(video_title)
  2781. # Step 2, Stimulate clicking the image box to launch video
  2782. pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]'
  2783. mobj = re.search(pattern, webpage)
  2784. if mobj is None:
  2785. self._downloader.trouble(u'ERROR: unable to extract video page URL')
  2786. video_page = mobj.group(1)
  2787. request = compat_urllib_request.Request(video_page)
  2788. try:
  2789. webpage = compat_urllib_request.urlopen(request).read().decode('utf-8')
  2790. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2791. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  2792. return
  2793. self.report_extract_vid_page(video_page)
  2794. # Extract video links on video page
  2795. """Extract video links of all sizes"""
  2796. pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
  2797. mobj = re.findall(pattern, webpage)
  2798. if len(mobj) == 0:
  2799. self._downloader.trouble(u'ERROR: unable to extract video links')
  2800. # Sort in resolution
  2801. links = sorted(mobj)
  2802. # Choose the lowest of the sort, i.e. highest resolution
  2803. video_url = links[-1]
  2804. # Only get the url. The resolution part in the tuple has no use anymore
  2805. video_url = video_url[-1]
  2806. # Treat escaped \u0026 style hex
  2807. try:
  2808. video_url = video_url.decode("unicode_escape")
  2809. except AttributeError: # Python 3
  2810. video_url = bytes(video_url, 'ascii').decode('unicode-escape')
  2811. return [{
  2812. 'id': video_id,
  2813. 'url': video_url,
  2814. 'uploader': uploader,
  2815. 'upload_date': upload_date,
  2816. 'title': video_title,
  2817. 'ext': video_extension,
  2818. }]
  2819. class NBAIE(InfoExtractor):
  2820. _VALID_URL = r'^(?:https?://)?(?:watch\.|www\.)?nba\.com/(?:nba/)?video(/[^?]*)(\?.*)?$'
  2821. IE_NAME = u'nba'
  2822. def _real_extract(self, url):
  2823. mobj = re.match(self._VALID_URL, url)
  2824. if mobj is None:
  2825. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2826. return
  2827. video_id = mobj.group(1)
  2828. if video_id.endswith('/index.html'):
  2829. video_id = video_id[:-len('/index.html')]
  2830. webpage = self._download_webpage(url, video_id)
  2831. video_url = u'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
  2832. def _findProp(rexp, default=None):
  2833. m = re.search(rexp, webpage)
  2834. if m:
  2835. return unescapeHTML(m.group(1))
  2836. else:
  2837. return default
  2838. shortened_video_id = video_id.rpartition('/')[2]
  2839. title = _findProp(r'<meta property="og:title" content="(.*?)"', shortened_video_id).replace('NBA.com: ', '')
  2840. info = {
  2841. 'id': shortened_video_id,
  2842. 'url': video_url,
  2843. 'ext': 'mp4',
  2844. 'title': title,
  2845. 'uploader_date': _findProp(r'<b>Date:</b> (.*?)</div>'),
  2846. 'description': _findProp(r'<div class="description">(.*?)</h1>'),
  2847. }
  2848. return [info]
  2849. class JustinTVIE(InfoExtractor):
  2850. """Information extractor for justin.tv and twitch.tv"""
  2851. # TODO: One broadcast may be split into multiple videos. The key
  2852. # 'broadcast_id' is the same for all parts, and 'broadcast_part'
  2853. # starts at 1 and increases. Can we treat all parts as one video?
  2854. _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?(?:twitch|justin)\.tv/
  2855. ([^/]+)(?:/b/([^/]+))?/?(?:\#.*)?$"""
  2856. _JUSTIN_PAGE_LIMIT = 100
  2857. IE_NAME = u'justin.tv'
  2858. def report_extraction(self, file_id):
  2859. """Report information extraction."""
  2860. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
  2861. def report_download_page(self, channel, offset):
  2862. """Report attempt to download a single page of videos."""
  2863. self._downloader.to_screen(u'[%s] %s: Downloading video information from %d to %d' %
  2864. (self.IE_NAME, channel, offset, offset + self._JUSTIN_PAGE_LIMIT))
  2865. # Return count of items, list of *valid* items
  2866. def _parse_page(self, url):
  2867. try:
  2868. urlh = compat_urllib_request.urlopen(url)
  2869. webpage_bytes = urlh.read()
  2870. webpage = webpage_bytes.decode('utf-8', 'ignore')
  2871. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2872. self._downloader.trouble(u'ERROR: unable to download video info JSON: %s' % compat_str(err))
  2873. return
  2874. response = json.loads(webpage)
  2875. if type(response) != list:
  2876. error_text = response.get('error', 'unknown error')
  2877. self._downloader.trouble(u'ERROR: Justin.tv API: %s' % error_text)
  2878. return
  2879. info = []
  2880. for clip in response:
  2881. video_url = clip['video_file_url']
  2882. if video_url:
  2883. video_extension = os.path.splitext(video_url)[1][1:]
  2884. video_date = re.sub('-', '', clip['start_time'][:10])
  2885. video_uploader_id = clip.get('user_id', clip.get('channel_id'))
  2886. video_id = clip['id']
  2887. video_title = clip.get('title', video_id)
  2888. info.append({
  2889. 'id': video_id,
  2890. 'url': video_url,
  2891. 'title': video_title,
  2892. 'uploader': clip.get('channel_name', video_uploader_id),
  2893. 'uploader_id': video_uploader_id,
  2894. 'upload_date': video_date,
  2895. 'ext': video_extension,
  2896. })
  2897. return (len(response), info)
  2898. def _real_extract(self, url):
  2899. mobj = re.match(self._VALID_URL, url)
  2900. if mobj is None:
  2901. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2902. return
  2903. api = 'http://api.justin.tv'
  2904. video_id = mobj.group(mobj.lastindex)
  2905. paged = False
  2906. if mobj.lastindex == 1:
  2907. paged = True
  2908. api += '/channel/archives/%s.json'
  2909. else:
  2910. api += '/broadcast/by_archive/%s.json'
  2911. api = api % (video_id,)
  2912. self.report_extraction(video_id)
  2913. info = []
  2914. offset = 0
  2915. limit = self._JUSTIN_PAGE_LIMIT
  2916. while True:
  2917. if paged:
  2918. self.report_download_page(video_id, offset)
  2919. page_url = api + ('?offset=%d&limit=%d' % (offset, limit))
  2920. page_count, page_info = self._parse_page(page_url)
  2921. info.extend(page_info)
  2922. if not paged or page_count != limit:
  2923. break
  2924. offset += limit
  2925. return info
  2926. class FunnyOrDieIE(InfoExtractor):
  2927. _VALID_URL = r'^(?:https?://)?(?:www\.)?funnyordie\.com/videos/(?P<id>[0-9a-f]+)/.*$'
  2928. def _real_extract(self, url):
  2929. mobj = re.match(self._VALID_URL, url)
  2930. if mobj is None:
  2931. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2932. return
  2933. video_id = mobj.group('id')
  2934. webpage = self._download_webpage(url, video_id)
  2935. m = re.search(r'<video[^>]*>\s*<source[^>]*>\s*<source src="(?P<url>[^"]+)"', webpage, re.DOTALL)
  2936. if not m:
  2937. self._downloader.trouble(u'ERROR: unable to find video information')
  2938. video_url = unescapeHTML(m.group('url'))
  2939. m = re.search(r"class='player_page_h1'>\s+<a.*?>(?P<title>.*?)</a>", webpage)
  2940. if not m:
  2941. self._downloader.trouble(u'Cannot find video title')
  2942. title = unescapeHTML(m.group('title'))
  2943. m = re.search(r'<meta property="og:description" content="(?P<desc>.*?)"', webpage)
  2944. if m:
  2945. desc = unescapeHTML(m.group('desc'))
  2946. else:
  2947. desc = None
  2948. info = {
  2949. 'id': video_id,
  2950. 'url': video_url,
  2951. 'ext': 'mp4',
  2952. 'title': title,
  2953. 'description': desc,
  2954. }
  2955. return [info]
  2956. class TweetReelIE(InfoExtractor):
  2957. _VALID_URL = r'^(?:https?://)?(?:www\.)?tweetreel\.com/[?](?P<id>[0-9a-z]+)$'
  2958. def _real_extract(self, url):
  2959. mobj = re.match(self._VALID_URL, url)
  2960. if mobj is None:
  2961. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2962. return
  2963. video_id = mobj.group('id')
  2964. webpage = self._download_webpage(url, video_id)
  2965. m = re.search(r'<div id="left" status_id="([0-9]+)">', webpage)
  2966. if not m:
  2967. self._downloader.trouble(u'ERROR: Cannot find status ID')
  2968. status_id = m.group(1)
  2969. m = re.search(r'<div class="tweet_text">(.*?)</div>', webpage, flags=re.DOTALL)
  2970. if not m:
  2971. self._downloader.trouble(u'WARNING: Cannot find description')
  2972. desc = unescapeHTML(re.sub('<a.*?</a>', '', m.group(1))).strip()
  2973. m = re.search(r'<div class="tweet_info">.*?from <a target="_blank" href="https?://twitter.com/(?P<uploader_id>.+?)">(?P<uploader>.+?)</a>', webpage, flags=re.DOTALL)
  2974. if not m:
  2975. self._downloader.trouble(u'ERROR: Cannot find uploader')
  2976. uploader = unescapeHTML(m.group('uploader'))
  2977. uploader_id = unescapeHTML(m.group('uploader_id'))
  2978. m = re.search(r'<span unixtime="([0-9]+)"', webpage)
  2979. if not m:
  2980. self._downloader.trouble(u'ERROR: Cannot find upload date')
  2981. upload_date = datetime.datetime.fromtimestamp(int(m.group(1))).strftime('%Y%m%d')
  2982. title = desc
  2983. video_url = 'http://files.tweetreel.com/video/' + status_id + '.mov'
  2984. info = {
  2985. 'id': video_id,
  2986. 'url': video_url,
  2987. 'ext': 'mov',
  2988. 'title': title,
  2989. 'description': desc,
  2990. 'uploader': uploader,
  2991. 'uploader_id': uploader_id,
  2992. 'internal_id': status_id,
  2993. 'upload_date': upload_date
  2994. }
  2995. return [info]
  2996. class SteamIE(InfoExtractor):
  2997. _VALID_URL = r"""http://store.steampowered.com/
  2998. (?P<urltype>video|app)/ #If the page is only for videos or for a game
  2999. (?P<gameID>\d+)/?
  3000. (?P<videoID>\d*)(?P<extra>\??) #For urltype == video we sometimes get the videoID
  3001. """
  3002. def suitable(self, url):
  3003. """Receives a URL and returns True if suitable for this IE."""
  3004. return re.match(self._VALID_URL, url, re.VERBOSE) is not None
  3005. def _real_extract(self, url):
  3006. m = re.match(self._VALID_URL, url, re.VERBOSE)
  3007. urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},"
  3008. gameID = m.group('gameID')
  3009. videourl = 'http://store.steampowered.com/video/%s/' % gameID
  3010. webpage = self._download_webpage(videourl, gameID)
  3011. mweb = re.finditer(urlRE, webpage)
  3012. namesRE = r'<span class="title">(?P<videoName>.+?)</span>'
  3013. titles = re.finditer(namesRE, webpage)
  3014. videos = []
  3015. for vid,vtitle in zip(mweb,titles):
  3016. video_id = vid.group('videoID')
  3017. title = vtitle.group('videoName')
  3018. video_url = vid.group('videoURL')
  3019. if not video_url:
  3020. self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id)
  3021. info = {
  3022. 'id':video_id,
  3023. 'url':video_url,
  3024. 'ext': 'flv',
  3025. 'title': unescapeHTML(title)
  3026. }
  3027. videos.append(info)
  3028. return videos
  3029. class UstreamIE(InfoExtractor):
  3030. _VALID_URL = r'https?://www\.ustream\.tv/recorded/(?P<videoID>\d+)'
  3031. IE_NAME = u'ustream'
  3032. def _real_extract(self, url):
  3033. m = re.match(self._VALID_URL, url)
  3034. video_id = m.group('videoID')
  3035. video_url = u'http://tcdn.ustream.tv/video/%s' % video_id
  3036. webpage = self._download_webpage(url, video_id)
  3037. m = re.search(r'data-title="(?P<title>.+)"',webpage)
  3038. title = m.group('title')
  3039. m = re.search(r'<a class="state" data-content-type="channel" data-content-id="(?P<uploader>\d+)"',webpage)
  3040. uploader = m.group('uploader')
  3041. info = {
  3042. 'id':video_id,
  3043. 'url':video_url,
  3044. 'ext': 'flv',
  3045. 'title': title,
  3046. 'uploader': uploader
  3047. }
  3048. return [info]
  3049. class RBMARadioIE(InfoExtractor):
  3050. _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P<videoID>[^/]+)$'
  3051. def _real_extract(self, url):
  3052. m = re.match(self._VALID_URL, url)
  3053. video_id = m.group('videoID')
  3054. webpage = self._download_webpage(url, video_id)
  3055. m = re.search(r'<script>window.gon = {.*?};gon\.show=(.+?);</script>', webpage)
  3056. if not m:
  3057. raise ExtractorError(u'Cannot find metadata')
  3058. json_data = m.group(1)
  3059. try:
  3060. data = json.loads(json_data)
  3061. except ValueError as e:
  3062. raise ExtractorError(u'Invalid JSON: ' + str(e))
  3063. video_url = data['akamai_url'] + '&cbr=256'
  3064. url_parts = compat_urllib_parse_urlparse(video_url)
  3065. video_ext = url_parts.path.rpartition('.')[2]
  3066. info = {
  3067. 'id': video_id,
  3068. 'url': video_url,
  3069. 'ext': video_ext,
  3070. 'title': data['title'],
  3071. 'description': data.get('teaser_text'),
  3072. 'location': data.get('country_of_origin'),
  3073. 'uploader': data.get('host', {}).get('name'),
  3074. 'uploader_id': data.get('host', {}).get('slug'),
  3075. 'thumbnail': data.get('image', {}).get('large_url_2x'),
  3076. 'duration': data.get('duration'),
  3077. }
  3078. return [info]
  3079. class YouPornIE(InfoExtractor):
  3080. """Information extractor for youporn.com."""
  3081. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+)'
  3082. def _print_formats(self, formats):
  3083. """Print all available formats"""
  3084. print(u'Available formats:')
  3085. print(u'ext\t\tformat')
  3086. print(u'---------------------------------')
  3087. for format in formats:
  3088. print(u'%s\t\t%s' % (format['ext'], format['format']))
  3089. def _specific(self, req_format, formats):
  3090. for x in formats:
  3091. if(x["format"]==req_format):
  3092. return x
  3093. return None
  3094. def _real_extract(self, url):
  3095. mobj = re.match(self._VALID_URL, url)
  3096. if mobj is None:
  3097. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  3098. return
  3099. video_id = mobj.group('videoid')
  3100. req = compat_urllib_request.Request(url)
  3101. req.add_header('Cookie', 'age_verified=1')
  3102. webpage = self._download_webpage(req, video_id)
  3103. # Get the video title
  3104. result = re.search(r'videoTitleArea">(?P<title>.*)</h1>', webpage)
  3105. if result is None:
  3106. raise ExtractorError(u'ERROR: unable to extract video title')
  3107. video_title = result.group('title').strip()
  3108. # Get the video date
  3109. result = re.search(r'Date:</b>(?P<date>.*)</li>', webpage)
  3110. if result is None:
  3111. self._downloader.to_stderr(u'WARNING: unable to extract video date')
  3112. upload_date = None
  3113. else:
  3114. upload_date = result.group('date').strip()
  3115. # Get the video uploader
  3116. result = re.search(r'Submitted:</b>(?P<uploader>.*)</li>', webpage)
  3117. if result is None:
  3118. self._downloader.to_stderr(u'ERROR: unable to extract uploader')
  3119. video_uploader = None
  3120. else:
  3121. video_uploader = result.group('uploader').strip()
  3122. video_uploader = clean_html( video_uploader )
  3123. # Get all of the formats available
  3124. DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
  3125. result = re.search(DOWNLOAD_LIST_RE, webpage)
  3126. if result is None:
  3127. raise ExtractorError(u'Unable to extract download list')
  3128. download_list_html = result.group('download_list').strip()
  3129. # Get all of the links from the page
  3130. LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">'
  3131. links = re.findall(LINK_RE, download_list_html)
  3132. if(len(links) == 0):
  3133. raise ExtractorError(u'ERROR: no known formats available for video')
  3134. self._downloader.to_screen(u'[youporn] Links found: %d' % len(links))
  3135. formats = []
  3136. for link in links:
  3137. # A link looks like this:
  3138. # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
  3139. # A path looks like this:
  3140. # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
  3141. video_url = unescapeHTML( link )
  3142. path = compat_urllib_parse_urlparse( video_url ).path
  3143. extension = os.path.splitext( path )[1][1:]
  3144. format = path.split('/')[4].split('_')[:2]
  3145. size = format[0]
  3146. bitrate = format[1]
  3147. format = "-".join( format )
  3148. title = u'%s-%s-%s' % (video_title, size, bitrate)
  3149. formats.append({
  3150. 'id': video_id,
  3151. 'url': video_url,
  3152. 'uploader': video_uploader,
  3153. 'upload_date': upload_date,
  3154. 'title': title,
  3155. 'ext': extension,
  3156. 'format': format,
  3157. 'thumbnail': None,
  3158. 'description': None,
  3159. 'player_url': None
  3160. })
  3161. if self._downloader.params.get('listformats', None):
  3162. self._print_formats(formats)
  3163. return
  3164. req_format = self._downloader.params.get('format', None)
  3165. self._downloader.to_screen(u'[youporn] Format: %s' % req_format)
  3166. if req_format is None or req_format == 'best':
  3167. return [formats[0]]
  3168. elif req_format == 'worst':
  3169. return [formats[-1]]
  3170. elif req_format in ('-1', 'all'):
  3171. return formats
  3172. else:
  3173. format = self._specific( req_format, formats )
  3174. if result is None:
  3175. self._downloader.trouble(u'ERROR: requested format not available')
  3176. return
  3177. return [format]
  3178. class PornotubeIE(InfoExtractor):
  3179. """Information extractor for pornotube.com."""
  3180. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$'
  3181. def _real_extract(self, url):
  3182. mobj = re.match(self._VALID_URL, url)
  3183. if mobj is None:
  3184. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  3185. return
  3186. video_id = mobj.group('videoid')
  3187. video_title = mobj.group('title')
  3188. # Get webpage content
  3189. webpage = self._download_webpage(url, video_id)
  3190. # Get the video URL
  3191. VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",'
  3192. result = re.search(VIDEO_URL_RE, webpage)
  3193. if result is None:
  3194. self._downloader.trouble(u'ERROR: unable to extract video url')
  3195. return
  3196. video_url = compat_urllib_parse.unquote(result.group('url'))
  3197. #Get the uploaded date
  3198. VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
  3199. result = re.search(VIDEO_UPLOADED_RE, webpage)
  3200. if result is None:
  3201. self._downloader.trouble(u'ERROR: unable to extract video title')
  3202. return
  3203. upload_date = result.group('date')
  3204. info = {'id': video_id,
  3205. 'url': video_url,
  3206. 'uploader': None,
  3207. 'upload_date': upload_date,
  3208. 'title': video_title,
  3209. 'ext': 'flv',
  3210. 'format': 'flv'}
  3211. return [info]
  3212. class YouJizzIE(InfoExtractor):
  3213. """Information extractor for youjizz.com."""
  3214. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+).html$'
  3215. def _real_extract(self, url):
  3216. mobj = re.match(self._VALID_URL, url)
  3217. if mobj is None:
  3218. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  3219. return
  3220. video_id = mobj.group('videoid')
  3221. # Get webpage content
  3222. webpage = self._download_webpage(url, video_id)
  3223. # Get the video title
  3224. result = re.search(r'<title>(?P<title>.*)</title>', webpage)
  3225. if result is None:
  3226. raise ExtractorError(u'ERROR: unable to extract video title')
  3227. video_title = result.group('title').strip()
  3228. # Get the embed page
  3229. result = re.search(r'https?://www.youjizz.com/videos/embed/(?P<videoid>[0-9]+)', webpage)
  3230. if result is None:
  3231. raise ExtractorError(u'ERROR: unable to extract embed page')
  3232. embed_page_url = result.group(0).strip()
  3233. video_id = result.group('videoid')
  3234. webpage = self._download_webpage(embed_page_url, video_id)
  3235. # Get the video URL
  3236. result = re.search(r'so.addVariable\("file",encodeURIComponent\("(?P<source>[^"]+)"\)\);', webpage)
  3237. if result is None:
  3238. raise ExtractorError(u'ERROR: unable to extract video url')
  3239. video_url = result.group('source')
  3240. info = {'id': video_id,
  3241. 'url': video_url,
  3242. 'title': video_title,
  3243. 'ext': 'flv',
  3244. 'format': 'flv',
  3245. 'player_url': embed_page_url}
  3246. return [info]
  3247. class EightTracksIE(InfoExtractor):
  3248. IE_NAME = '8tracks'
  3249. _VALID_URL = r'https?://8tracks.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'
  3250. def _real_extract(self, url):
  3251. mobj = re.match(self._VALID_URL, url)
  3252. if mobj is None:
  3253. raise ExtractorError(u'Invalid URL: %s' % url)
  3254. playlist_id = mobj.group('id')
  3255. webpage = self._download_webpage(url, playlist_id)
  3256. m = re.search(r"new TRAX.Mix\((.*?)\);\n*\s*TRAX.initSearchAutocomplete\('#search'\);", webpage, flags=re.DOTALL)
  3257. if not m:
  3258. raise ExtractorError(u'Cannot find trax information')
  3259. json_like = m.group(1)
  3260. data = json.loads(json_like)
  3261. session = str(random.randint(0, 1000000000))
  3262. mix_id = data['id']
  3263. track_count = data['tracks_count']
  3264. first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
  3265. next_url = first_url
  3266. res = []
  3267. for i in itertools.count():
  3268. api_json = self._download_webpage(next_url, playlist_id,
  3269. note=u'Downloading song information %s/%s' % (str(i+1), track_count),
  3270. errnote=u'Failed to download song information')
  3271. api_data = json.loads(api_json)
  3272. track_data = api_data[u'set']['track']
  3273. info = {
  3274. 'id': track_data['id'],
  3275. 'url': track_data['track_file_stream_url'],
  3276. 'title': track_data['performer'] + u' - ' + track_data['name'],
  3277. 'raw_title': track_data['name'],
  3278. 'uploader_id': data['user']['login'],
  3279. 'ext': 'm4a',
  3280. }
  3281. res.append(info)
  3282. if api_data['set']['at_last_track']:
  3283. break
  3284. next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id'])
  3285. return res
  3286. class KeekIE(InfoExtractor):
  3287. _VALID_URL = r'http://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P<videoID>\w+)'
  3288. IE_NAME = u'keek'
  3289. def _real_extract(self, url):
  3290. m = re.match(self._VALID_URL, url)
  3291. video_id = m.group('videoID')
  3292. video_url = u'http://cdn.keek.com/keek/video/%s' % video_id
  3293. thumbnail = u'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id
  3294. webpage = self._download_webpage(url, video_id)
  3295. m = re.search(r'<meta property="og:title" content="(?P<title>.+)"', webpage)
  3296. title = unescapeHTML(m.group('title'))
  3297. m = re.search(r'<div class="bio-names-and-report">[\s\n]+<h4>(?P<uploader>\w+)</h4>', webpage)
  3298. uploader = unescapeHTML(m.group('uploader'))
  3299. info = {
  3300. 'id':video_id,
  3301. 'url':video_url,
  3302. 'ext': 'mp4',
  3303. 'title': title,
  3304. 'thumbnail': thumbnail,
  3305. 'uploader': uploader
  3306. }
  3307. return [info]
  3308. class MyspassIE(InfoExtractor):
  3309. _VALID_URL = r'http://www.myspass.de/.*'
  3310. IE_NAME = u'myspass'
  3311. def _real_extract(self, url):
  3312. META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s'
  3313. # video id is the last path element of the URL
  3314. # usually there is a trailing slash, so also try the second but last
  3315. url_path = compat_urllib_parse_urlparse(url).path
  3316. url_parent_path, video_id = os.path.split(url_path)
  3317. if not video_id:
  3318. _, video_id = os.path.split(url_parent_path)
  3319. # get metadata
  3320. metadata_url = META_DATA_URL_TEMPLATE % video_id
  3321. metadata_text = self._download_webpage(metadata_url, video_id)
  3322. metadata = xml.etree.ElementTree.fromstring(metadata_text.encode('utf-8'))
  3323. # extract values from metadata
  3324. url_flv_el = metadata.find('url_flv')
  3325. if url_flv_el is None:
  3326. self._downloader.trouble(u'ERROR: unable to extract download url')
  3327. return
  3328. video_url = url_flv_el.text
  3329. extension = os.path.splitext(video_url)[1][1:]
  3330. title_el = metadata.find('title')
  3331. if title_el is None:
  3332. self._downloader.trouble(u'ERROR: unable to extract title')
  3333. return
  3334. title = title_el.text
  3335. format_id_el = metadata.find('format_id')
  3336. if format_id_el is None:
  3337. format = ext
  3338. else:
  3339. format = format_id_el.text
  3340. description_el = metadata.find('description')
  3341. if description_el is not None:
  3342. description = description_el.text
  3343. else:
  3344. description = None
  3345. imagePreview_el = metadata.find('imagePreview')
  3346. if imagePreview_el is not None:
  3347. thumbnail = imagePreview_el.text
  3348. else:
  3349. thumbnail = None
  3350. info = {
  3351. 'id': video_id,
  3352. 'url': video_url,
  3353. 'title': title,
  3354. 'ext': extension,
  3355. 'format': format,
  3356. 'thumbnail': thumbnail,
  3357. 'description': description
  3358. }
  3359. return [info]
  3360. def gen_extractors():
  3361. """ Return a list of an instance of every supported extractor.
  3362. The order does matter; the first extractor matched is the one handling the URL.
  3363. """
  3364. return [
  3365. YoutubePlaylistIE(),
  3366. YoutubeChannelIE(),
  3367. YoutubeUserIE(),
  3368. YoutubeSearchIE(),
  3369. YoutubeIE(),
  3370. MetacafeIE(),
  3371. DailymotionIE(),
  3372. GoogleSearchIE(),
  3373. PhotobucketIE(),
  3374. YahooIE(),
  3375. YahooSearchIE(),
  3376. DepositFilesIE(),
  3377. FacebookIE(),
  3378. BlipTVUserIE(),
  3379. BlipTVIE(),
  3380. VimeoIE(),
  3381. MyVideoIE(),
  3382. ComedyCentralIE(),
  3383. EscapistIE(),
  3384. CollegeHumorIE(),
  3385. XVideosIE(),
  3386. SoundcloudIE(),
  3387. InfoQIE(),
  3388. MixcloudIE(),
  3389. StanfordOpenClassroomIE(),
  3390. MTVIE(),
  3391. YoukuIE(),
  3392. XNXXIE(),
  3393. YouJizzIE(),
  3394. PornotubeIE(),
  3395. YouPornIE(),
  3396. GooglePlusIE(),
  3397. ArteTvIE(),
  3398. NBAIE(),
  3399. JustinTVIE(),
  3400. FunnyOrDieIE(),
  3401. TweetReelIE(),
  3402. SteamIE(),
  3403. UstreamIE(),
  3404. RBMARadioIE(),
  3405. EightTracksIE(),
  3406. KeekIE(),
  3407. MyspassIE(),
  3408. GenericIE()
  3409. ]