You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

4110 lines
163 KiB

13 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
13 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. from __future__ import absolute_import
  4. import base64
  5. import datetime
  6. import netrc
  7. import os
  8. import re
  9. import socket
  10. import time
  11. import email.utils
  12. import xml.etree.ElementTree
  13. import random
  14. import math
  15. import urllib
  16. import urllib2
  17. import httplib
  18. from urlparse import parse_qs, urlparse
  19. from .utils import *
  20. class InfoExtractor(object):
  21. """Information Extractor class.
  22. Information extractors are the classes that, given a URL, extract
  23. information about the video (or videos) the URL refers to. This
  24. information includes the real video URL, the video title, author and
  25. others. The information is stored in a dictionary which is then
  26. passed to the FileDownloader. The FileDownloader processes this
  27. information possibly downloading the video to the file system, among
  28. other possible outcomes.
  29. The dictionaries must include the following fields:
  30. id: Video identifier.
  31. url: Final video URL.
  32. title: Video title, unescaped.
  33. ext: Video filename extension.
  34. uploader: Full name of the video uploader.
  35. upload_date: Video upload date (YYYYMMDD).
  36. The following fields are optional:
  37. format: The video format, defaults to ext (used for --get-format)
  38. thumbnail: Full URL to a video thumbnail image.
  39. description: One-line video description.
  40. uploader_id: Nickname or id of the video uploader.
  41. player_url: SWF Player URL (used for rtmpdump).
  42. subtitles: The .srt file contents.
  43. urlhandle: [internal] The urlHandle to be used to download the file,
  44. like returned by urllib.request.urlopen
  45. The fields should all be Unicode strings.
  46. Subclasses of this one should re-define the _real_initialize() and
  47. _real_extract() methods and define a _VALID_URL regexp.
  48. Probably, they should also be added to the list of extractors.
  49. _real_extract() must return a *list* of information dictionaries as
  50. described above.
  51. Finally, the _WORKING attribute should be set to False for broken IEs
  52. in order to warn the users and skip the tests.
  53. """
  54. _ready = False
  55. _downloader = None
  56. _WORKING = True
  57. def __init__(self, downloader=None):
  58. """Constructor. Receives an optional downloader."""
  59. self._ready = False
  60. self.set_downloader(downloader)
  61. def suitable(self, url):
  62. """Receives a URL and returns True if suitable for this IE."""
  63. return re.match(self._VALID_URL, url) is not None
  64. def working(self):
  65. """Getter method for _WORKING."""
  66. return self._WORKING
  67. def initialize(self):
  68. """Initializes an instance (authentication, etc)."""
  69. if not self._ready:
  70. self._real_initialize()
  71. self._ready = True
  72. def extract(self, url):
  73. """Extracts URL information and returns it in list of dicts."""
  74. self.initialize()
  75. return self._real_extract(url)
  76. def set_downloader(self, downloader):
  77. """Sets the downloader for this IE."""
  78. self._downloader = downloader
  79. def _real_initialize(self):
  80. """Real initialization process. Redefine in subclasses."""
  81. pass
  82. def _real_extract(self, url):
  83. """Real extraction process. Redefine in subclasses."""
  84. pass
  85. @property
  86. def IE_NAME(self):
  87. return type(self).__name__[:-2]
  88. def _download_webpage(self, url_or_request, video_id, note=None, errnote=None):
  89. if note is None:
  90. note = u'Downloading video webpage'
  91. self._downloader.to_screen(u'[%s] %s: %s' % (self.IE_NAME, video_id, note))
  92. try:
  93. urlh = compat_urllib_request.urlopen(url_or_request)
  94. webpage_bytes = urlh.read()
  95. return webpage_bytes.decode('utf-8', 'replace')
  96. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  97. if errnote is None:
  98. errnote = u'Unable to download webpage'
  99. raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2])
  100. class YoutubeIE(InfoExtractor):
  101. """Information extractor for youtube.com."""
  102. _VALID_URL = r"""^
  103. (
  104. (?:https?://)? # http(s):// (optional)
  105. (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
  106. tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains
  107. (?:.*?\#/)? # handle anchor (#/) redirect urls
  108. (?!view_play_list|my_playlists|artist|playlist) # ignore playlist URLs
  109. (?: # the various things that can precede the ID:
  110. (?:(?:v|embed|e)/) # v/ or embed/ or e/
  111. |(?: # or the v= param in all its forms
  112. (?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
  113. (?:\?|\#!?) # the params delimiter ? or # or #!
  114. (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
  115. v=
  116. )
  117. )? # optional -> youtube.com/xxxx is OK
  118. )? # all until now is optional -> you can pass the naked ID
  119. ([0-9A-Za-z_-]+) # here is it! the YouTube video ID
  120. (?(1).+)? # if we found the ID, everything can follow
  121. $"""
  122. _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
  123. _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
  124. _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
  125. _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
  126. _NETRC_MACHINE = 'youtube'
  127. # Listed in order of quality
  128. _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
  129. _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
  130. _video_extensions = {
  131. '13': '3gp',
  132. '17': 'mp4',
  133. '18': 'mp4',
  134. '22': 'mp4',
  135. '37': 'mp4',
  136. '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
  137. '43': 'webm',
  138. '44': 'webm',
  139. '45': 'webm',
  140. '46': 'webm',
  141. }
  142. _video_dimensions = {
  143. '5': '240x400',
  144. '6': '???',
  145. '13': '???',
  146. '17': '144x176',
  147. '18': '360x640',
  148. '22': '720x1280',
  149. '34': '360x640',
  150. '35': '480x854',
  151. '37': '1080x1920',
  152. '38': '3072x4096',
  153. '43': '360x640',
  154. '44': '480x854',
  155. '45': '720x1280',
  156. '46': '1080x1920',
  157. }
  158. IE_NAME = u'youtube'
  159. def suitable(self, url):
  160. """Receives a URL and returns True if suitable for this IE."""
  161. return re.match(self._VALID_URL, url, re.VERBOSE) is not None
  162. def report_lang(self):
  163. """Report attempt to set language."""
  164. self._downloader.to_screen(u'[youtube] Setting language')
  165. def report_login(self):
  166. """Report attempt to log in."""
  167. self._downloader.to_screen(u'[youtube] Logging in')
  168. def report_age_confirmation(self):
  169. """Report attempt to confirm age."""
  170. self._downloader.to_screen(u'[youtube] Confirming age')
  171. def report_video_webpage_download(self, video_id):
  172. """Report attempt to download video webpage."""
  173. self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
  174. def report_video_info_webpage_download(self, video_id):
  175. """Report attempt to download video info webpage."""
  176. self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
  177. def report_video_subtitles_download(self, video_id):
  178. """Report attempt to download video info webpage."""
  179. self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id)
  180. def report_information_extraction(self, video_id):
  181. """Report attempt to extract video information."""
  182. self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
  183. def report_unavailable_format(self, video_id, format):
  184. """Report extracted video URL."""
  185. self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
  186. def report_rtmp_download(self):
  187. """Indicate the download will use the RTMP protocol."""
  188. self._downloader.to_screen(u'[youtube] RTMP download detected')
  189. def _closed_captions_xml_to_srt(self, xml_string):
  190. srt = ''
  191. texts = re.findall(r'<text start="([\d\.]+)"( dur="([\d\.]+)")?>([^<]+)</text>', xml_string, re.MULTILINE)
  192. # TODO parse xml instead of regex
  193. for n, (start, dur_tag, dur, caption) in enumerate(texts):
  194. if not dur: dur = '4'
  195. start = float(start)
  196. end = start + float(dur)
  197. start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
  198. end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
  199. caption = unescapeHTML(caption)
  200. caption = unescapeHTML(caption) # double cycle, intentional
  201. srt += str(n+1) + '\n'
  202. srt += start + ' --> ' + end + '\n'
  203. srt += caption + '\n\n'
  204. return srt
  205. def _extract_subtitles(self, video_id):
  206. self.report_video_subtitles_download(video_id)
  207. request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
  208. try:
  209. srt_list = compat_urllib_request.urlopen(request).read().decode('utf-8')
  210. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  211. return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None)
  212. srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
  213. srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
  214. if not srt_lang_list:
  215. return (u'WARNING: video has no closed captions', None)
  216. if self._downloader.params.get('subtitleslang', False):
  217. srt_lang = self._downloader.params.get('subtitleslang')
  218. elif 'en' in srt_lang_list:
  219. srt_lang = 'en'
  220. else:
  221. srt_lang = list(srt_lang_list.keys())[0]
  222. if not srt_lang in srt_lang_list:
  223. return (u'WARNING: no closed captions found in the specified language', None)
  224. request = compat_urllib_request.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
  225. try:
  226. srt_xml = compat_urllib_request.urlopen(request).read().decode('utf-8')
  227. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  228. return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None)
  229. if not srt_xml:
  230. return (u'WARNING: unable to download video subtitles', None)
  231. return (None, self._closed_captions_xml_to_srt(srt_xml))
  232. def _print_formats(self, formats):
  233. print('Available formats:')
  234. for x in formats:
  235. print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
  236. def _real_initialize(self):
  237. if self._downloader is None:
  238. return
  239. username = None
  240. password = None
  241. downloader_params = self._downloader.params
  242. # Attempt to use provided username and password or .netrc data
  243. if downloader_params.get('username', None) is not None:
  244. username = downloader_params['username']
  245. password = downloader_params['password']
  246. elif downloader_params.get('usenetrc', False):
  247. try:
  248. info = netrc.netrc().authenticators(self._NETRC_MACHINE)
  249. if info is not None:
  250. username = info[0]
  251. password = info[2]
  252. else:
  253. raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
  254. except (IOError, netrc.NetrcParseError) as err:
  255. self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
  256. return
  257. # Set language
  258. request = compat_urllib_request.Request(self._LANG_URL)
  259. try:
  260. self.report_lang()
  261. compat_urllib_request.urlopen(request).read()
  262. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  263. self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
  264. return
  265. # No authentication to be performed
  266. if username is None:
  267. return
  268. # Log in
  269. login_form = {
  270. 'current_form': 'loginForm',
  271. 'next': '/',
  272. 'action_login': 'Log In',
  273. 'username': username,
  274. 'password': password,
  275. }
  276. request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
  277. try:
  278. self.report_login()
  279. login_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
  280. if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
  281. self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
  282. return
  283. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  284. self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
  285. return
  286. # Confirm age
  287. age_form = {
  288. 'next_url': '/',
  289. 'action_confirm': 'Confirm',
  290. }
  291. request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
  292. try:
  293. self.report_age_confirmation()
  294. age_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
  295. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  296. self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
  297. return
  298. def _extract_id(self, url):
  299. mobj = re.match(self._VALID_URL, url, re.VERBOSE)
  300. if mobj is None:
  301. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  302. return
  303. video_id = mobj.group(2)
  304. return video_id
  305. def _real_extract(self, url):
  306. # Extract original video URL from URL with redirection, like age verification, using next_url parameter
  307. mobj = re.search(self._NEXT_URL_RE, url)
  308. if mobj:
  309. url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
  310. video_id = self._extract_id(url)
  311. # Get video webpage
  312. self.report_video_webpage_download(video_id)
  313. url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
  314. request = compat_urllib_request.Request(url)
  315. try:
  316. video_webpage_bytes = compat_urllib_request.urlopen(request).read()
  317. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  318. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
  319. return
  320. video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
  321. # Attempt to extract SWF player URL
  322. mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
  323. if mobj is not None:
  324. player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
  325. else:
  326. player_url = None
  327. # Get video info
  328. self.report_video_info_webpage_download(video_id)
  329. for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
  330. video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
  331. % (video_id, el_type))
  332. request = compat_urllib_request.Request(video_info_url)
  333. try:
  334. video_info_webpage_bytes = compat_urllib_request.urlopen(request).read()
  335. video_info_webpage = video_info_webpage_bytes.decode('utf-8', 'ignore')
  336. video_info = compat_parse_qs(video_info_webpage)
  337. if 'token' in video_info:
  338. break
  339. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  340. self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
  341. return
  342. if 'token' not in video_info:
  343. if 'reason' in video_info:
  344. self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0])
  345. else:
  346. self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
  347. return
  348. # Check for "rental" videos
  349. if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
  350. self._downloader.trouble(u'ERROR: "rental" videos not supported')
  351. return
  352. # Start extracting information
  353. self.report_information_extraction(video_id)
  354. # uploader
  355. if 'author' not in video_info:
  356. self._downloader.trouble(u'ERROR: unable to extract uploader name')
  357. return
  358. video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
  359. # uploader_id
  360. video_uploader_id = None
  361. mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
  362. if mobj is not None:
  363. video_uploader_id = mobj.group(1)
  364. else:
  365. self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
  366. # title
  367. if 'title' not in video_info:
  368. self._downloader.trouble(u'ERROR: unable to extract video title')
  369. return
  370. video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
  371. # thumbnail image
  372. if 'thumbnail_url' not in video_info:
  373. self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
  374. video_thumbnail = ''
  375. else: # don't panic if we can't find it
  376. video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
  377. # upload date
  378. upload_date = None
  379. mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
  380. if mobj is not None:
  381. upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
  382. format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
  383. for expression in format_expressions:
  384. try:
  385. upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
  386. except:
  387. pass
  388. # description
  389. video_description = get_element_by_id("eow-description", video_webpage)
  390. if video_description:
  391. video_description = clean_html(video_description)
  392. else:
  393. video_description = ''
  394. # closed captions
  395. video_subtitles = None
  396. if self._downloader.params.get('writesubtitles', False):
  397. (srt_error, video_subtitles) = self._extract_subtitles(video_id)
  398. if srt_error:
  399. self._downloader.trouble(srt_error)
  400. if 'length_seconds' not in video_info:
  401. self._downloader.trouble(u'WARNING: unable to extract video duration')
  402. video_duration = ''
  403. else:
  404. video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
  405. # token
  406. video_token = compat_urllib_parse.unquote_plus(video_info['token'][0])
  407. # Decide which formats to download
  408. req_format = self._downloader.params.get('format', None)
  409. if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
  410. self.report_rtmp_download()
  411. video_url_list = [(None, video_info['conn'][0])]
  412. elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
  413. url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
  414. url_data = [compat_parse_qs(uds) for uds in url_data_strs]
  415. url_data = [ud for ud in url_data if 'itag' in ud and 'url' in ud]
  416. url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
  417. format_limit = self._downloader.params.get('format_limit', None)
  418. available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
  419. if format_limit is not None and format_limit in available_formats:
  420. format_list = available_formats[available_formats.index(format_limit):]
  421. else:
  422. format_list = available_formats
  423. existing_formats = [x for x in format_list if x in url_map]
  424. if len(existing_formats) == 0:
  425. self._downloader.trouble(u'ERROR: no known formats available for video')
  426. return
  427. if self._downloader.params.get('listformats', None):
  428. self._print_formats(existing_formats)
  429. return
  430. if req_format is None or req_format == 'best':
  431. video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
  432. elif req_format == 'worst':
  433. video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
  434. elif req_format in ('-1', 'all'):
  435. video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
  436. else:
  437. # Specific formats. We pick the first in a slash-delimeted sequence.
  438. # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
  439. req_formats = req_format.split('/')
  440. video_url_list = None
  441. for rf in req_formats:
  442. if rf in url_map:
  443. video_url_list = [(rf, url_map[rf])]
  444. break
  445. if video_url_list is None:
  446. self._downloader.trouble(u'ERROR: requested format not available')
  447. return
  448. else:
  449. self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
  450. return
  451. results = []
  452. for format_param, video_real_url in video_url_list:
  453. # Extension
  454. video_extension = self._video_extensions.get(format_param, 'flv')
  455. video_format = '{0} - {1}'.format(format_param if format_param else video_extension,
  456. self._video_dimensions.get(format_param, '???'))
  457. results.append({
  458. 'id': video_id,
  459. 'url': video_real_url,
  460. 'uploader': video_uploader,
  461. 'uploader_id': video_uploader_id,
  462. 'upload_date': upload_date,
  463. 'title': video_title,
  464. 'ext': video_extension,
  465. 'format': video_format,
  466. 'thumbnail': video_thumbnail,
  467. 'description': video_description,
  468. 'player_url': player_url,
  469. 'subtitles': video_subtitles,
  470. 'duration': video_duration
  471. })
  472. return results
  473. class MetacafeIE(InfoExtractor):
  474. """Information Extractor for metacafe.com."""
  475. _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
  476. _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
  477. _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
  478. IE_NAME = u'metacafe'
  479. def __init__(self, downloader=None):
  480. InfoExtractor.__init__(self, downloader)
  481. def report_disclaimer(self):
  482. """Report disclaimer retrieval."""
  483. self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
  484. def report_age_confirmation(self):
  485. """Report attempt to confirm age."""
  486. self._downloader.to_screen(u'[metacafe] Confirming age')
  487. def report_download_webpage(self, video_id):
  488. """Report webpage download."""
  489. self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
  490. def report_extraction(self, video_id):
  491. """Report information extraction."""
  492. self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
  493. def _real_initialize(self):
  494. # Retrieve disclaimer
  495. request = compat_urllib_request.Request(self._DISCLAIMER)
  496. try:
  497. self.report_disclaimer()
  498. disclaimer = compat_urllib_request.urlopen(request).read()
  499. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  500. self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
  501. return
  502. # Confirm age
  503. disclaimer_form = {
  504. 'filters': '0',
  505. 'submit': "Continue - I'm over 18",
  506. }
  507. request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
  508. try:
  509. self.report_age_confirmation()
  510. disclaimer = compat_urllib_request.urlopen(request).read()
  511. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  512. self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
  513. return
  514. def _real_extract(self, url):
  515. # Extract id and simplified title from URL
  516. mobj = re.match(self._VALID_URL, url)
  517. if mobj is None:
  518. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  519. return
  520. video_id = mobj.group(1)
  521. # Check if video comes from YouTube
  522. mobj2 = re.match(r'^yt-(.*)$', video_id)
  523. if mobj2 is not None:
  524. self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
  525. return
  526. # Retrieve video webpage to extract further information
  527. request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
  528. try:
  529. self.report_download_webpage(video_id)
  530. webpage = compat_urllib_request.urlopen(request).read()
  531. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  532. self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
  533. return
  534. # Extract URL, uploader and title from webpage
  535. self.report_extraction(video_id)
  536. mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
  537. if mobj is not None:
  538. mediaURL = compat_urllib_parse.unquote(mobj.group(1))
  539. video_extension = mediaURL[-3:]
  540. # Extract gdaKey if available
  541. mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
  542. if mobj is None:
  543. video_url = mediaURL
  544. else:
  545. gdaKey = mobj.group(1)
  546. video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
  547. else:
  548. mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
  549. if mobj is None:
  550. self._downloader.trouble(u'ERROR: unable to extract media URL')
  551. return
  552. vardict = compat_parse_qs(mobj.group(1))
  553. if 'mediaData' not in vardict:
  554. self._downloader.trouble(u'ERROR: unable to extract media URL')
  555. return
  556. mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
  557. if mobj is None:
  558. self._downloader.trouble(u'ERROR: unable to extract media URL')
  559. return
  560. mediaURL = mobj.group(1).replace('\\/', '/')
  561. video_extension = mediaURL[-3:]
  562. video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
  563. mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
  564. if mobj is None:
  565. self._downloader.trouble(u'ERROR: unable to extract title')
  566. return
  567. video_title = mobj.group(1).decode('utf-8')
  568. mobj = re.search(r'submitter=(.*?);', webpage)
  569. if mobj is None:
  570. self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
  571. return
  572. video_uploader = mobj.group(1)
  573. return [{
  574. 'id': video_id.decode('utf-8'),
  575. 'url': video_url.decode('utf-8'),
  576. 'uploader': video_uploader.decode('utf-8'),
  577. 'upload_date': None,
  578. 'title': video_title,
  579. 'ext': video_extension.decode('utf-8'),
  580. }]
  581. class DailymotionIE(InfoExtractor):
  582. """Information Extractor for Dailymotion"""
  583. _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
  584. IE_NAME = u'dailymotion'
  585. def __init__(self, downloader=None):
  586. InfoExtractor.__init__(self, downloader)
  587. def report_extraction(self, video_id):
  588. """Report information extraction."""
  589. self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
  590. def _real_extract(self, url):
  591. # Extract id and simplified title from URL
  592. mobj = re.match(self._VALID_URL, url)
  593. if mobj is None:
  594. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  595. return
  596. video_id = mobj.group(1).split('_')[0].split('?')[0]
  597. video_extension = 'mp4'
  598. # Retrieve video webpage to extract further information
  599. request = compat_urllib_request.Request(url)
  600. request.add_header('Cookie', 'family_filter=off')
  601. webpage = self._download_webpage(request, video_id)
  602. # Extract URL, uploader and title from webpage
  603. self.report_extraction(video_id)
  604. mobj = re.search(r'\s*var flashvars = (.*)', webpage)
  605. if mobj is None:
  606. self._downloader.trouble(u'ERROR: unable to extract media URL')
  607. return
  608. flashvars = compat_urllib_parse.unquote(mobj.group(1))
  609. for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
  610. if key in flashvars:
  611. max_quality = key
  612. self._downloader.to_screen(u'[dailymotion] Using %s' % key)
  613. break
  614. else:
  615. self._downloader.trouble(u'ERROR: unable to extract video URL')
  616. return
  617. mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
  618. if mobj is None:
  619. self._downloader.trouble(u'ERROR: unable to extract video URL')
  620. return
  621. video_url = compat_urllib_parse.unquote(mobj.group(1)).replace('\\/', '/')
  622. # TODO: support choosing qualities
  623. mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
  624. if mobj is None:
  625. self._downloader.trouble(u'ERROR: unable to extract title')
  626. return
  627. video_title = unescapeHTML(mobj.group('title'))
  628. video_uploader = None
  629. mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
  630. if mobj is None:
  631. # lookin for official user
  632. mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage)
  633. if mobj_official is None:
  634. self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
  635. else:
  636. video_uploader = mobj_official.group(1)
  637. else:
  638. video_uploader = mobj.group(1)
  639. video_upload_date = None
  640. mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
  641. if mobj is not None:
  642. video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
  643. return [{
  644. 'id': video_id,
  645. 'url': video_url,
  646. 'uploader': video_uploader,
  647. 'upload_date': video_upload_date,
  648. 'title': video_title,
  649. 'ext': video_extension,
  650. }]
  651. class PhotobucketIE(InfoExtractor):
  652. """Information extractor for photobucket.com."""
  653. _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
  654. IE_NAME = u'photobucket'
  655. def __init__(self, downloader=None):
  656. InfoExtractor.__init__(self, downloader)
  657. def report_download_webpage(self, video_id):
  658. """Report webpage download."""
  659. self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
  660. def report_extraction(self, video_id):
  661. """Report information extraction."""
  662. self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
  663. def _real_extract(self, url):
  664. # Extract id from URL
  665. mobj = re.match(self._VALID_URL, url)
  666. if mobj is None:
  667. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  668. return
  669. video_id = mobj.group(1)
  670. video_extension = 'flv'
  671. # Retrieve video webpage to extract further information
  672. request = compat_urllib_request.Request(url)
  673. try:
  674. self.report_download_webpage(video_id)
  675. webpage = compat_urllib_request.urlopen(request).read()
  676. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  677. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  678. return
  679. # Extract URL, uploader, and title from webpage
  680. self.report_extraction(video_id)
  681. mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
  682. if mobj is None:
  683. self._downloader.trouble(u'ERROR: unable to extract media URL')
  684. return
  685. mediaURL = compat_urllib_parse.unquote(mobj.group(1))
  686. video_url = mediaURL
  687. mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
  688. if mobj is None:
  689. self._downloader.trouble(u'ERROR: unable to extract title')
  690. return
  691. video_title = mobj.group(1).decode('utf-8')
  692. video_uploader = mobj.group(2).decode('utf-8')
  693. return [{
  694. 'id': video_id.decode('utf-8'),
  695. 'url': video_url.decode('utf-8'),
  696. 'uploader': video_uploader,
  697. 'upload_date': None,
  698. 'title': video_title,
  699. 'ext': video_extension.decode('utf-8'),
  700. }]
  701. class YahooIE(InfoExtractor):
  702. """Information extractor for video.yahoo.com."""
  703. _WORKING = False
  704. # _VALID_URL matches all Yahoo! Video URLs
  705. # _VPAGE_URL matches only the extractable '/watch/' URLs
  706. _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
  707. _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
  708. IE_NAME = u'video.yahoo'
  709. def __init__(self, downloader=None):
  710. InfoExtractor.__init__(self, downloader)
  711. def report_download_webpage(self, video_id):
  712. """Report webpage download."""
  713. self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
  714. def report_extraction(self, video_id):
  715. """Report information extraction."""
  716. self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
  717. def _real_extract(self, url, new_video=True):
  718. # Extract ID from URL
  719. mobj = re.match(self._VALID_URL, url)
  720. if mobj is None:
  721. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  722. return
  723. video_id = mobj.group(2)
  724. video_extension = 'flv'
  725. # Rewrite valid but non-extractable URLs as
  726. # extractable English language /watch/ URLs
  727. if re.match(self._VPAGE_URL, url) is None:
  728. request = compat_urllib_request.Request(url)
  729. try:
  730. webpage = compat_urllib_request.urlopen(request).read()
  731. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  732. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  733. return
  734. mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
  735. if mobj is None:
  736. self._downloader.trouble(u'ERROR: Unable to extract id field')
  737. return
  738. yahoo_id = mobj.group(1)
  739. mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
  740. if mobj is None:
  741. self._downloader.trouble(u'ERROR: Unable to extract vid field')
  742. return
  743. yahoo_vid = mobj.group(1)
  744. url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
  745. return self._real_extract(url, new_video=False)
  746. # Retrieve video webpage to extract further information
  747. request = compat_urllib_request.Request(url)
  748. try:
  749. self.report_download_webpage(video_id)
  750. webpage = compat_urllib_request.urlopen(request).read()
  751. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  752. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  753. return
  754. # Extract uploader and title from webpage
  755. self.report_extraction(video_id)
  756. mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
  757. if mobj is None:
  758. self._downloader.trouble(u'ERROR: unable to extract video title')
  759. return
  760. video_title = mobj.group(1).decode('utf-8')
  761. mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
  762. if mobj is None:
  763. self._downloader.trouble(u'ERROR: unable to extract video uploader')
  764. return
  765. video_uploader = mobj.group(1).decode('utf-8')
  766. # Extract video thumbnail
  767. mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
  768. if mobj is None:
  769. self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
  770. return
  771. video_thumbnail = mobj.group(1).decode('utf-8')
  772. # Extract video description
  773. mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
  774. if mobj is None:
  775. self._downloader.trouble(u'ERROR: unable to extract video description')
  776. return
  777. video_description = mobj.group(1).decode('utf-8')
  778. if not video_description:
  779. video_description = 'No description available.'
  780. # Extract video height and width
  781. mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
  782. if mobj is None:
  783. self._downloader.trouble(u'ERROR: unable to extract video height')
  784. return
  785. yv_video_height = mobj.group(1)
  786. mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
  787. if mobj is None:
  788. self._downloader.trouble(u'ERROR: unable to extract video width')
  789. return
  790. yv_video_width = mobj.group(1)
  791. # Retrieve video playlist to extract media URL
  792. # I'm not completely sure what all these options are, but we
  793. # seem to need most of them, otherwise the server sends a 401.
  794. yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
  795. yv_bitrate = '700' # according to Wikipedia this is hard-coded
  796. request = compat_urllib_request.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
  797. '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
  798. '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
  799. try:
  800. self.report_download_webpage(video_id)
  801. webpage = compat_urllib_request.urlopen(request).read()
  802. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  803. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  804. return
  805. # Extract media URL from playlist XML
  806. mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
  807. if mobj is None:
  808. self._downloader.trouble(u'ERROR: Unable to extract media URL')
  809. return
  810. video_url = compat_urllib_parse.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
  811. video_url = unescapeHTML(video_url)
  812. return [{
  813. 'id': video_id.decode('utf-8'),
  814. 'url': video_url,
  815. 'uploader': video_uploader,
  816. 'upload_date': None,
  817. 'title': video_title,
  818. 'ext': video_extension.decode('utf-8'),
  819. 'thumbnail': video_thumbnail.decode('utf-8'),
  820. 'description': video_description,
  821. }]
  822. class VimeoIE(InfoExtractor):
  823. """Information extractor for vimeo.com."""
  824. # _VALID_URL matches Vimeo URLs
  825. _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?:videos?/)?([0-9]+)'
  826. IE_NAME = u'vimeo'
  827. def __init__(self, downloader=None):
  828. InfoExtractor.__init__(self, downloader)
  829. def report_download_webpage(self, video_id):
  830. """Report webpage download."""
  831. self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
  832. def report_extraction(self, video_id):
  833. """Report information extraction."""
  834. self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
  835. def _real_extract(self, url, new_video=True):
  836. # Extract ID from URL
  837. mobj = re.match(self._VALID_URL, url)
  838. if mobj is None:
  839. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  840. return
  841. video_id = mobj.group(1)
  842. # Retrieve video webpage to extract further information
  843. request = compat_urllib_request.Request(url, None, std_headers)
  844. try:
  845. self.report_download_webpage(video_id)
  846. webpage_bytes = compat_urllib_request.urlopen(request).read()
  847. webpage = webpage_bytes.decode('utf-8')
  848. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  849. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  850. return
  851. # Now we begin extracting as much information as we can from what we
  852. # retrieved. First we extract the information common to all extractors,
  853. # and latter we extract those that are Vimeo specific.
  854. self.report_extraction(video_id)
  855. # Extract the config JSON
  856. try:
  857. config = webpage.split(' = {config:')[1].split(',assets:')[0]
  858. config = json.loads(config)
  859. except:
  860. self._downloader.trouble(u'ERROR: unable to extract info section')
  861. return
  862. # Extract title
  863. video_title = config["video"]["title"]
  864. # Extract uploader and uploader_id
  865. video_uploader = config["video"]["owner"]["name"]
  866. video_uploader_id = config["video"]["owner"]["url"].split('/')[-1]
  867. # Extract video thumbnail
  868. video_thumbnail = config["video"]["thumbnail"]
  869. # Extract video description
  870. video_description = get_element_by_attribute("itemprop", "description", webpage)
  871. if video_description: video_description = clean_html(video_description)
  872. else: video_description = ''
  873. # Extract upload date
  874. video_upload_date = None
  875. mobj = re.search(r'<meta itemprop="dateCreated" content="(\d{4})-(\d{2})-(\d{2})T', webpage)
  876. if mobj is not None:
  877. video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
  878. # Vimeo specific: extract request signature and timestamp
  879. sig = config['request']['signature']
  880. timestamp = config['request']['timestamp']
  881. # Vimeo specific: extract video codec and quality information
  882. # First consider quality, then codecs, then take everything
  883. # TODO bind to format param
  884. codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
  885. files = { 'hd': [], 'sd': [], 'other': []}
  886. for codec_name, codec_extension in codecs:
  887. if codec_name in config["video"]["files"]:
  888. if 'hd' in config["video"]["files"][codec_name]:
  889. files['hd'].append((codec_name, codec_extension, 'hd'))
  890. elif 'sd' in config["video"]["files"][codec_name]:
  891. files['sd'].append((codec_name, codec_extension, 'sd'))
  892. else:
  893. files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0]))
  894. for quality in ('hd', 'sd', 'other'):
  895. if len(files[quality]) > 0:
  896. video_quality = files[quality][0][2]
  897. video_codec = files[quality][0][0]
  898. video_extension = files[quality][0][1]
  899. self._downloader.to_screen(u'[vimeo] %s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality))
  900. break
  901. else:
  902. self._downloader.trouble(u'ERROR: no known codec found')
  903. return
  904. video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
  905. %(video_id, sig, timestamp, video_quality, video_codec.upper())
  906. return [{
  907. 'id': video_id,
  908. 'url': video_url,
  909. 'uploader': video_uploader,
  910. 'uploader_id': video_uploader_id,
  911. 'upload_date': video_upload_date,
  912. 'title': video_title,
  913. 'ext': video_extension,
  914. 'thumbnail': video_thumbnail,
  915. 'description': video_description,
  916. }]
  917. class ArteTvIE(InfoExtractor):
  918. """arte.tv information extractor."""
  919. _VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*'
  920. _LIVE_URL = r'index-[0-9]+\.html$'
  921. IE_NAME = u'arte.tv'
  922. def __init__(self, downloader=None):
  923. InfoExtractor.__init__(self, downloader)
  924. def report_download_webpage(self, video_id):
  925. """Report webpage download."""
  926. self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id)
  927. def report_extraction(self, video_id):
  928. """Report information extraction."""
  929. self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id)
  930. def fetch_webpage(self, url):
  931. request = compat_urllib_request.Request(url)
  932. try:
  933. self.report_download_webpage(url)
  934. webpage = compat_urllib_request.urlopen(request).read()
  935. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  936. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  937. return
  938. except ValueError as err:
  939. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  940. return
  941. return webpage
  942. def grep_webpage(self, url, regex, regexFlags, matchTuples):
  943. page = self.fetch_webpage(url)
  944. mobj = re.search(regex, page, regexFlags)
  945. info = {}
  946. if mobj is None:
  947. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  948. return
  949. for (i, key, err) in matchTuples:
  950. if mobj.group(i) is None:
  951. self._downloader.trouble(err)
  952. return
  953. else:
  954. info[key] = mobj.group(i)
  955. return info
  956. def extractLiveStream(self, url):
  957. video_lang = url.split('/')[-4]
  958. info = self.grep_webpage(
  959. url,
  960. r'src="(.*?/videothek_js.*?\.js)',
  961. 0,
  962. [
  963. (1, 'url', u'ERROR: Invalid URL: %s' % url)
  964. ]
  965. )
  966. http_host = url.split('/')[2]
  967. next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url')))
  968. info = self.grep_webpage(
  969. next_url,
  970. r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
  971. '(http://.*?\.swf).*?' +
  972. '(rtmp://.*?)\'',
  973. re.DOTALL,
  974. [
  975. (1, 'path', u'ERROR: could not extract video path: %s' % url),
  976. (2, 'player', u'ERROR: could not extract video player: %s' % url),
  977. (3, 'url', u'ERROR: could not extract video url: %s' % url)
  978. ]
  979. )
  980. video_url = u'%s/%s' % (info.get('url'), info.get('path'))
  981. def extractPlus7Stream(self, url):
  982. video_lang = url.split('/')[-3]
  983. info = self.grep_webpage(
  984. url,
  985. r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
  986. 0,
  987. [
  988. (1, 'url', u'ERROR: Invalid URL: %s' % url)
  989. ]
  990. )
  991. next_url = compat_urllib_parse.unquote(info.get('url'))
  992. info = self.grep_webpage(
  993. next_url,
  994. r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
  995. 0,
  996. [
  997. (1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
  998. ]
  999. )
  1000. next_url = compat_urllib_parse.unquote(info.get('url'))
  1001. info = self.grep_webpage(
  1002. next_url,
  1003. r'<video id="(.*?)".*?>.*?' +
  1004. '<name>(.*?)</name>.*?' +
  1005. '<dateVideo>(.*?)</dateVideo>.*?' +
  1006. '<url quality="hd">(.*?)</url>',
  1007. re.DOTALL,
  1008. [
  1009. (1, 'id', u'ERROR: could not extract video id: %s' % url),
  1010. (2, 'title', u'ERROR: could not extract video title: %s' % url),
  1011. (3, 'date', u'ERROR: could not extract video date: %s' % url),
  1012. (4, 'url', u'ERROR: could not extract video url: %s' % url)
  1013. ]
  1014. )
  1015. return {
  1016. 'id': info.get('id'),
  1017. 'url': compat_urllib_parse.unquote(info.get('url')),
  1018. 'uploader': u'arte.tv',
  1019. 'upload_date': info.get('date'),
  1020. 'title': info.get('title').decode('utf-8'),
  1021. 'ext': u'mp4',
  1022. 'format': u'NA',
  1023. 'player_url': None,
  1024. }
  1025. def _real_extract(self, url):
  1026. video_id = url.split('/')[-1]
  1027. self.report_extraction(video_id)
  1028. if re.search(self._LIVE_URL, video_id) is not None:
  1029. self.extractLiveStream(url)
  1030. return
  1031. else:
  1032. info = self.extractPlus7Stream(url)
  1033. return [info]
  1034. class GenericIE(InfoExtractor):
  1035. """Generic last-resort information extractor."""
  1036. _VALID_URL = r'.*'
  1037. IE_NAME = u'generic'
  1038. def __init__(self, downloader=None):
  1039. InfoExtractor.__init__(self, downloader)
  1040. def report_download_webpage(self, video_id):
  1041. """Report webpage download."""
  1042. self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
  1043. self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
  1044. def report_extraction(self, video_id):
  1045. """Report information extraction."""
  1046. self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
  1047. def report_following_redirect(self, new_url):
  1048. """Report information extraction."""
  1049. self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
  1050. def _test_redirect(self, url):
  1051. """Check if it is a redirect, like url shorteners, in case restart chain."""
  1052. class HeadRequest(compat_urllib_request.Request):
  1053. def get_method(self):
  1054. return "HEAD"
  1055. class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
  1056. """
  1057. Subclass the HTTPRedirectHandler to make it use our
  1058. HeadRequest also on the redirected URL
  1059. """
  1060. def redirect_request(self, req, fp, code, msg, headers, newurl):
  1061. if code in (301, 302, 303, 307):
  1062. newurl = newurl.replace(' ', '%20')
  1063. newheaders = dict((k,v) for k,v in req.headers.items()
  1064. if k.lower() not in ("content-length", "content-type"))
  1065. return HeadRequest(newurl,
  1066. headers=newheaders,
  1067. origin_req_host=req.get_origin_req_host(),
  1068. unverifiable=True)
  1069. else:
  1070. raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp)
  1071. class HTTPMethodFallback(compat_urllib_request.BaseHandler):
  1072. """
  1073. Fallback to GET if HEAD is not allowed (405 HTTP error)
  1074. """
  1075. def http_error_405(self, req, fp, code, msg, headers):
  1076. fp.read()
  1077. fp.close()
  1078. newheaders = dict((k,v) for k,v in req.headers.items()
  1079. if k.lower() not in ("content-length", "content-type"))
  1080. return self.parent.open(compat_urllib_request.Request(req.get_full_url(),
  1081. headers=newheaders,
  1082. origin_req_host=req.get_origin_req_host(),
  1083. unverifiable=True))
  1084. # Build our opener
  1085. opener = compat_urllib_request.OpenerDirector()
  1086. for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler,
  1087. HTTPMethodFallback, HEADRedirectHandler,
  1088. compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
  1089. opener.add_handler(handler())
  1090. response = opener.open(HeadRequest(url))
  1091. new_url = response.geturl()
  1092. if url == new_url:
  1093. return False
  1094. self.report_following_redirect(new_url)
  1095. self._downloader.download([new_url])
  1096. return True
  1097. def _real_extract(self, url):
  1098. if self._test_redirect(url): return
  1099. video_id = url.split('/')[-1]
  1100. request = compat_urllib_request.Request(url)
  1101. try:
  1102. self.report_download_webpage(video_id)
  1103. webpage = compat_urllib_request.urlopen(request).read()
  1104. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1105. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  1106. return
  1107. except ValueError as err:
  1108. # since this is the last-resort InfoExtractor, if
  1109. # this error is thrown, it'll be thrown here
  1110. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  1111. return
  1112. self.report_extraction(video_id)
  1113. # Start with something easy: JW Player in SWFObject
  1114. mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
  1115. if mobj is None:
  1116. # Broaden the search a little bit
  1117. mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
  1118. if mobj is None:
  1119. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  1120. return
  1121. # It's possible that one of the regexes
  1122. # matched, but returned an empty group:
  1123. if mobj.group(1) is None:
  1124. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  1125. return
  1126. video_url = compat_urllib_parse.unquote(mobj.group(1))
  1127. video_id = os.path.basename(video_url)
  1128. # here's a fun little line of code for you:
  1129. video_extension = os.path.splitext(video_id)[1][1:]
  1130. video_id = os.path.splitext(video_id)[0]
  1131. # it's tempting to parse this further, but you would
  1132. # have to take into account all the variations like
  1133. # Video Title - Site Name
  1134. # Site Name | Video Title
  1135. # Video Title - Tagline | Site Name
  1136. # and so on and so forth; it's just not practical
  1137. mobj = re.search(r'<title>(.*)</title>', webpage)
  1138. if mobj is None:
  1139. self._downloader.trouble(u'ERROR: unable to extract title')
  1140. return
  1141. video_title = mobj.group(1)
  1142. # video uploader is domain name
  1143. mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
  1144. if mobj is None:
  1145. self._downloader.trouble(u'ERROR: unable to extract title')
  1146. return
  1147. video_uploader = mobj.group(1)
  1148. return [{
  1149. 'id': video_id,
  1150. 'url': video_url,
  1151. 'uploader': video_uploader,
  1152. 'upload_date': None,
  1153. 'title': video_title,
  1154. 'ext': video_extension,
  1155. }]
  1156. class YoutubeSearchIE(InfoExtractor):
  1157. """Information Extractor for YouTube search queries."""
  1158. _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
  1159. _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
  1160. _max_youtube_results = 1000
  1161. IE_NAME = u'youtube:search'
  1162. def __init__(self, downloader=None):
  1163. InfoExtractor.__init__(self, downloader)
  1164. def report_download_page(self, query, pagenum):
  1165. """Report attempt to download search page with given number."""
  1166. query = query.decode(preferredencoding())
  1167. self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
  1168. def _real_extract(self, query):
  1169. mobj = re.match(self._VALID_URL, query)
  1170. if mobj is None:
  1171. self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
  1172. return
  1173. prefix, query = query.split(':')
  1174. prefix = prefix[8:]
  1175. query = query.encode('utf-8')
  1176. if prefix == '':
  1177. self._download_n_results(query, 1)
  1178. return
  1179. elif prefix == 'all':
  1180. self._download_n_results(query, self._max_youtube_results)
  1181. return
  1182. else:
  1183. try:
  1184. n = int(prefix)
  1185. if n <= 0:
  1186. self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
  1187. return
  1188. elif n > self._max_youtube_results:
  1189. self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
  1190. n = self._max_youtube_results
  1191. self._download_n_results(query, n)
  1192. return
  1193. except ValueError: # parsing prefix as integer fails
  1194. self._download_n_results(query, 1)
  1195. return
  1196. def _download_n_results(self, query, n):
  1197. """Downloads a specified number of results for a query"""
  1198. video_ids = []
  1199. pagenum = 0
  1200. limit = n
  1201. while (50 * pagenum) < limit:
  1202. self.report_download_page(query, pagenum+1)
  1203. result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
  1204. request = compat_urllib_request.Request(result_url)
  1205. try:
  1206. data = compat_urllib_request.urlopen(request).read()
  1207. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1208. self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
  1209. return
  1210. api_response = json.loads(data)['data']
  1211. new_ids = list(video['id'] for video in api_response['items'])
  1212. video_ids += new_ids
  1213. limit = min(n, api_response['totalItems'])
  1214. pagenum += 1
  1215. if len(video_ids) > n:
  1216. video_ids = video_ids[:n]
  1217. for id in video_ids:
  1218. self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
  1219. return
  1220. class GoogleSearchIE(InfoExtractor):
  1221. """Information Extractor for Google Video search queries."""
  1222. _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
  1223. _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
  1224. _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
  1225. _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
  1226. _max_google_results = 1000
  1227. IE_NAME = u'video.google:search'
  1228. def __init__(self, downloader=None):
  1229. InfoExtractor.__init__(self, downloader)
  1230. def report_download_page(self, query, pagenum):
  1231. """Report attempt to download playlist page with given number."""
  1232. query = query.decode(preferredencoding())
  1233. self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
  1234. def _real_extract(self, query):
  1235. mobj = re.match(self._VALID_URL, query)
  1236. if mobj is None:
  1237. self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
  1238. return
  1239. prefix, query = query.split(':')
  1240. prefix = prefix[8:]
  1241. query = query.encode('utf-8')
  1242. if prefix == '':
  1243. self._download_n_results(query, 1)
  1244. return
  1245. elif prefix == 'all':
  1246. self._download_n_results(query, self._max_google_results)
  1247. return
  1248. else:
  1249. try:
  1250. n = int(prefix)
  1251. if n <= 0:
  1252. self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
  1253. return
  1254. elif n > self._max_google_results:
  1255. self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
  1256. n = self._max_google_results
  1257. self._download_n_results(query, n)
  1258. return
  1259. except ValueError: # parsing prefix as integer fails
  1260. self._download_n_results(query, 1)
  1261. return
  1262. def _download_n_results(self, query, n):
  1263. """Downloads a specified number of results for a query"""
  1264. video_ids = []
  1265. pagenum = 0
  1266. while True:
  1267. self.report_download_page(query, pagenum)
  1268. result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum*10)
  1269. request = compat_urllib_request.Request(result_url)
  1270. try:
  1271. page = compat_urllib_request.urlopen(request).read()
  1272. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1273. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1274. return
  1275. # Extract video identifiers
  1276. for mobj in re.finditer(self._VIDEO_INDICATOR, page):
  1277. video_id = mobj.group(1)
  1278. if video_id not in video_ids:
  1279. video_ids.append(video_id)
  1280. if len(video_ids) == n:
  1281. # Specified n videos reached
  1282. for id in video_ids:
  1283. self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
  1284. return
  1285. if re.search(self._MORE_PAGES_INDICATOR, page) is None:
  1286. for id in video_ids:
  1287. self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
  1288. return
  1289. pagenum = pagenum + 1
  1290. class YahooSearchIE(InfoExtractor):
  1291. """Information Extractor for Yahoo! Video search queries."""
  1292. _WORKING = False
  1293. _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
  1294. _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
  1295. _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
  1296. _MORE_PAGES_INDICATOR = r'\s*Next'
  1297. _max_yahoo_results = 1000
  1298. IE_NAME = u'video.yahoo:search'
  1299. def __init__(self, downloader=None):
  1300. InfoExtractor.__init__(self, downloader)
  1301. def report_download_page(self, query, pagenum):
  1302. """Report attempt to download playlist page with given number."""
  1303. query = query.decode(preferredencoding())
  1304. self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
  1305. def _real_extract(self, query):
  1306. mobj = re.match(self._VALID_URL, query)
  1307. if mobj is None:
  1308. self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
  1309. return
  1310. prefix, query = query.split(':')
  1311. prefix = prefix[8:]
  1312. query = query.encode('utf-8')
  1313. if prefix == '':
  1314. self._download_n_results(query, 1)
  1315. return
  1316. elif prefix == 'all':
  1317. self._download_n_results(query, self._max_yahoo_results)
  1318. return
  1319. else:
  1320. try:
  1321. n = int(prefix)
  1322. if n <= 0:
  1323. self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
  1324. return
  1325. elif n > self._max_yahoo_results:
  1326. self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
  1327. n = self._max_yahoo_results
  1328. self._download_n_results(query, n)
  1329. return
  1330. except ValueError: # parsing prefix as integer fails
  1331. self._download_n_results(query, 1)
  1332. return
  1333. def _download_n_results(self, query, n):
  1334. """Downloads a specified number of results for a query"""
  1335. video_ids = []
  1336. already_seen = set()
  1337. pagenum = 1
  1338. while True:
  1339. self.report_download_page(query, pagenum)
  1340. result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum)
  1341. request = compat_urllib_request.Request(result_url)
  1342. try:
  1343. page = compat_urllib_request.urlopen(request).read()
  1344. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1345. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1346. return
  1347. # Extract video identifiers
  1348. for mobj in re.finditer(self._VIDEO_INDICATOR, page):
  1349. video_id = mobj.group(1)
  1350. if video_id not in already_seen:
  1351. video_ids.append(video_id)
  1352. already_seen.add(video_id)
  1353. if len(video_ids) == n:
  1354. # Specified n videos reached
  1355. for id in video_ids:
  1356. self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
  1357. return
  1358. if re.search(self._MORE_PAGES_INDICATOR, page) is None:
  1359. for id in video_ids:
  1360. self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
  1361. return
  1362. pagenum = pagenum + 1
  1363. class YoutubePlaylistIE(InfoExtractor):
  1364. """Information Extractor for YouTube playlists."""
  1365. _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
  1366. _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
  1367. _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;([^&"]+&amp;)*list=.*?%s'
  1368. _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
  1369. IE_NAME = u'youtube:playlist'
  1370. def __init__(self, downloader=None):
  1371. InfoExtractor.__init__(self, downloader)
  1372. def report_download_page(self, playlist_id, pagenum):
  1373. """Report attempt to download playlist page with given number."""
  1374. self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
  1375. def _real_extract(self, url):
  1376. # Extract playlist id
  1377. mobj = re.match(self._VALID_URL, url)
  1378. if mobj is None:
  1379. self._downloader.trouble(u'ERROR: invalid url: %s' % url)
  1380. return
  1381. # Single video case
  1382. if mobj.group(3) is not None:
  1383. self._downloader.download([mobj.group(3)])
  1384. return
  1385. # Download playlist pages
  1386. # prefix is 'p' as default for playlists but there are other types that need extra care
  1387. playlist_prefix = mobj.group(1)
  1388. if playlist_prefix == 'a':
  1389. playlist_access = 'artist'
  1390. else:
  1391. playlist_prefix = 'p'
  1392. playlist_access = 'view_play_list'
  1393. playlist_id = mobj.group(2)
  1394. video_ids = []
  1395. pagenum = 1
  1396. while True:
  1397. self.report_download_page(playlist_id, pagenum)
  1398. url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
  1399. request = compat_urllib_request.Request(url)
  1400. try:
  1401. page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  1402. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1403. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1404. return
  1405. # Extract video identifiers
  1406. ids_in_page = []
  1407. for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page):
  1408. if mobj.group(1) not in ids_in_page:
  1409. ids_in_page.append(mobj.group(1))
  1410. video_ids.extend(ids_in_page)
  1411. if self._MORE_PAGES_INDICATOR not in page:
  1412. break
  1413. pagenum = pagenum + 1
  1414. total = len(video_ids)
  1415. playliststart = self._downloader.params.get('playliststart', 1) - 1
  1416. playlistend = self._downloader.params.get('playlistend', -1)
  1417. if playlistend == -1:
  1418. video_ids = video_ids[playliststart:]
  1419. else:
  1420. video_ids = video_ids[playliststart:playlistend]
  1421. if len(video_ids) == total:
  1422. self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total))
  1423. else:
  1424. self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids)))
  1425. for id in video_ids:
  1426. self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
  1427. return
  1428. class YoutubeChannelIE(InfoExtractor):
  1429. """Information Extractor for YouTube channels."""
  1430. _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
  1431. _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
  1432. _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
  1433. IE_NAME = u'youtube:channel'
  1434. def report_download_page(self, channel_id, pagenum):
  1435. """Report attempt to download channel page with given number."""
  1436. self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
  1437. def _real_extract(self, url):
  1438. # Extract channel id
  1439. mobj = re.match(self._VALID_URL, url)
  1440. if mobj is None:
  1441. self._downloader.trouble(u'ERROR: invalid url: %s' % url)
  1442. return
  1443. # Download channel pages
  1444. channel_id = mobj.group(1)
  1445. video_ids = []
  1446. pagenum = 1
  1447. while True:
  1448. self.report_download_page(channel_id, pagenum)
  1449. url = self._TEMPLATE_URL % (channel_id, pagenum)
  1450. request = compat_urllib_request.Request(url)
  1451. try:
  1452. page = compat_urllib_request.urlopen(request).read().decode('utf8')
  1453. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1454. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1455. return
  1456. # Extract video identifiers
  1457. ids_in_page = []
  1458. for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
  1459. if mobj.group(1) not in ids_in_page:
  1460. ids_in_page.append(mobj.group(1))
  1461. video_ids.extend(ids_in_page)
  1462. if self._MORE_PAGES_INDICATOR not in page:
  1463. break
  1464. pagenum = pagenum + 1
  1465. self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
  1466. for id in video_ids:
  1467. self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
  1468. return
  1469. class YoutubeUserIE(InfoExtractor):
  1470. """Information Extractor for YouTube users."""
  1471. _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
  1472. _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
  1473. _GDATA_PAGE_SIZE = 50
  1474. _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
  1475. _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
  1476. IE_NAME = u'youtube:user'
  1477. def __init__(self, downloader=None):
  1478. InfoExtractor.__init__(self, downloader)
  1479. def report_download_page(self, username, start_index):
  1480. """Report attempt to download user page."""
  1481. self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
  1482. (username, start_index, start_index + self._GDATA_PAGE_SIZE))
  1483. def _real_extract(self, url):
  1484. # Extract username
  1485. mobj = re.match(self._VALID_URL, url)
  1486. if mobj is None:
  1487. self._downloader.trouble(u'ERROR: invalid url: %s' % url)
  1488. return
  1489. username = mobj.group(1)
  1490. # Download video ids using YouTube Data API. Result size per
  1491. # query is limited (currently to 50 videos) so we need to query
  1492. # page by page until there are no video ids - it means we got
  1493. # all of them.
  1494. video_ids = []
  1495. pagenum = 0
  1496. while True:
  1497. start_index = pagenum * self._GDATA_PAGE_SIZE + 1
  1498. self.report_download_page(username, start_index)
  1499. request = compat_urllib_request.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
  1500. try:
  1501. page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  1502. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1503. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1504. return
  1505. # Extract video identifiers
  1506. ids_in_page = []
  1507. for mobj in re.finditer(self._VIDEO_INDICATOR, page):
  1508. if mobj.group(1) not in ids_in_page:
  1509. ids_in_page.append(mobj.group(1))
  1510. video_ids.extend(ids_in_page)
  1511. # A little optimization - if current page is not
  1512. # "full", ie. does not contain PAGE_SIZE video ids then
  1513. # we can assume that this page is the last one - there
  1514. # are no more ids on further pages - no need to query
  1515. # again.
  1516. if len(ids_in_page) < self._GDATA_PAGE_SIZE:
  1517. break
  1518. pagenum += 1
  1519. all_ids_count = len(video_ids)
  1520. playliststart = self._downloader.params.get('playliststart', 1) - 1
  1521. playlistend = self._downloader.params.get('playlistend', -1)
  1522. if playlistend == -1:
  1523. video_ids = video_ids[playliststart:]
  1524. else:
  1525. video_ids = video_ids[playliststart:playlistend]
  1526. self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
  1527. (username, all_ids_count, len(video_ids)))
  1528. for video_id in video_ids:
  1529. self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
  1530. class BlipTVUserIE(InfoExtractor):
  1531. """Information Extractor for blip.tv users."""
  1532. _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
  1533. _PAGE_SIZE = 12
  1534. IE_NAME = u'blip.tv:user'
  1535. def __init__(self, downloader=None):
  1536. InfoExtractor.__init__(self, downloader)
  1537. def report_download_page(self, username, pagenum):
  1538. """Report attempt to download user page."""
  1539. self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
  1540. (self.IE_NAME, username, pagenum))
  1541. def _real_extract(self, url):
  1542. # Extract username
  1543. mobj = re.match(self._VALID_URL, url)
  1544. if mobj is None:
  1545. self._downloader.trouble(u'ERROR: invalid url: %s' % url)
  1546. return
  1547. username = mobj.group(1)
  1548. page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
  1549. request = compat_urllib_request.Request(url)
  1550. try:
  1551. page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  1552. mobj = re.search(r'data-users-id="([^"]+)"', page)
  1553. page_base = page_base % mobj.group(1)
  1554. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1555. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  1556. return
  1557. # Download video ids using BlipTV Ajax calls. Result size per
  1558. # query is limited (currently to 12 videos) so we need to query
  1559. # page by page until there are no video ids - it means we got
  1560. # all of them.
  1561. video_ids = []
  1562. pagenum = 1
  1563. while True:
  1564. self.report_download_page(username, pagenum)
  1565. request = compat_urllib_request.Request( page_base + "&page=" + str(pagenum) )
  1566. try:
  1567. page = compat_urllib_request.urlopen(request).read().decode('utf-8')
  1568. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1569. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
  1570. return
  1571. # Extract video identifiers
  1572. ids_in_page = []
  1573. for mobj in re.finditer(r'href="/([^"]+)"', page):
  1574. if mobj.group(1) not in ids_in_page:
  1575. ids_in_page.append(unescapeHTML(mobj.group(1)))
  1576. video_ids.extend(ids_in_page)
  1577. # A little optimization - if current page is not
  1578. # "full", ie. does not contain PAGE_SIZE video ids then
  1579. # we can assume that this page is the last one - there
  1580. # are no more ids on further pages - no need to query
  1581. # again.
  1582. if len(ids_in_page) < self._PAGE_SIZE:
  1583. break
  1584. pagenum += 1
  1585. all_ids_count = len(video_ids)
  1586. playliststart = self._downloader.params.get('playliststart', 1) - 1
  1587. playlistend = self._downloader.params.get('playlistend', -1)
  1588. if playlistend == -1:
  1589. video_ids = video_ids[playliststart:]
  1590. else:
  1591. video_ids = video_ids[playliststart:playlistend]
  1592. self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
  1593. (self.IE_NAME, username, all_ids_count, len(video_ids)))
  1594. for video_id in video_ids:
  1595. self._downloader.download([u'http://blip.tv/'+video_id])
  1596. class DepositFilesIE(InfoExtractor):
  1597. """Information extractor for depositfiles.com"""
  1598. _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
  1599. def report_download_webpage(self, file_id):
  1600. """Report webpage download."""
  1601. self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
  1602. def report_extraction(self, file_id):
  1603. """Report information extraction."""
  1604. self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
  1605. def _real_extract(self, url):
  1606. file_id = url.split('/')[-1]
  1607. # Rebuild url in english locale
  1608. url = 'http://depositfiles.com/en/files/' + file_id
  1609. # Retrieve file webpage with 'Free download' button pressed
  1610. free_download_indication = { 'gateway_result' : '1' }
  1611. request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication))
  1612. try:
  1613. self.report_download_webpage(file_id)
  1614. webpage = compat_urllib_request.urlopen(request).read()
  1615. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1616. self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
  1617. return
  1618. # Search for the real file URL
  1619. mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
  1620. if (mobj is None) or (mobj.group(1) is None):
  1621. # Try to figure out reason of the error.
  1622. mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
  1623. if (mobj is not None) and (mobj.group(1) is not None):
  1624. restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
  1625. self._downloader.trouble(u'ERROR: %s' % restriction_message)
  1626. else:
  1627. self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
  1628. return
  1629. file_url = mobj.group(1)
  1630. file_extension = os.path.splitext(file_url)[1][1:]
  1631. # Search for file title
  1632. mobj = re.search(r'<b title="(.*?)">', webpage)
  1633. if mobj is None:
  1634. self._downloader.trouble(u'ERROR: unable to extract title')
  1635. return
  1636. file_title = mobj.group(1).decode('utf-8')
  1637. return [{
  1638. 'id': file_id.decode('utf-8'),
  1639. 'url': file_url.decode('utf-8'),
  1640. 'uploader': None,
  1641. 'upload_date': None,
  1642. 'title': file_title,
  1643. 'ext': file_extension.decode('utf-8'),
  1644. }]
  1645. class FacebookIE(InfoExtractor):
  1646. """Information Extractor for Facebook"""
  1647. _WORKING = False
  1648. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
  1649. _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
  1650. _NETRC_MACHINE = 'facebook'
  1651. _available_formats = ['video', 'highqual', 'lowqual']
  1652. _video_extensions = {
  1653. 'video': 'mp4',
  1654. 'highqual': 'mp4',
  1655. 'lowqual': 'mp4',
  1656. }
  1657. IE_NAME = u'facebook'
  1658. def __init__(self, downloader=None):
  1659. InfoExtractor.__init__(self, downloader)
  1660. def _reporter(self, message):
  1661. """Add header and report message."""
  1662. self._downloader.to_screen(u'[facebook] %s' % message)
  1663. def report_login(self):
  1664. """Report attempt to log in."""
  1665. self._reporter(u'Logging in')
  1666. def report_video_webpage_download(self, video_id):
  1667. """Report attempt to download video webpage."""
  1668. self._reporter(u'%s: Downloading video webpage' % video_id)
  1669. def report_information_extraction(self, video_id):
  1670. """Report attempt to extract video information."""
  1671. self._reporter(u'%s: Extracting video information' % video_id)
  1672. def _parse_page(self, video_webpage):
  1673. """Extract video information from page"""
  1674. # General data
  1675. data = {'title': r'\("video_title", "(.*?)"\)',
  1676. 'description': r'<div class="datawrap">(.*?)</div>',
  1677. 'owner': r'\("video_owner_name", "(.*?)"\)',
  1678. 'thumbnail': r'\("thumb_url", "(?P<THUMB>.*?)"\)',
  1679. }
  1680. video_info = {}
  1681. for piece in data.keys():
  1682. mobj = re.search(data[piece], video_webpage)
  1683. if mobj is not None:
  1684. video_info[piece] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape"))
  1685. # Video urls
  1686. video_urls = {}
  1687. for fmt in self._available_formats:
  1688. mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
  1689. if mobj is not None:
  1690. # URL is in a Javascript segment inside an escaped Unicode format within
  1691. # the generally utf-8 page
  1692. video_urls[fmt] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape"))
  1693. video_info['video_urls'] = video_urls
  1694. return video_info
  1695. def _real_initialize(self):
  1696. if self._downloader is None:
  1697. return
  1698. useremail = None
  1699. password = None
  1700. downloader_params = self._downloader.params
  1701. # Attempt to use provided username and password or .netrc data
  1702. if downloader_params.get('username', None) is not None:
  1703. useremail = downloader_params['username']
  1704. password = downloader_params['password']
  1705. elif downloader_params.get('usenetrc', False):
  1706. try:
  1707. info = netrc.netrc().authenticators(self._NETRC_MACHINE)
  1708. if info is not None:
  1709. useremail = info[0]
  1710. password = info[2]
  1711. else:
  1712. raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
  1713. except (IOError, netrc.NetrcParseError) as err:
  1714. self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
  1715. return
  1716. if useremail is None:
  1717. return
  1718. # Log in
  1719. login_form = {
  1720. 'email': useremail,
  1721. 'pass': password,
  1722. 'login': 'Log+In'
  1723. }
  1724. request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
  1725. try:
  1726. self.report_login()
  1727. login_results = compat_urllib_request.urlopen(request).read()
  1728. if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
  1729. self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
  1730. return
  1731. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1732. self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
  1733. return
  1734. def _real_extract(self, url):
  1735. mobj = re.match(self._VALID_URL, url)
  1736. if mobj is None:
  1737. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  1738. return
  1739. video_id = mobj.group('ID')
  1740. # Get video webpage
  1741. self.report_video_webpage_download(video_id)
  1742. request = compat_urllib_request.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
  1743. try:
  1744. page = compat_urllib_request.urlopen(request)
  1745. video_webpage = page.read()
  1746. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1747. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
  1748. return
  1749. # Start extracting information
  1750. self.report_information_extraction(video_id)
  1751. # Extract information
  1752. video_info = self._parse_page(video_webpage)
  1753. # uploader
  1754. if 'owner' not in video_info:
  1755. self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
  1756. return
  1757. video_uploader = video_info['owner']
  1758. # title
  1759. if 'title' not in video_info:
  1760. self._downloader.trouble(u'ERROR: unable to extract video title')
  1761. return
  1762. video_title = video_info['title']
  1763. video_title = video_title.decode('utf-8')
  1764. # thumbnail image
  1765. if 'thumbnail' not in video_info:
  1766. self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
  1767. video_thumbnail = ''
  1768. else:
  1769. video_thumbnail = video_info['thumbnail']
  1770. # upload date
  1771. upload_date = None
  1772. if 'upload_date' in video_info:
  1773. upload_time = video_info['upload_date']
  1774. timetuple = email.utils.parsedate_tz(upload_time)
  1775. if timetuple is not None:
  1776. try:
  1777. upload_date = time.strftime('%Y%m%d', timetuple[0:9])
  1778. except:
  1779. pass
  1780. # description
  1781. video_description = video_info.get('description', 'No description available.')
  1782. url_map = video_info['video_urls']
  1783. if url_map:
  1784. # Decide which formats to download
  1785. req_format = self._downloader.params.get('format', None)
  1786. format_limit = self._downloader.params.get('format_limit', None)
  1787. if format_limit is not None and format_limit in self._available_formats:
  1788. format_list = self._available_formats[self._available_formats.index(format_limit):]
  1789. else:
  1790. format_list = self._available_formats
  1791. existing_formats = [x for x in format_list if x in url_map]
  1792. if len(existing_formats) == 0:
  1793. self._downloader.trouble(u'ERROR: no known formats available for video')
  1794. return
  1795. if req_format is None:
  1796. video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
  1797. elif req_format == 'worst':
  1798. video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
  1799. elif req_format == '-1':
  1800. video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
  1801. else:
  1802. # Specific format
  1803. if req_format not in url_map:
  1804. self._downloader.trouble(u'ERROR: requested format not available')
  1805. return
  1806. video_url_list = [(req_format, url_map[req_format])] # Specific format
  1807. results = []
  1808. for format_param, video_real_url in video_url_list:
  1809. # Extension
  1810. video_extension = self._video_extensions.get(format_param, 'mp4')
  1811. results.append({
  1812. 'id': video_id.decode('utf-8'),
  1813. 'url': video_real_url.decode('utf-8'),
  1814. 'uploader': video_uploader.decode('utf-8'),
  1815. 'upload_date': upload_date,
  1816. 'title': video_title,
  1817. 'ext': video_extension.decode('utf-8'),
  1818. 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
  1819. 'thumbnail': video_thumbnail.decode('utf-8'),
  1820. 'description': video_description.decode('utf-8'),
  1821. })
  1822. return results
  1823. class BlipTVIE(InfoExtractor):
  1824. """Information extractor for blip.tv"""
  1825. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
  1826. _URL_EXT = r'^.*\.([a-z0-9]+)$'
  1827. IE_NAME = u'blip.tv'
  1828. def report_extraction(self, file_id):
  1829. """Report information extraction."""
  1830. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
  1831. def report_direct_download(self, title):
  1832. """Report information extraction."""
  1833. self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
  1834. def _real_extract(self, url):
  1835. mobj = re.match(self._VALID_URL, url)
  1836. if mobj is None:
  1837. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  1838. return
  1839. if '?' in url:
  1840. cchar = '&'
  1841. else:
  1842. cchar = '?'
  1843. json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
  1844. request = compat_urllib_request.Request(json_url)
  1845. self.report_extraction(mobj.group(1))
  1846. info = None
  1847. try:
  1848. urlh = compat_urllib_request.urlopen(request)
  1849. if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
  1850. basename = url.split('/')[-1]
  1851. title,ext = os.path.splitext(basename)
  1852. title = title.decode('UTF-8')
  1853. ext = ext.replace('.', '')
  1854. self.report_direct_download(title)
  1855. info = {
  1856. 'id': title,
  1857. 'url': url,
  1858. 'uploader': None,
  1859. 'upload_date': None,
  1860. 'title': title,
  1861. 'ext': ext,
  1862. 'urlhandle': urlh
  1863. }
  1864. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1865. self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
  1866. return
  1867. if info is None: # Regular URL
  1868. try:
  1869. json_code_bytes = urlh.read()
  1870. json_code = json_code_bytes.decode('utf-8')
  1871. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  1872. self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
  1873. return
  1874. try:
  1875. json_data = json.loads(json_code)
  1876. if 'Post' in json_data:
  1877. data = json_data['Post']
  1878. else:
  1879. data = json_data
  1880. upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
  1881. video_url = data['media']['url']
  1882. umobj = re.match(self._URL_EXT, video_url)
  1883. if umobj is None:
  1884. raise ValueError('Can not determine filename extension')
  1885. ext = umobj.group(1)
  1886. info = {
  1887. 'id': data['item_id'],
  1888. 'url': video_url,
  1889. 'uploader': data['display_name'],
  1890. 'upload_date': upload_date,
  1891. 'title': data['title'],
  1892. 'ext': ext,
  1893. 'format': data['media']['mimeType'],
  1894. 'thumbnail': data['thumbnailUrl'],
  1895. 'description': data['description'],
  1896. 'player_url': data['embedUrl']
  1897. }
  1898. except (ValueError,KeyError) as err:
  1899. self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
  1900. return
  1901. std_headers['User-Agent'] = 'iTunes/10.6.1'
  1902. return [info]
  1903. class MyVideoIE(InfoExtractor):
  1904. """Information Extractor for myvideo.de."""
  1905. _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
  1906. IE_NAME = u'myvideo'
  1907. def __init__(self, downloader=None):
  1908. InfoExtractor.__init__(self, downloader)
  1909. def report_extraction(self, video_id):
  1910. """Report information extraction."""
  1911. self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
  1912. def _real_extract(self,url):
  1913. mobj = re.match(self._VALID_URL, url)
  1914. if mobj is None:
  1915. self._download.trouble(u'ERROR: invalid URL: %s' % url)
  1916. return
  1917. video_id = mobj.group(1)
  1918. # Get video webpage
  1919. webpage_url = 'http://www.myvideo.de/watch/%s' % video_id
  1920. webpage = self._download_webpage(webpage_url, video_id)
  1921. self.report_extraction(video_id)
  1922. mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
  1923. webpage)
  1924. if mobj is None:
  1925. self._downloader.trouble(u'ERROR: unable to extract media URL')
  1926. return
  1927. video_url = mobj.group(1) + ('/%s.flv' % video_id)
  1928. mobj = re.search('<title>([^<]+)</title>', webpage)
  1929. if mobj is None:
  1930. self._downloader.trouble(u'ERROR: unable to extract title')
  1931. return
  1932. video_title = mobj.group(1)
  1933. return [{
  1934. 'id': video_id,
  1935. 'url': video_url,
  1936. 'uploader': None,
  1937. 'upload_date': None,
  1938. 'title': video_title,
  1939. 'ext': u'flv',
  1940. }]
  1941. class ComedyCentralIE(InfoExtractor):
  1942. """Information extractor for The Daily Show and Colbert Report """
  1943. # urls can be abbreviations like :thedailyshow or :colbert
  1944. # urls for episodes like:
  1945. # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
  1946. # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
  1947. # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
  1948. _VALID_URL = r"""^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport)
  1949. |(https?://)?(www\.)?
  1950. (?P<showname>thedailyshow|colbertnation)\.com/
  1951. (full-episodes/(?P<episode>.*)|
  1952. (?P<clip>
  1953. (the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
  1954. |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*)))))
  1955. $"""
  1956. IE_NAME = u'comedycentral'
  1957. _available_formats = ['3500', '2200', '1700', '1200', '750', '400']
  1958. _video_extensions = {
  1959. '3500': 'mp4',
  1960. '2200': 'mp4',
  1961. '1700': 'mp4',
  1962. '1200': 'mp4',
  1963. '750': 'mp4',
  1964. '400': 'mp4',
  1965. }
  1966. _video_dimensions = {
  1967. '3500': '1280x720',
  1968. '2200': '960x540',
  1969. '1700': '768x432',
  1970. '1200': '640x360',
  1971. '750': '512x288',
  1972. '400': '384x216',
  1973. }
  1974. def suitable(self, url):
  1975. """Receives a URL and returns True if suitable for this IE."""
  1976. return re.match(self._VALID_URL, url, re.VERBOSE) is not None
  1977. def report_extraction(self, episode_id):
  1978. self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
  1979. def report_config_download(self, episode_id):
  1980. self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
  1981. def report_index_download(self, episode_id):
  1982. self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
  1983. def report_player_url(self, episode_id):
  1984. self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
  1985. def _print_formats(self, formats):
  1986. print('Available formats:')
  1987. for x in formats:
  1988. print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
  1989. def _real_extract(self, url):
  1990. mobj = re.match(self._VALID_URL, url, re.VERBOSE)
  1991. if mobj is None:
  1992. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  1993. return
  1994. if mobj.group('shortname'):
  1995. if mobj.group('shortname') in ('tds', 'thedailyshow'):
  1996. url = u'http://www.thedailyshow.com/full-episodes/'
  1997. else:
  1998. url = u'http://www.colbertnation.com/full-episodes/'
  1999. mobj = re.match(self._VALID_URL, url, re.VERBOSE)
  2000. assert mobj is not None
  2001. if mobj.group('clip'):
  2002. if mobj.group('showname') == 'thedailyshow':
  2003. epTitle = mobj.group('tdstitle')
  2004. else:
  2005. epTitle = mobj.group('cntitle')
  2006. dlNewest = False
  2007. else:
  2008. dlNewest = not mobj.group('episode')
  2009. if dlNewest:
  2010. epTitle = mobj.group('showname')
  2011. else:
  2012. epTitle = mobj.group('episode')
  2013. req = compat_urllib_request.Request(url)
  2014. self.report_extraction(epTitle)
  2015. try:
  2016. htmlHandle = compat_urllib_request.urlopen(req)
  2017. html = htmlHandle.read()
  2018. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2019. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  2020. return
  2021. if dlNewest:
  2022. url = htmlHandle.geturl()
  2023. mobj = re.match(self._VALID_URL, url, re.VERBOSE)
  2024. if mobj is None:
  2025. self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
  2026. return
  2027. if mobj.group('episode') == '':
  2028. self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
  2029. return
  2030. epTitle = mobj.group('episode')
  2031. mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', html)
  2032. if len(mMovieParams) == 0:
  2033. # The Colbert Report embeds the information in a without
  2034. # a URL prefix; so extract the alternate reference
  2035. # and then add the URL prefix manually.
  2036. altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video).*?:.*?)"', html)
  2037. if len(altMovieParams) == 0:
  2038. self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
  2039. return
  2040. else:
  2041. mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
  2042. playerUrl_raw = mMovieParams[0][0]
  2043. self.report_player_url(epTitle)
  2044. try:
  2045. urlHandle = compat_urllib_request.urlopen(playerUrl_raw)
  2046. playerUrl = urlHandle.geturl()
  2047. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2048. self._downloader.trouble(u'ERROR: unable to find out player URL: ' + compat_str(err))
  2049. return
  2050. uri = mMovieParams[0][1]
  2051. indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri})
  2052. self.report_index_download(epTitle)
  2053. try:
  2054. indexXml = compat_urllib_request.urlopen(indexUrl).read()
  2055. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2056. self._downloader.trouble(u'ERROR: unable to download episode index: ' + compat_str(err))
  2057. return
  2058. results = []
  2059. idoc = xml.etree.ElementTree.fromstring(indexXml)
  2060. itemEls = idoc.findall('.//item')
  2061. for itemEl in itemEls:
  2062. mediaId = itemEl.findall('./guid')[0].text
  2063. shortMediaId = mediaId.split(':')[-1]
  2064. showId = mediaId.split(':')[-2].replace('.com', '')
  2065. officialTitle = itemEl.findall('./title')[0].text
  2066. officialDate = itemEl.findall('./pubDate')[0].text
  2067. configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
  2068. compat_urllib_parse.urlencode({'uri': mediaId}))
  2069. configReq = compat_urllib_request.Request(configUrl)
  2070. self.report_config_download(epTitle)
  2071. try:
  2072. configXml = compat_urllib_request.urlopen(configReq).read()
  2073. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2074. self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
  2075. return
  2076. cdoc = xml.etree.ElementTree.fromstring(configXml)
  2077. turls = []
  2078. for rendition in cdoc.findall('.//rendition'):
  2079. finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
  2080. turls.append(finfo)
  2081. if len(turls) == 0:
  2082. self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
  2083. continue
  2084. if self._downloader.params.get('listformats', None):
  2085. self._print_formats([i[0] for i in turls])
  2086. return
  2087. # For now, just pick the highest bitrate
  2088. format,video_url = turls[-1]
  2089. # Get the format arg from the arg stream
  2090. req_format = self._downloader.params.get('format', None)
  2091. # Select format if we can find one
  2092. for f,v in turls:
  2093. if f == req_format:
  2094. format, video_url = f, v
  2095. break
  2096. # Patch to download from alternative CDN, which does not
  2097. # break on current RTMPDump builds
  2098. broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/"
  2099. better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/"
  2100. if video_url.startswith(broken_cdn):
  2101. video_url = video_url.replace(broken_cdn, better_cdn)
  2102. effTitle = showId + u'-' + epTitle
  2103. info = {
  2104. 'id': shortMediaId,
  2105. 'url': video_url,
  2106. 'uploader': showId,
  2107. 'upload_date': officialDate,
  2108. 'title': effTitle,
  2109. 'ext': 'mp4',
  2110. 'format': format,
  2111. 'thumbnail': None,
  2112. 'description': officialTitle,
  2113. 'player_url': None #playerUrl
  2114. }
  2115. results.append(info)
  2116. return results
  2117. class EscapistIE(InfoExtractor):
  2118. """Information extractor for The Escapist """
  2119. _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
  2120. IE_NAME = u'escapist'
  2121. def report_extraction(self, showName):
  2122. self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
  2123. def report_config_download(self, showName):
  2124. self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
  2125. def _real_extract(self, url):
  2126. mobj = re.match(self._VALID_URL, url)
  2127. if mobj is None:
  2128. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2129. return
  2130. showName = mobj.group('showname')
  2131. videoId = mobj.group('episode')
  2132. self.report_extraction(showName)
  2133. try:
  2134. webPage = compat_urllib_request.urlopen(url)
  2135. webPageBytes = webPage.read()
  2136. m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
  2137. webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
  2138. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2139. self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err))
  2140. return
  2141. descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
  2142. description = unescapeHTML(descMatch.group(1))
  2143. imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
  2144. imgUrl = unescapeHTML(imgMatch.group(1))
  2145. playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
  2146. playerUrl = unescapeHTML(playerUrlMatch.group(1))
  2147. configUrlMatch = re.search('config=(.*)$', playerUrl)
  2148. configUrl = compat_urllib_parse.unquote(configUrlMatch.group(1))
  2149. self.report_config_download(showName)
  2150. try:
  2151. configJSON = compat_urllib_request.urlopen(configUrl)
  2152. m = re.match(r'text/html; charset="?([^"]+)"?', configJSON.headers['Content-Type'])
  2153. configJSON = configJSON.read().decode(m.group(1) if m else 'utf-8')
  2154. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2155. self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err))
  2156. return
  2157. # Technically, it's JavaScript, not JSON
  2158. configJSON = configJSON.replace("'", '"')
  2159. try:
  2160. config = json.loads(configJSON)
  2161. except (ValueError,) as err:
  2162. self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + compat_str(err))
  2163. return
  2164. playlist = config['playlist']
  2165. videoUrl = playlist[1]['url']
  2166. info = {
  2167. 'id': videoId,
  2168. 'url': videoUrl,
  2169. 'uploader': showName,
  2170. 'upload_date': None,
  2171. 'title': showName,
  2172. 'ext': 'flv',
  2173. 'thumbnail': imgUrl,
  2174. 'description': description,
  2175. 'player_url': playerUrl,
  2176. }
  2177. return [info]
  2178. class CollegeHumorIE(InfoExtractor):
  2179. """Information extractor for collegehumor.com"""
  2180. _WORKING = False
  2181. _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
  2182. IE_NAME = u'collegehumor'
  2183. def report_manifest(self, video_id):
  2184. """Report information extraction."""
  2185. self._downloader.to_screen(u'[%s] %s: Downloading XML manifest' % (self.IE_NAME, video_id))
  2186. def report_extraction(self, video_id):
  2187. """Report information extraction."""
  2188. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2189. def _real_extract(self, url):
  2190. mobj = re.match(self._VALID_URL, url)
  2191. if mobj is None:
  2192. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2193. return
  2194. video_id = mobj.group('videoid')
  2195. info = {
  2196. 'id': video_id,
  2197. 'uploader': None,
  2198. 'upload_date': None,
  2199. }
  2200. self.report_extraction(video_id)
  2201. xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id
  2202. try:
  2203. metaXml = compat_urllib_request.urlopen(xmlUrl).read()
  2204. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2205. self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
  2206. return
  2207. mdoc = xml.etree.ElementTree.fromstring(metaXml)
  2208. try:
  2209. videoNode = mdoc.findall('./video')[0]
  2210. info['description'] = videoNode.findall('./description')[0].text
  2211. info['title'] = videoNode.findall('./caption')[0].text
  2212. info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
  2213. manifest_url = videoNode.findall('./file')[0].text
  2214. except IndexError:
  2215. self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
  2216. return
  2217. manifest_url += '?hdcore=2.10.3'
  2218. self.report_manifest(video_id)
  2219. try:
  2220. manifestXml = compat_urllib_request.urlopen(manifest_url).read()
  2221. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2222. self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
  2223. return
  2224. adoc = xml.etree.ElementTree.fromstring(manifestXml)
  2225. try:
  2226. media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0]
  2227. node_id = media_node.attrib['url']
  2228. video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
  2229. except IndexError as err:
  2230. self._downloader.trouble(u'\nERROR: Invalid manifest file')
  2231. return
  2232. url_pr = compat_urllib_parse_urlparse(manifest_url)
  2233. url = url_pr.scheme + '://' + url_pr.netloc + '/z' + video_id[:-2] + '/' + node_id + 'Seg1-Frag1'
  2234. info['url'] = url
  2235. info['ext'] = 'f4f'
  2236. return [info]
  2237. class XVideosIE(InfoExtractor):
  2238. """Information extractor for xvideos.com"""
  2239. _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
  2240. IE_NAME = u'xvideos'
  2241. def report_extraction(self, video_id):
  2242. """Report information extraction."""
  2243. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2244. def _real_extract(self, url):
  2245. mobj = re.match(self._VALID_URL, url)
  2246. if mobj is None:
  2247. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2248. return
  2249. video_id = mobj.group(1)
  2250. webpage = self._download_webpage(url, video_id)
  2251. self.report_extraction(video_id)
  2252. # Extract video URL
  2253. mobj = re.search(r'flv_url=(.+?)&', webpage)
  2254. if mobj is None:
  2255. self._downloader.trouble(u'ERROR: unable to extract video url')
  2256. return
  2257. video_url = compat_urllib_parse.unquote(mobj.group(1))
  2258. # Extract title
  2259. mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
  2260. if mobj is None:
  2261. self._downloader.trouble(u'ERROR: unable to extract video title')
  2262. return
  2263. video_title = mobj.group(1)
  2264. # Extract video thumbnail
  2265. mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
  2266. if mobj is None:
  2267. self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
  2268. return
  2269. video_thumbnail = mobj.group(0)
  2270. info = {
  2271. 'id': video_id,
  2272. 'url': video_url,
  2273. 'uploader': None,
  2274. 'upload_date': None,
  2275. 'title': video_title,
  2276. 'ext': 'flv',
  2277. 'thumbnail': video_thumbnail,
  2278. 'description': None,
  2279. }
  2280. return [info]
  2281. class SoundcloudIE(InfoExtractor):
  2282. """Information extractor for soundcloud.com
  2283. To access the media, the uid of the song and a stream token
  2284. must be extracted from the page source and the script must make
  2285. a request to media.soundcloud.com/crossdomain.xml. Then
  2286. the media can be grabbed by requesting from an url composed
  2287. of the stream token and uid
  2288. """
  2289. _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
  2290. IE_NAME = u'soundcloud'
  2291. def __init__(self, downloader=None):
  2292. InfoExtractor.__init__(self, downloader)
  2293. def report_resolve(self, video_id):
  2294. """Report information extraction."""
  2295. self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
  2296. def report_extraction(self, video_id):
  2297. """Report information extraction."""
  2298. self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
  2299. def _real_extract(self, url):
  2300. mobj = re.match(self._VALID_URL, url)
  2301. if mobj is None:
  2302. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2303. return
  2304. # extract uploader (which is in the url)
  2305. uploader = mobj.group(1)
  2306. # extract simple title (uploader + slug of song title)
  2307. slug_title = mobj.group(2)
  2308. simple_title = uploader + u'-' + slug_title
  2309. self.report_resolve('%s/%s' % (uploader, slug_title))
  2310. url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
  2311. resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
  2312. request = compat_urllib_request.Request(resolv_url)
  2313. try:
  2314. info_json_bytes = compat_urllib_request.urlopen(request).read()
  2315. info_json = info_json_bytes.decode('utf-8')
  2316. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2317. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
  2318. return
  2319. info = json.loads(info_json)
  2320. video_id = info['id']
  2321. self.report_extraction('%s/%s' % (uploader, slug_title))
  2322. streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
  2323. request = compat_urllib_request.Request(streams_url)
  2324. try:
  2325. stream_json_bytes = compat_urllib_request.urlopen(request).read()
  2326. stream_json = stream_json_bytes.decode('utf-8')
  2327. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2328. self._downloader.trouble(u'ERROR: unable to download stream definitions: %s' % compat_str(err))
  2329. return
  2330. streams = json.loads(stream_json)
  2331. mediaURL = streams['http_mp3_128_url']
  2332. return [{
  2333. 'id': info['id'],
  2334. 'url': mediaURL,
  2335. 'uploader': info['user']['username'],
  2336. 'upload_date': info['created_at'],
  2337. 'title': info['title'],
  2338. 'ext': u'mp3',
  2339. 'description': info['description'],
  2340. }]
  2341. class InfoQIE(InfoExtractor):
  2342. """Information extractor for infoq.com"""
  2343. _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
  2344. def report_extraction(self, video_id):
  2345. """Report information extraction."""
  2346. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2347. def _real_extract(self, url):
  2348. mobj = re.match(self._VALID_URL, url)
  2349. if mobj is None:
  2350. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2351. return
  2352. webpage = self._download_webpage(url, video_id=url)
  2353. self.report_extraction(url)
  2354. # Extract video URL
  2355. mobj = re.search(r"jsclassref='([^']*)'", webpage)
  2356. if mobj is None:
  2357. self._downloader.trouble(u'ERROR: unable to extract video url')
  2358. return
  2359. real_id = compat_urllib_parse.unquote(base64.b64decode(mobj.group(1).encode('ascii')).decode('utf-8'))
  2360. video_url = 'rtmpe://video.infoq.com/cfx/st/' + real_id
  2361. # Extract title
  2362. mobj = re.search(r'contentTitle = "(.*?)";', webpage)
  2363. if mobj is None:
  2364. self._downloader.trouble(u'ERROR: unable to extract video title')
  2365. return
  2366. video_title = mobj.group(1)
  2367. # Extract description
  2368. video_description = u'No description available.'
  2369. mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
  2370. if mobj is not None:
  2371. video_description = mobj.group(1)
  2372. video_filename = video_url.split('/')[-1]
  2373. video_id, extension = video_filename.split('.')
  2374. info = {
  2375. 'id': video_id,
  2376. 'url': video_url,
  2377. 'uploader': None,
  2378. 'upload_date': None,
  2379. 'title': video_title,
  2380. 'ext': extension, # Extension is always(?) mp4, but seems to be flv
  2381. 'thumbnail': None,
  2382. 'description': video_description,
  2383. }
  2384. return [info]
  2385. class MixcloudIE(InfoExtractor):
  2386. """Information extractor for www.mixcloud.com"""
  2387. _WORKING = False # New API, but it seems good http://www.mixcloud.com/developers/documentation/
  2388. _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
  2389. IE_NAME = u'mixcloud'
  2390. def __init__(self, downloader=None):
  2391. InfoExtractor.__init__(self, downloader)
  2392. def report_download_json(self, file_id):
  2393. """Report JSON download."""
  2394. self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
  2395. def report_extraction(self, file_id):
  2396. """Report information extraction."""
  2397. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
  2398. def get_urls(self, jsonData, fmt, bitrate='best'):
  2399. """Get urls from 'audio_formats' section in json"""
  2400. file_url = None
  2401. try:
  2402. bitrate_list = jsonData[fmt]
  2403. if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
  2404. bitrate = max(bitrate_list) # select highest
  2405. url_list = jsonData[fmt][bitrate]
  2406. except TypeError: # we have no bitrate info.
  2407. url_list = jsonData[fmt]
  2408. return url_list
  2409. def check_urls(self, url_list):
  2410. """Returns 1st active url from list"""
  2411. for url in url_list:
  2412. try:
  2413. compat_urllib_request.urlopen(url)
  2414. return url
  2415. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2416. url = None
  2417. return None
  2418. def _print_formats(self, formats):
  2419. print('Available formats:')
  2420. for fmt in formats.keys():
  2421. for b in formats[fmt]:
  2422. try:
  2423. ext = formats[fmt][b][0]
  2424. print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
  2425. except TypeError: # we have no bitrate info
  2426. ext = formats[fmt][0]
  2427. print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
  2428. break
  2429. def _real_extract(self, url):
  2430. mobj = re.match(self._VALID_URL, url)
  2431. if mobj is None:
  2432. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2433. return
  2434. # extract uploader & filename from url
  2435. uploader = mobj.group(1).decode('utf-8')
  2436. file_id = uploader + "-" + mobj.group(2).decode('utf-8')
  2437. # construct API request
  2438. file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
  2439. # retrieve .json file with links to files
  2440. request = compat_urllib_request.Request(file_url)
  2441. try:
  2442. self.report_download_json(file_url)
  2443. jsonData = compat_urllib_request.urlopen(request).read()
  2444. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2445. self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
  2446. return
  2447. # parse JSON
  2448. json_data = json.loads(jsonData)
  2449. player_url = json_data['player_swf_url']
  2450. formats = dict(json_data['audio_formats'])
  2451. req_format = self._downloader.params.get('format', None)
  2452. bitrate = None
  2453. if self._downloader.params.get('listformats', None):
  2454. self._print_formats(formats)
  2455. return
  2456. if req_format is None or req_format == 'best':
  2457. for format_param in formats.keys():
  2458. url_list = self.get_urls(formats, format_param)
  2459. # check urls
  2460. file_url = self.check_urls(url_list)
  2461. if file_url is not None:
  2462. break # got it!
  2463. else:
  2464. if req_format not in formats:
  2465. self._downloader.trouble(u'ERROR: format is not available')
  2466. return
  2467. url_list = self.get_urls(formats, req_format)
  2468. file_url = self.check_urls(url_list)
  2469. format_param = req_format
  2470. return [{
  2471. 'id': file_id.decode('utf-8'),
  2472. 'url': file_url.decode('utf-8'),
  2473. 'uploader': uploader.decode('utf-8'),
  2474. 'upload_date': None,
  2475. 'title': json_data['name'],
  2476. 'ext': file_url.split('.')[-1].decode('utf-8'),
  2477. 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
  2478. 'thumbnail': json_data['thumbnail_url'],
  2479. 'description': json_data['description'],
  2480. 'player_url': player_url.decode('utf-8'),
  2481. }]
  2482. class StanfordOpenClassroomIE(InfoExtractor):
  2483. """Information extractor for Stanford's Open ClassRoom"""
  2484. _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
  2485. IE_NAME = u'stanfordoc'
  2486. def report_download_webpage(self, objid):
  2487. """Report information extraction."""
  2488. self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
  2489. def report_extraction(self, video_id):
  2490. """Report information extraction."""
  2491. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2492. def _real_extract(self, url):
  2493. mobj = re.match(self._VALID_URL, url)
  2494. if mobj is None:
  2495. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2496. return
  2497. if mobj.group('course') and mobj.group('video'): # A specific video
  2498. course = mobj.group('course')
  2499. video = mobj.group('video')
  2500. info = {
  2501. 'id': course + '_' + video,
  2502. 'uploader': None,
  2503. 'upload_date': None,
  2504. }
  2505. self.report_extraction(info['id'])
  2506. baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
  2507. xmlUrl = baseUrl + video + '.xml'
  2508. try:
  2509. metaXml = compat_urllib_request.urlopen(xmlUrl).read()
  2510. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2511. self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
  2512. return
  2513. mdoc = xml.etree.ElementTree.fromstring(metaXml)
  2514. try:
  2515. info['title'] = mdoc.findall('./title')[0].text
  2516. info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
  2517. except IndexError:
  2518. self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
  2519. return
  2520. info['ext'] = info['url'].rpartition('.')[2]
  2521. return [info]
  2522. elif mobj.group('course'): # A course page
  2523. course = mobj.group('course')
  2524. info = {
  2525. 'id': course,
  2526. 'type': 'playlist',
  2527. 'uploader': None,
  2528. 'upload_date': None,
  2529. }
  2530. self.report_download_webpage(info['id'])
  2531. try:
  2532. coursepage = compat_urllib_request.urlopen(url).read()
  2533. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2534. self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
  2535. return
  2536. m = re.search('<h1>([^<]+)</h1>', coursepage)
  2537. if m:
  2538. info['title'] = unescapeHTML(m.group(1))
  2539. else:
  2540. info['title'] = info['id']
  2541. m = re.search('<description>([^<]+)</description>', coursepage)
  2542. if m:
  2543. info['description'] = unescapeHTML(m.group(1))
  2544. links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
  2545. info['list'] = [
  2546. {
  2547. 'type': 'reference',
  2548. 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
  2549. }
  2550. for vpage in links]
  2551. results = []
  2552. for entry in info['list']:
  2553. assert entry['type'] == 'reference'
  2554. results += self.extract(entry['url'])
  2555. return results
  2556. else: # Root page
  2557. info = {
  2558. 'id': 'Stanford OpenClassroom',
  2559. 'type': 'playlist',
  2560. 'uploader': None,
  2561. 'upload_date': None,
  2562. }
  2563. self.report_download_webpage(info['id'])
  2564. rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
  2565. try:
  2566. rootpage = compat_urllib_request.urlopen(rootURL).read()
  2567. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2568. self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
  2569. return
  2570. info['title'] = info['id']
  2571. links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
  2572. info['list'] = [
  2573. {
  2574. 'type': 'reference',
  2575. 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
  2576. }
  2577. for cpage in links]
  2578. results = []
  2579. for entry in info['list']:
  2580. assert entry['type'] == 'reference'
  2581. results += self.extract(entry['url'])
  2582. return results
  2583. class MTVIE(InfoExtractor):
  2584. """Information extractor for MTV.com"""
  2585. _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
  2586. IE_NAME = u'mtv'
  2587. def report_extraction(self, video_id):
  2588. """Report information extraction."""
  2589. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2590. def _real_extract(self, url):
  2591. mobj = re.match(self._VALID_URL, url)
  2592. if mobj is None:
  2593. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2594. return
  2595. if not mobj.group('proto'):
  2596. url = 'http://' + url
  2597. video_id = mobj.group('videoid')
  2598. webpage = self._download_webpage(url, video_id)
  2599. mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
  2600. if mobj is None:
  2601. self._downloader.trouble(u'ERROR: unable to extract song name')
  2602. return
  2603. song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
  2604. mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
  2605. if mobj is None:
  2606. self._downloader.trouble(u'ERROR: unable to extract performer')
  2607. return
  2608. performer = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
  2609. video_title = performer + ' - ' + song_name
  2610. mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
  2611. if mobj is None:
  2612. self._downloader.trouble(u'ERROR: unable to mtvn_uri')
  2613. return
  2614. mtvn_uri = mobj.group(1)
  2615. mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
  2616. if mobj is None:
  2617. self._downloader.trouble(u'ERROR: unable to extract content id')
  2618. return
  2619. content_id = mobj.group(1)
  2620. videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
  2621. self.report_extraction(video_id)
  2622. request = compat_urllib_request.Request(videogen_url)
  2623. try:
  2624. metadataXml = compat_urllib_request.urlopen(request).read()
  2625. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2626. self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
  2627. return
  2628. mdoc = xml.etree.ElementTree.fromstring(metadataXml)
  2629. renditions = mdoc.findall('.//rendition')
  2630. # For now, always pick the highest quality.
  2631. rendition = renditions[-1]
  2632. try:
  2633. _,_,ext = rendition.attrib['type'].partition('/')
  2634. format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
  2635. video_url = rendition.find('./src').text
  2636. except KeyError:
  2637. self._downloader.trouble('Invalid rendition field.')
  2638. return
  2639. info = {
  2640. 'id': video_id,
  2641. 'url': video_url,
  2642. 'uploader': performer,
  2643. 'upload_date': None,
  2644. 'title': video_title,
  2645. 'ext': ext,
  2646. 'format': format,
  2647. }
  2648. return [info]
  2649. class YoukuIE(InfoExtractor):
  2650. _VALID_URL = r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
  2651. def report_download_webpage(self, file_id):
  2652. """Report webpage download."""
  2653. self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, file_id))
  2654. def report_extraction(self, file_id):
  2655. """Report information extraction."""
  2656. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
  2657. def _gen_sid(self):
  2658. nowTime = int(time.time() * 1000)
  2659. random1 = random.randint(1000,1998)
  2660. random2 = random.randint(1000,9999)
  2661. return "%d%d%d" %(nowTime,random1,random2)
  2662. def _get_file_ID_mix_string(self, seed):
  2663. mixed = []
  2664. source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
  2665. seed = float(seed)
  2666. for i in range(len(source)):
  2667. seed = (seed * 211 + 30031 ) % 65536
  2668. index = math.floor(seed / 65536 * len(source) )
  2669. mixed.append(source[int(index)])
  2670. source.remove(source[int(index)])
  2671. #return ''.join(mixed)
  2672. return mixed
  2673. def _get_file_id(self, fileId, seed):
  2674. mixed = self._get_file_ID_mix_string(seed)
  2675. ids = fileId.split('*')
  2676. realId = []
  2677. for ch in ids:
  2678. if ch:
  2679. realId.append(mixed[int(ch)])
  2680. return ''.join(realId)
  2681. def _real_extract(self, url):
  2682. mobj = re.match(self._VALID_URL, url)
  2683. if mobj is None:
  2684. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2685. return
  2686. video_id = mobj.group('ID')
  2687. info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
  2688. request = compat_urllib_request.Request(info_url, None, std_headers)
  2689. try:
  2690. self.report_download_webpage(video_id)
  2691. jsondata = compat_urllib_request.urlopen(request).read()
  2692. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2693. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  2694. return
  2695. self.report_extraction(video_id)
  2696. try:
  2697. jsonstr = jsondata.decode('utf-8')
  2698. config = json.loads(jsonstr)
  2699. video_title = config['data'][0]['title']
  2700. seed = config['data'][0]['seed']
  2701. format = self._downloader.params.get('format', None)
  2702. supported_format = list(config['data'][0]['streamfileids'].keys())
  2703. if format is None or format == 'best':
  2704. if 'hd2' in supported_format:
  2705. format = 'hd2'
  2706. else:
  2707. format = 'flv'
  2708. ext = u'flv'
  2709. elif format == 'worst':
  2710. format = 'mp4'
  2711. ext = u'mp4'
  2712. else:
  2713. format = 'flv'
  2714. ext = u'flv'
  2715. fileid = config['data'][0]['streamfileids'][format]
  2716. keys = [s['k'] for s in config['data'][0]['segs'][format]]
  2717. except (UnicodeDecodeError, ValueError, KeyError):
  2718. self._downloader.trouble(u'ERROR: unable to extract info section')
  2719. return
  2720. files_info=[]
  2721. sid = self._gen_sid()
  2722. fileid = self._get_file_id(fileid, seed)
  2723. #column 8,9 of fileid represent the segment number
  2724. #fileid[7:9] should be changed
  2725. for index, key in enumerate(keys):
  2726. temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
  2727. download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
  2728. info = {
  2729. 'id': '%s_part%02d' % (video_id, index),
  2730. 'url': download_url,
  2731. 'uploader': None,
  2732. 'upload_date': None,
  2733. 'title': video_title,
  2734. 'ext': ext,
  2735. }
  2736. files_info.append(info)
  2737. return files_info
  2738. class XNXXIE(InfoExtractor):
  2739. """Information extractor for xnxx.com"""
  2740. _VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)'
  2741. IE_NAME = u'xnxx'
  2742. VIDEO_URL_RE = r'flv_url=(.*?)&amp;'
  2743. VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
  2744. VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&amp;'
  2745. def report_webpage(self, video_id):
  2746. """Report information extraction"""
  2747. self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
  2748. def report_extraction(self, video_id):
  2749. """Report information extraction"""
  2750. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
  2751. def _real_extract(self, url):
  2752. mobj = re.match(self._VALID_URL, url)
  2753. if mobj is None:
  2754. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2755. return
  2756. video_id = mobj.group(1)
  2757. self.report_webpage(video_id)
  2758. # Get webpage content
  2759. try:
  2760. webpage_bytes = compat_urllib_request.urlopen(url).read()
  2761. webpage = webpage_bytes.decode('utf-8')
  2762. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2763. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
  2764. return
  2765. result = re.search(self.VIDEO_URL_RE, webpage)
  2766. if result is None:
  2767. self._downloader.trouble(u'ERROR: unable to extract video url')
  2768. return
  2769. video_url = compat_urllib_parse.unquote(result.group(1))
  2770. result = re.search(self.VIDEO_TITLE_RE, webpage)
  2771. if result is None:
  2772. self._downloader.trouble(u'ERROR: unable to extract video title')
  2773. return
  2774. video_title = result.group(1)
  2775. result = re.search(self.VIDEO_THUMB_RE, webpage)
  2776. if result is None:
  2777. self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
  2778. return
  2779. video_thumbnail = result.group(1)
  2780. return [{
  2781. 'id': video_id,
  2782. 'url': video_url,
  2783. 'uploader': None,
  2784. 'upload_date': None,
  2785. 'title': video_title,
  2786. 'ext': 'flv',
  2787. 'thumbnail': video_thumbnail,
  2788. 'description': None,
  2789. }]
  2790. class GooglePlusIE(InfoExtractor):
  2791. """Information extractor for plus.google.com."""
  2792. _VALID_URL = r'(?:https://)?plus\.google\.com/(?:[^/]+/)*?posts/(\w+)'
  2793. IE_NAME = u'plus.google'
  2794. def __init__(self, downloader=None):
  2795. InfoExtractor.__init__(self, downloader)
  2796. def report_extract_entry(self, url):
  2797. """Report downloading extry"""
  2798. self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url)
  2799. def report_date(self, upload_date):
  2800. """Report downloading extry"""
  2801. self._downloader.to_screen(u'[plus.google] Entry date: %s' % upload_date)
  2802. def report_uploader(self, uploader):
  2803. """Report downloading extry"""
  2804. self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader)
  2805. def report_title(self, video_title):
  2806. """Report downloading extry"""
  2807. self._downloader.to_screen(u'[plus.google] Title: %s' % video_title)
  2808. def report_extract_vid_page(self, video_page):
  2809. """Report information extraction."""
  2810. self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page)
  2811. def _real_extract(self, url):
  2812. # Extract id from URL
  2813. mobj = re.match(self._VALID_URL, url)
  2814. if mobj is None:
  2815. self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
  2816. return
  2817. post_url = mobj.group(0)
  2818. video_id = mobj.group(1)
  2819. video_extension = 'flv'
  2820. # Step 1, Retrieve post webpage to extract further information
  2821. self.report_extract_entry(post_url)
  2822. request = compat_urllib_request.Request(post_url)
  2823. try:
  2824. webpage = compat_urllib_request.urlopen(request).read().decode('utf-8')
  2825. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2826. self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
  2827. return
  2828. # Extract update date
  2829. upload_date = None
  2830. pattern = 'title="Timestamp">(.*?)</a>'
  2831. mobj = re.search(pattern, webpage)
  2832. if mobj:
  2833. upload_date = mobj.group(1)
  2834. # Convert timestring to a format suitable for filename
  2835. upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
  2836. upload_date = upload_date.strftime('%Y%m%d')
  2837. self.report_date(upload_date)
  2838. # Extract uploader
  2839. uploader = None
  2840. pattern = r'rel\="author".*?>(.*?)</a>'
  2841. mobj = re.search(pattern, webpage)
  2842. if mobj:
  2843. uploader = mobj.group(1)
  2844. self.report_uploader(uploader)
  2845. # Extract title
  2846. # Get the first line for title
  2847. video_title = u'NA'
  2848. pattern = r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]'
  2849. mobj = re.search(pattern, webpage)
  2850. if mobj:
  2851. video_title = mobj.group(1)
  2852. self.report_title(video_title)
  2853. # Step 2, Stimulate clicking the image box to launch video
  2854. pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]'
  2855. mobj = re.search(pattern, webpage)
  2856. if mobj is None:
  2857. self._downloader.trouble(u'ERROR: unable to extract video page URL')
  2858. video_page = mobj.group(1)
  2859. request = compat_urllib_request.Request(video_page)
  2860. try:
  2861. webpage = compat_urllib_request.urlopen(request).read().decode('utf-8')
  2862. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2863. self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
  2864. return
  2865. self.report_extract_vid_page(video_page)
  2866. # Extract video links on video page
  2867. """Extract video links of all sizes"""
  2868. pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
  2869. mobj = re.findall(pattern, webpage)
  2870. if len(mobj) == 0:
  2871. self._downloader.trouble(u'ERROR: unable to extract video links')
  2872. # Sort in resolution
  2873. links = sorted(mobj)
  2874. # Choose the lowest of the sort, i.e. highest resolution
  2875. video_url = links[-1]
  2876. # Only get the url. The resolution part in the tuple has no use anymore
  2877. video_url = video_url[-1]
  2878. # Treat escaped \u0026 style hex
  2879. try:
  2880. video_url = video_url.decode("unicode_escape")
  2881. except AttributeError: # Python 3
  2882. video_url = bytes(video_url, 'ascii').decode('unicode-escape')
  2883. return [{
  2884. 'id': video_id,
  2885. 'url': video_url,
  2886. 'uploader': uploader,
  2887. 'upload_date': upload_date,
  2888. 'title': video_title,
  2889. 'ext': video_extension,
  2890. }]
  2891. class NBAIE(InfoExtractor):
  2892. _VALID_URL = r'^(?:https?://)?(?:watch\.|www\.)?nba\.com/(?:nba/)?video(/[^?]*)(\?.*)?$'
  2893. IE_NAME = u'nba'
  2894. def _real_extract(self, url):
  2895. mobj = re.match(self._VALID_URL, url)
  2896. if mobj is None:
  2897. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2898. return
  2899. video_id = mobj.group(1)
  2900. if video_id.endswith('/index.html'):
  2901. video_id = video_id[:-len('/index.html')]
  2902. webpage = self._download_webpage(url, video_id)
  2903. video_url = u'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
  2904. def _findProp(rexp, default=None):
  2905. m = re.search(rexp, webpage)
  2906. if m:
  2907. return unescapeHTML(m.group(1))
  2908. else:
  2909. return default
  2910. shortened_video_id = video_id.rpartition('/')[2]
  2911. title = _findProp(r'<meta property="og:title" content="(.*?)"', shortened_video_id).replace('NBA.com: ', '')
  2912. info = {
  2913. 'id': shortened_video_id,
  2914. 'url': video_url,
  2915. 'ext': 'mp4',
  2916. 'title': title,
  2917. 'uploader_date': _findProp(r'<b>Date:</b> (.*?)</div>'),
  2918. 'description': _findProp(r'<div class="description">(.*?)</h1>'),
  2919. }
  2920. return [info]
  2921. class JustinTVIE(InfoExtractor):
  2922. """Information extractor for justin.tv and twitch.tv"""
  2923. # TODO: One broadcast may be split into multiple videos. The key
  2924. # 'broadcast_id' is the same for all parts, and 'broadcast_part'
  2925. # starts at 1 and increases. Can we treat all parts as one video?
  2926. _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?(?:twitch|justin)\.tv/
  2927. ([^/]+)(?:/b/([^/]+))?/?(?:\#.*)?$"""
  2928. _JUSTIN_PAGE_LIMIT = 100
  2929. IE_NAME = u'justin.tv'
  2930. def report_extraction(self, file_id):
  2931. """Report information extraction."""
  2932. self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
  2933. def report_download_page(self, channel, offset):
  2934. """Report attempt to download a single page of videos."""
  2935. self._downloader.to_screen(u'[%s] %s: Downloading video information from %d to %d' %
  2936. (self.IE_NAME, channel, offset, offset + self._JUSTIN_PAGE_LIMIT))
  2937. # Return count of items, list of *valid* items
  2938. def _parse_page(self, url):
  2939. try:
  2940. urlh = compat_urllib_request.urlopen(url)
  2941. webpage_bytes = urlh.read()
  2942. webpage = webpage_bytes.decode('utf-8', 'ignore')
  2943. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  2944. self._downloader.trouble(u'ERROR: unable to download video info JSON: %s' % compat_str(err))
  2945. return
  2946. response = json.loads(webpage)
  2947. info = []
  2948. for clip in response:
  2949. video_url = clip['video_file_url']
  2950. if video_url:
  2951. video_extension = os.path.splitext(video_url)[1][1:]
  2952. video_date = re.sub('-', '', clip['created_on'][:10])
  2953. info.append({
  2954. 'id': clip['id'],
  2955. 'url': video_url,
  2956. 'title': clip['title'],
  2957. 'uploader': clip.get('user_id', clip.get('channel_id')),
  2958. 'upload_date': video_date,
  2959. 'ext': video_extension,
  2960. })
  2961. return (len(response), info)
  2962. def _real_extract(self, url):
  2963. mobj = re.match(self._VALID_URL, url)
  2964. if mobj is None:
  2965. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2966. return
  2967. api = 'http://api.justin.tv'
  2968. video_id = mobj.group(mobj.lastindex)
  2969. paged = False
  2970. if mobj.lastindex == 1:
  2971. paged = True
  2972. api += '/channel/archives/%s.json'
  2973. else:
  2974. api += '/clip/show/%s.json'
  2975. api = api % (video_id,)
  2976. self.report_extraction(video_id)
  2977. info = []
  2978. offset = 0
  2979. limit = self._JUSTIN_PAGE_LIMIT
  2980. while True:
  2981. if paged:
  2982. self.report_download_page(video_id, offset)
  2983. page_url = api + ('?offset=%d&limit=%d' % (offset, limit))
  2984. page_count, page_info = self._parse_page(page_url)
  2985. info.extend(page_info)
  2986. if not paged or page_count != limit:
  2987. break
  2988. offset += limit
  2989. return info
  2990. class FunnyOrDieIE(InfoExtractor):
  2991. _VALID_URL = r'^(?:https?://)?(?:www\.)?funnyordie\.com/videos/(?P<id>[0-9a-f]+)/.*$'
  2992. def _real_extract(self, url):
  2993. mobj = re.match(self._VALID_URL, url)
  2994. if mobj is None:
  2995. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  2996. return
  2997. video_id = mobj.group('id')
  2998. webpage = self._download_webpage(url, video_id)
  2999. m = re.search(r'<video[^>]*>\s*<source[^>]*>\s*<source src="(?P<url>[^"]+)"', webpage, re.DOTALL)
  3000. if not m:
  3001. self._downloader.trouble(u'ERROR: unable to find video information')
  3002. video_url = unescapeHTML(m.group('url'))
  3003. m = re.search(r"class='player_page_h1'>\s+<a.*?>(?P<title>.*?)</a>", webpage)
  3004. if not m:
  3005. self._downloader.trouble(u'Cannot find video title')
  3006. title = unescapeHTML(m.group('title'))
  3007. m = re.search(r'<meta property="og:description" content="(?P<desc>.*?)"', webpage)
  3008. if m:
  3009. desc = unescapeHTML(m.group('desc'))
  3010. else:
  3011. desc = None
  3012. info = {
  3013. 'id': video_id,
  3014. 'url': video_url,
  3015. 'ext': 'mp4',
  3016. 'title': title,
  3017. 'description': desc,
  3018. }
  3019. return [info]
  3020. class TweetReelIE(InfoExtractor):
  3021. _VALID_URL = r'^(?:https?://)?(?:www\.)?tweetreel\.com/[?](?P<id>[0-9a-z]+)$'
  3022. def _real_extract(self, url):
  3023. mobj = re.match(self._VALID_URL, url)
  3024. if mobj is None:
  3025. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  3026. return
  3027. video_id = mobj.group('id')
  3028. webpage = self._download_webpage(url, video_id)
  3029. m = re.search(r'<div id="left" status_id="([0-9]+)">', webpage)
  3030. if not m:
  3031. self._downloader.trouble(u'ERROR: Cannot find status ID')
  3032. status_id = m.group(1)
  3033. m = re.search(r'<div class="tweet_text">(.*?)</div>', webpage, flags=re.DOTALL)
  3034. if not m:
  3035. self._downloader.trouble(u'WARNING: Cannot find description')
  3036. desc = unescapeHTML(re.sub('<a.*?</a>', '', m.group(1))).strip()
  3037. m = re.search(r'<div class="tweet_info">.*?from <a target="_blank" href="https?://twitter.com/(?P<uploader_id>.+?)">(?P<uploader>.+?)</a>', webpage, flags=re.DOTALL)
  3038. if not m:
  3039. self._downloader.trouble(u'ERROR: Cannot find uploader')
  3040. uploader = unescapeHTML(m.group('uploader'))
  3041. uploader_id = unescapeHTML(m.group('uploader_id'))
  3042. m = re.search(r'<span unixtime="([0-9]+)"', webpage)
  3043. if not m:
  3044. self._downloader.trouble(u'ERROR: Cannot find upload date')
  3045. upload_date = datetime.datetime.fromtimestamp(int(m.group(1))).strftime('%Y%m%d')
  3046. title = desc
  3047. video_url = 'http://files.tweetreel.com/video/' + status_id + '.mov'
  3048. info = {
  3049. 'id': video_id,
  3050. 'url': video_url,
  3051. 'ext': 'mov',
  3052. 'title': title,
  3053. 'description': desc,
  3054. 'uploader': uploader,
  3055. 'uploader_id': uploader_id,
  3056. 'internal_id': status_id,
  3057. 'upload_date': upload_date
  3058. }
  3059. return [info]
  3060. class SteamIE(InfoExtractor):
  3061. _VALID_URL = r"""http://store.steampowered.com/
  3062. (?P<urltype>video|app)/ #If the page is only for videos or for a game
  3063. (?P<gameID>\d+)/?
  3064. (?P<videoID>\d*)(?P<extra>\??) #For urltype == video we sometimes get the videoID
  3065. """
  3066. def suitable(self, url):
  3067. """Receives a URL and returns True if suitable for this IE."""
  3068. return re.match(self._VALID_URL, url, re.VERBOSE) is not None
  3069. def _real_extract(self, url):
  3070. m = re.match(self._VALID_URL, url, re.VERBOSE)
  3071. urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},"
  3072. gameID = m.group('gameID')
  3073. videourl = 'http://store.steampowered.com/video/%s/' % gameID
  3074. webpage = self._download_webpage(videourl, gameID)
  3075. mweb = re.finditer(urlRE, webpage)
  3076. namesRE = r'<span class="title">(?P<videoName>.+?)</span>'
  3077. titles = re.finditer(namesRE, webpage)
  3078. videos = []
  3079. for vid,vtitle in zip(mweb,titles):
  3080. video_id = vid.group('videoID')
  3081. title = vtitle.group('videoName')
  3082. video_url = vid.group('videoURL')
  3083. if not video_url:
  3084. self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id)
  3085. info = {
  3086. 'id':video_id,
  3087. 'url':video_url,
  3088. 'ext': 'flv',
  3089. 'title': unescapeHTML(title)
  3090. }
  3091. videos.append(info)
  3092. return videos
  3093. class UstreamIE(InfoExtractor):
  3094. _VALID_URL = r'http://www.ustream.tv/recorded/(?P<videoID>\d+)'
  3095. IE_NAME = u'ustream'
  3096. def _real_extract(self, url):
  3097. m = re.match(self._VALID_URL, url)
  3098. video_id = m.group('videoID')
  3099. video_url = u'http://tcdn.ustream.tv/video/%s' % video_id
  3100. webpage = self._download_webpage(url, video_id)
  3101. m = re.search(r'data-title="(?P<title>.+)"',webpage)
  3102. title = m.group('title')
  3103. m = re.search(r'<a class="state" data-content-type="channel" data-content-id="(?P<uploader>\d+)"',webpage)
  3104. uploader = m.group('uploader')
  3105. info = {
  3106. 'id':video_id,
  3107. 'url':video_url,
  3108. 'ext': 'flv',
  3109. 'title': title,
  3110. 'uploader': uploader
  3111. }
  3112. return [info]
  3113. class YouPornIE(InfoExtractor):
  3114. """Information extractor for youporn.com."""
  3115. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+)'
  3116. IE_NAME = u'youporn'
  3117. VIDEO_TITLE_RE = r'videoTitleArea">(?P<title>.*)</h1>'
  3118. VIDEO_DATE_RE = r'Date:</b>(?P<date>.*)</li>'
  3119. VIDEO_UPLOADER_RE = r'Submitted:</b>(?P<uploader>.*)</li>'
  3120. DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
  3121. LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">'
  3122. def __init__(self, downloader=None):
  3123. InfoExtractor.__init__(self, downloader)
  3124. def report_id(self, video_id):
  3125. """Report finding video ID"""
  3126. self._downloader.to_screen(u'[youporn] Video ID: %s' % video_id)
  3127. def report_webpage(self, url):
  3128. """Report downloading page"""
  3129. self._downloader.to_screen(u'[youporn] Downloaded page: %s' % url)
  3130. def report_title(self, video_title):
  3131. """Report dfinding title"""
  3132. self._downloader.to_screen(u'[youporn] Title: %s' % video_title)
  3133. def report_uploader(self, uploader):
  3134. """Report dfinding title"""
  3135. self._downloader.to_screen(u'[youporn] Uploader: %s' % uploader)
  3136. def report_upload_date(self, video_date):
  3137. """Report finding date"""
  3138. self._downloader.to_screen(u'[youporn] Date: %s' % video_date)
  3139. def _print_formats(self, formats):
  3140. """Print all available formats"""
  3141. print 'Available formats:'
  3142. print u'ext\t\tformat'
  3143. print u'---------------------------------'
  3144. for format in formats:
  3145. print u'%s\t\t%s' % (format['ext'], format['format'])
  3146. def _specific(self, req_format, formats):
  3147. for x in formats:
  3148. if(x["format"]==req_format):
  3149. return x
  3150. return None
  3151. def _real_extract(self, url):
  3152. mobj = re.match(self._VALID_URL, url)
  3153. if mobj is None:
  3154. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  3155. return
  3156. video_id = mobj.group('videoid').decode('utf-8')
  3157. self.report_id(video_id)
  3158. # Get webpage content
  3159. try:
  3160. webpage = urllib2.urlopen(url).read()
  3161. except (urllib2.URLError, httplib.HTTPException, socket.error), err:
  3162. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
  3163. return
  3164. self.report_webpage(url)
  3165. # Get the video title
  3166. result = re.search(self.VIDEO_TITLE_RE, webpage)
  3167. if result is None:
  3168. self._downloader.trouble(u'ERROR: unable to extract video title')
  3169. return
  3170. video_title = result.group('title').decode('utf-8').strip()
  3171. self.report_title(video_title)
  3172. # Get the video date
  3173. result = re.search(self.VIDEO_DATE_RE, webpage)
  3174. if result is None:
  3175. self._downloader.trouble(u'ERROR: unable to extract video date')
  3176. return
  3177. upload_date = result.group('date').decode('utf-8').strip()
  3178. self.report_upload_date(upload_date)
  3179. # Get the video uploader
  3180. result = re.search(self.VIDEO_UPLOADER_RE, webpage)
  3181. if result is None:
  3182. self._downloader.trouble(u'ERROR: unable to extract uploader')
  3183. return
  3184. video_uploader = result.group('uploader').decode('utf-8').strip()
  3185. video_uploader = clean_html( video_uploader )
  3186. self.report_uploader(video_uploader)
  3187. # Get all of the formats available
  3188. result = re.search(self.DOWNLOAD_LIST_RE, webpage)
  3189. if result is None:
  3190. self._downloader.trouble(u'ERROR: unable to extract download list')
  3191. return
  3192. download_list_html = result.group('download_list').decode('utf-8').strip()
  3193. # Get all of the links from the page
  3194. links = re.findall(self.LINK_RE, download_list_html)
  3195. if(len(links) == 0):
  3196. self._downloader.trouble(u'ERROR: no known formats available for video')
  3197. return
  3198. self._downloader.to_screen(u'[youporn] Links found: %d' % len(links))
  3199. formats = []
  3200. for link in links:
  3201. # A link looks like this:
  3202. # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
  3203. # A path looks like this:
  3204. # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
  3205. video_url = unescapeHTML( link.decode('utf-8') )
  3206. path = urlparse( video_url ).path
  3207. extension = os.path.splitext( path )[1][1:]
  3208. format = path.split('/')[4].split('_')[:2]
  3209. size = format[0]
  3210. bitrate = format[1]
  3211. format = "-".join( format )
  3212. title = u'%s-%s-%s' % (video_title, size, bitrate)
  3213. formats.append({
  3214. 'id': video_id,
  3215. 'url': video_url,
  3216. 'uploader': video_uploader,
  3217. 'upload_date': upload_date,
  3218. 'title': title,
  3219. 'ext': extension,
  3220. 'format': format,
  3221. 'thumbnail': None,
  3222. 'description': None,
  3223. 'player_url': None
  3224. })
  3225. if self._downloader.params.get('listformats', None):
  3226. self._print_formats(formats)
  3227. return
  3228. req_format = self._downloader.params.get('format', None)
  3229. #format_limit = self._downloader.params.get('format_limit', None)
  3230. self._downloader.to_screen(u'[youporn] Format: %s' % req_format)
  3231. if req_format is None or req_format == 'best':
  3232. return [formats[0]]
  3233. elif req_format == 'worst':
  3234. return [formats[-1]]
  3235. elif req_format in ('-1', 'all'):
  3236. return formats
  3237. else:
  3238. format = self._specific( req_format, formats )
  3239. if result is None:
  3240. self._downloader.trouble(u'ERROR: requested format not available')
  3241. return
  3242. return [format]
  3243. class PornotubeIE(InfoExtractor):
  3244. """Information extractor for pornotube.com."""
  3245. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$'
  3246. IE_NAME = u'pornotube'
  3247. VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",'
  3248. VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
  3249. def __init__(self, downloader=None):
  3250. InfoExtractor.__init__(self, downloader)
  3251. def report_extract_entry(self, url):
  3252. """Report downloading extry"""
  3253. self._downloader.to_screen(u'[pornotube] Downloading entry: %s' % url.decode('utf-8'))
  3254. def report_date(self, upload_date):
  3255. """Report finding uploaded date"""
  3256. self._downloader.to_screen(u'[pornotube] Entry date: %s' % upload_date)
  3257. def report_webpage(self, url):
  3258. """Report downloading page"""
  3259. self._downloader.to_screen(u'[pornotube] Downloaded page: %s' % url)
  3260. def report_title(self, video_title):
  3261. """Report downloading extry"""
  3262. self._downloader.to_screen(u'[pornotube] Title: %s' % video_title.decode('utf-8'))
  3263. def _real_extract(self, url):
  3264. mobj = re.match(self._VALID_URL, url)
  3265. if mobj is None:
  3266. self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
  3267. return
  3268. video_id = mobj.group('videoid').decode('utf-8')
  3269. video_title = mobj.group('title').decode('utf-8')
  3270. self.report_title(video_title);
  3271. # Get webpage content
  3272. try:
  3273. webpage = urllib2.urlopen(url).read()
  3274. except (urllib2.URLError, httplib.HTTPException, socket.error), err:
  3275. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
  3276. return
  3277. self.report_webpage(url)
  3278. # Get the video URL
  3279. result = re.search(self.VIDEO_URL_RE, webpage)
  3280. if result is None:
  3281. self._downloader.trouble(u'ERROR: unable to extract video url')
  3282. return
  3283. video_url = urllib.unquote(result.group('url').decode('utf-8'))
  3284. self.report_extract_entry(video_url)
  3285. #Get the uploaded date
  3286. result = re.search(self.VIDEO_UPLOADED_RE, webpage)
  3287. if result is None:
  3288. self._downloader.trouble(u'ERROR: unable to extract video title')
  3289. return
  3290. upload_date = result.group('date').decode('utf-8')
  3291. self.report_date(upload_date);
  3292. info = {'id': video_id,
  3293. 'url': video_url,
  3294. 'uploader': None,
  3295. 'upload_date': upload_date,
  3296. 'title': video_title,
  3297. 'ext': 'flv',
  3298. 'format': 'flv',
  3299. 'thumbnail': None,
  3300. 'description': None,
  3301. 'player_url': None}
  3302. return [info]
  3303. class YouJizzIE(InfoExtractor):
  3304. """Information extractor for youjizz.com."""
  3305. _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/([^.]+).html$'
  3306. IE_NAME = u'youjizz'
  3307. VIDEO_TITLE_RE = r'<title>(?P<title>.*)</title>'
  3308. EMBED_PAGE_RE = r'http://www.youjizz.com/videos/embed/(?P<videoid>[0-9]+)'
  3309. SOURCE_RE = r'so.addVariable\("file",encodeURIComponent\("(?P<source>[^"]+)"\)\);'
  3310. def __init__(self, downloader=None):
  3311. InfoExtractor.__init__(self, downloader)
  3312. def report_extract_entry(self, url):
  3313. """Report downloading extry"""
  3314. self._downloader.to_screen(u'[youjizz] Downloading entry: %s' % url.decode('utf-8'))
  3315. def report_webpage(self, url):
  3316. """Report downloading page"""
  3317. self._downloader.to_screen(u'[youjizz] Downloaded page: %s' % url)
  3318. def report_title(self, video_title):
  3319. """Report downloading extry"""
  3320. self._downloader.to_screen(u'[youjizz] Title: %s' % video_title.decode('utf-8'))
  3321. def report_embed_page(self, embed_page):
  3322. """Report downloading extry"""
  3323. self._downloader.to_screen(u'[youjizz] Embed Page: %s' % embed_page.decode('utf-8'))
  3324. def _real_extract(self, url):
  3325. # Get webpage content
  3326. try:
  3327. webpage = urllib2.urlopen(url).read()
  3328. except (urllib2.URLError, httplib.HTTPException, socket.error), err:
  3329. self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
  3330. return
  3331. self.report_webpage(url)
  3332. # Get the video title
  3333. result = re.search(self.VIDEO_TITLE_RE, webpage)
  3334. if result is None:
  3335. self._downloader.trouble(u'ERROR: unable to extract video title')
  3336. return
  3337. video_title = result.group('title').decode('utf-8').strip()
  3338. self.report_title(video_title)
  3339. # Get the embed page
  3340. result = re.search(self.EMBED_PAGE_RE, webpage)
  3341. if result is None:
  3342. self._downloader.trouble(u'ERROR: unable to extract embed page')
  3343. return
  3344. embed_page_url = result.group(0).decode('utf-8').strip()
  3345. video_id = result.group('videoid').decode('utf-8')
  3346. self.report_embed_page(embed_page_url)
  3347. try:
  3348. webpage = urllib2.urlopen(embed_page_url).read()
  3349. except (urllib2.URLError, httplib.HTTPException, socket.error), err:
  3350. self._downloader.trouble(u'ERROR: unable to download video embed page: %s' % err)
  3351. return
  3352. # Get the video URL
  3353. result = re.search(self.SOURCE_RE, webpage)
  3354. if result is None:
  3355. self._downloader.trouble(u'ERROR: unable to extract video url')
  3356. return
  3357. video_url = result.group('source').decode('utf-8')
  3358. self.report_extract_entry(video_url)
  3359. info = {'id': video_id,
  3360. 'url': video_url,
  3361. 'uploader': None,
  3362. 'upload_date': None,
  3363. 'title': video_title,
  3364. 'ext': 'flv',
  3365. 'format': 'flv',
  3366. 'thumbnail': None,
  3367. 'description': None,
  3368. 'player_url': embed_page_url}
  3369. return [info]
  3370. def gen_extractors():
  3371. """ Return a list of an instance of every supported extractor.
  3372. The order does matter; the first extractor matched is the one handling the URL.
  3373. """
  3374. return [
  3375. YoutubePlaylistIE(),
  3376. YoutubeChannelIE(),
  3377. YoutubeUserIE(),
  3378. YoutubeSearchIE(),
  3379. YoutubeIE(),
  3380. MetacafeIE(),
  3381. DailymotionIE(),
  3382. GoogleSearchIE(),
  3383. PhotobucketIE(),
  3384. YahooIE(),
  3385. YahooSearchIE(),
  3386. DepositFilesIE(),
  3387. FacebookIE(),
  3388. BlipTVUserIE(),
  3389. BlipTVIE(),
  3390. VimeoIE(),
  3391. MyVideoIE(),
  3392. ComedyCentralIE(),
  3393. EscapistIE(),
  3394. CollegeHumorIE(),
  3395. XVideosIE(),
  3396. SoundcloudIE(),
  3397. InfoQIE(),
  3398. MixcloudIE(),
  3399. StanfordOpenClassroomIE(),
  3400. MTVIE(),
  3401. YoukuIE(),
  3402. XNXXIE(),
  3403. YouJizzIE(),
  3404. PornotubeIE(),
  3405. YouPornIE(),
  3406. GooglePlusIE(),
  3407. ArteTvIE(),
  3408. NBAIE(),
  3409. JustinTVIE(),
  3410. FunnyOrDieIE(),
  3411. TweetReelIE(),
  3412. SteamIE(),
  3413. UstreamIE(),
  3414. GenericIE()
  3415. ]