You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

922 lines
36 KiB

10 years ago
11 years ago
11 years ago
11 years ago
  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import os
  4. import re
  5. from .common import InfoExtractor
  6. from .youtube import YoutubeIE
  7. from ..utils import (
  8. compat_urllib_parse,
  9. compat_urlparse,
  10. compat_xml_parse_error,
  11. determine_ext,
  12. ExtractorError,
  13. float_or_none,
  14. HEADRequest,
  15. orderedSet,
  16. parse_xml,
  17. smuggle_url,
  18. unescapeHTML,
  19. unified_strdate,
  20. unsmuggle_url,
  21. url_basename,
  22. )
  23. from .brightcove import BrightcoveIE
  24. from .ooyala import OoyalaIE
  25. from .rutv import RUTVIE
  26. from .smotri import SmotriIE
  27. class GenericIE(InfoExtractor):
  28. IE_DESC = 'Generic downloader that works on some sites'
  29. _VALID_URL = r'.*'
  30. IE_NAME = 'generic'
  31. _TESTS = [
  32. {
  33. 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
  34. 'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
  35. 'info_dict': {
  36. 'id': '13601338388002',
  37. 'ext': 'mp4',
  38. 'uploader': 'www.hodiho.fr',
  39. 'title': 'R\u00e9gis plante sa Jeep',
  40. }
  41. },
  42. # bandcamp page with custom domain
  43. {
  44. 'add_ie': ['Bandcamp'],
  45. 'url': 'http://bronyrock.com/track/the-pony-mash',
  46. 'info_dict': {
  47. 'id': '3235767654',
  48. 'ext': 'mp3',
  49. 'title': 'The Pony Mash',
  50. 'uploader': 'M_Pallante',
  51. },
  52. 'skip': 'There is a limit of 200 free downloads / month for the test song',
  53. },
  54. # embedded brightcove video
  55. # it also tests brightcove videos that need to set the 'Referer' in the
  56. # http requests
  57. {
  58. 'add_ie': ['Brightcove'],
  59. 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
  60. 'info_dict': {
  61. 'id': '2765128793001',
  62. 'ext': 'mp4',
  63. 'title': 'Le cours de bourse : l’analyse technique',
  64. 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
  65. 'uploader': 'BFM BUSINESS',
  66. },
  67. 'params': {
  68. 'skip_download': True,
  69. },
  70. },
  71. {
  72. # https://github.com/rg3/youtube-dl/issues/2253
  73. 'url': 'http://bcove.me/i6nfkrc3',
  74. 'md5': '0ba9446db037002366bab3b3eb30c88c',
  75. 'info_dict': {
  76. 'id': '3101154703001',
  77. 'ext': 'mp4',
  78. 'title': 'Still no power',
  79. 'uploader': 'thestar.com',
  80. 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
  81. },
  82. 'add_ie': ['Brightcove'],
  83. },
  84. {
  85. 'url': 'http://www.championat.com/video/football/v/87/87499.html',
  86. 'md5': 'fb973ecf6e4a78a67453647444222983',
  87. 'info_dict': {
  88. 'id': '3414141473001',
  89. 'ext': 'mp4',
  90. 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
  91. 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
  92. 'uploader': 'Championat',
  93. },
  94. },
  95. # Direct link to a video
  96. {
  97. 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
  98. 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
  99. 'info_dict': {
  100. 'id': 'trailer',
  101. 'ext': 'mp4',
  102. 'title': 'trailer',
  103. 'upload_date': '20100513',
  104. }
  105. },
  106. # ooyala video
  107. {
  108. 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
  109. 'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
  110. 'info_dict': {
  111. 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
  112. 'ext': 'mp4',
  113. 'title': '2cc213299525360.mov', # that's what we get
  114. },
  115. },
  116. # google redirect
  117. {
  118. 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
  119. 'info_dict': {
  120. 'id': 'cmQHVoWB5FY',
  121. 'ext': 'mp4',
  122. 'upload_date': '20130224',
  123. 'uploader_id': 'TheVerge',
  124. 'description': 'Chris Ziegler takes a look at the Alcatel OneTouch Fire and the ZTE Open; two of the first Firefox OS handsets to be officially announced.',
  125. 'uploader': 'The Verge',
  126. 'title': 'First Firefox OS phones side-by-side',
  127. },
  128. 'params': {
  129. 'skip_download': False,
  130. }
  131. },
  132. # embed.ly video
  133. {
  134. 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
  135. 'info_dict': {
  136. 'id': '9ODmcdjQcHQ',
  137. 'ext': 'mp4',
  138. 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
  139. 'upload_date': '20140225',
  140. 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
  141. 'uploader': 'Tested',
  142. 'uploader_id': 'testedcom',
  143. },
  144. # No need to test YoutubeIE here
  145. 'params': {
  146. 'skip_download': True,
  147. },
  148. },
  149. # funnyordie embed
  150. {
  151. 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
  152. 'md5': '7cf780be104d40fea7bae52eed4a470e',
  153. 'info_dict': {
  154. 'id': '18e820ec3f',
  155. 'ext': 'mp4',
  156. 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
  157. 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
  158. },
  159. },
  160. # RUTV embed
  161. {
  162. 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
  163. 'info_dict': {
  164. 'id': '776940',
  165. 'ext': 'mp4',
  166. 'title': 'Охотское море стало целиком российским',
  167. 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
  168. },
  169. 'params': {
  170. # m3u8 download
  171. 'skip_download': True,
  172. },
  173. },
  174. # Embedded TED video
  175. {
  176. 'url': 'http://en.support.wordpress.com/videos/ted-talks/',
  177. 'md5': 'deeeabcc1085eb2ba205474e7235a3d5',
  178. 'info_dict': {
  179. 'id': '981',
  180. 'ext': 'mp4',
  181. 'title': 'My web playroom',
  182. 'uploader': 'Ze Frank',
  183. 'description': 'md5:ddb2a40ecd6b6a147e400e535874947b',
  184. }
  185. },
  186. # Embeded Ustream video
  187. {
  188. 'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
  189. 'md5': '27b99cdb639c9b12a79bca876a073417',
  190. 'info_dict': {
  191. 'id': '45734260',
  192. 'ext': 'flv',
  193. 'uploader': 'AU SPA: The NSA and Privacy',
  194. 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
  195. }
  196. },
  197. # nowvideo embed hidden behind percent encoding
  198. {
  199. 'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
  200. 'md5': '2baf4ddd70f697d94b1c18cf796d5107',
  201. 'info_dict': {
  202. 'id': '06e53103ca9aa',
  203. 'ext': 'flv',
  204. 'title': 'Macross Episode 001 Watch Macross Episode 001 onl',
  205. 'description': 'No description',
  206. },
  207. },
  208. # arte embed
  209. {
  210. 'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
  211. 'md5': '7653032cbb25bf6c80d80f217055fa43',
  212. 'info_dict': {
  213. 'id': '048195-004_PLUS7-F',
  214. 'ext': 'flv',
  215. 'title': 'X:enius',
  216. 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
  217. 'upload_date': '20140320',
  218. },
  219. 'params': {
  220. 'skip_download': 'Requires rtmpdump'
  221. }
  222. },
  223. # smotri embed
  224. {
  225. 'url': 'http://rbctv.rbc.ru/archive/news/562949990879132.shtml',
  226. 'md5': 'ec40048448e9284c9a1de77bb188108b',
  227. 'info_dict': {
  228. 'id': 'v27008541fad',
  229. 'ext': 'mp4',
  230. 'title': 'Крым и Севастополь вошли в состав России',
  231. 'description': 'md5:fae01b61f68984c7bd2fa741e11c3175',
  232. 'duration': 900,
  233. 'upload_date': '20140318',
  234. 'uploader': 'rbctv_2012_4',
  235. 'uploader_id': 'rbctv_2012_4',
  236. },
  237. },
  238. # Condé Nast embed
  239. {
  240. 'url': 'http://www.wired.com/2014/04/honda-asimo/',
  241. 'md5': 'ba0dfe966fa007657bd1443ee672db0f',
  242. 'info_dict': {
  243. 'id': '53501be369702d3275860000',
  244. 'ext': 'mp4',
  245. 'title': 'Honda’s New Asimo Robot Is More Human Than Ever',
  246. }
  247. },
  248. # Dailymotion embed
  249. {
  250. 'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
  251. 'md5': '441aeeb82eb72c422c7f14ec533999cd',
  252. 'info_dict': {
  253. 'id': 'k2mm4bCdJ6CQ2i7c8o2',
  254. 'ext': 'mp4',
  255. 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
  256. 'uploader': 'Spi0n',
  257. },
  258. 'add_ie': ['Dailymotion'],
  259. },
  260. # YouTube embed
  261. {
  262. 'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
  263. 'info_dict': {
  264. 'id': 'FXRb4ykk4S0',
  265. 'ext': 'mp4',
  266. 'title': 'The NBL Auction 2014',
  267. 'uploader': 'BADMINTON England',
  268. 'uploader_id': 'BADMINTONEvents',
  269. 'upload_date': '20140603',
  270. 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
  271. },
  272. 'add_ie': ['Youtube'],
  273. 'params': {
  274. 'skip_download': True,
  275. }
  276. },
  277. # MTVSercices embed
  278. {
  279. 'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
  280. 'md5': '35727f82f58c76d996fc188f9755b0d5',
  281. 'info_dict': {
  282. 'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
  283. 'ext': 'mp4',
  284. 'title': 'Review',
  285. 'description': 'Mario\'s life in the fast lane has never looked so good.',
  286. },
  287. },
  288. # YouTube embed via <data-embed-url="">
  289. {
  290. 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
  291. 'info_dict': {
  292. 'id': 'jpSGZsgga_I',
  293. 'ext': 'mp4',
  294. 'title': 'Asphalt 8: Airborne - Launch Trailer',
  295. 'uploader': 'Gameloft',
  296. 'uploader_id': 'gameloft',
  297. 'upload_date': '20130821',
  298. 'description': 'md5:87bd95f13d8be3e7da87a5f2c443106a',
  299. },
  300. 'params': {
  301. 'skip_download': True,
  302. }
  303. },
  304. # Camtasia studio
  305. {
  306. 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
  307. 'playlist': [{
  308. 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
  309. 'info_dict': {
  310. 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
  311. 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
  312. 'ext': 'flv',
  313. 'duration': 2235.90,
  314. }
  315. }, {
  316. 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
  317. 'info_dict': {
  318. 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
  319. 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
  320. 'ext': 'flv',
  321. 'duration': 2235.93,
  322. }
  323. }],
  324. 'info_dict': {
  325. 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
  326. }
  327. },
  328. # Flowplayer
  329. {
  330. 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
  331. 'md5': '9d65602bf31c6e20014319c7d07fba27',
  332. 'info_dict': {
  333. 'id': '5123ea6d5e5a7',
  334. 'ext': 'mp4',
  335. 'age_limit': 18,
  336. 'uploader': 'www.handjobhub.com',
  337. 'title': 'Busty Blonde Siri Tit Fuck While Wank at Handjob Hub',
  338. }
  339. },
  340. # RSS feed
  341. {
  342. 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
  343. 'info_dict': {
  344. 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
  345. 'title': 'Zero Punctuation',
  346. 'description': 're:'
  347. },
  348. 'playlist_mincount': 11,
  349. },
  350. # Multiple brightcove videos
  351. # https://github.com/rg3/youtube-dl/issues/2283
  352. {
  353. 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
  354. 'info_dict': {
  355. 'id': 'always-never',
  356. 'title': 'Always / Never - The New Yorker',
  357. },
  358. 'playlist_count': 3,
  359. 'params': {
  360. 'extract_flat': False,
  361. 'skip_download': True,
  362. }
  363. },
  364. # MLB embed
  365. {
  366. 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
  367. 'md5': '96f09a37e44da40dd083e12d9a683327',
  368. 'info_dict': {
  369. 'id': '33322633',
  370. 'ext': 'mp4',
  371. 'title': 'Ump changes call to ball',
  372. 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
  373. 'duration': 48,
  374. 'timestamp': 1401537900,
  375. 'upload_date': '20140531',
  376. 'thumbnail': 're:^https?://.*\.jpg$',
  377. },
  378. },
  379. ]
  380. def report_download_webpage(self, video_id):
  381. """Report webpage download."""
  382. if not self._downloader.params.get('test', False):
  383. self._downloader.report_warning('Falling back on generic information extractor.')
  384. super(GenericIE, self).report_download_webpage(video_id)
  385. def report_following_redirect(self, new_url):
  386. """Report information extraction."""
  387. self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
  388. def _extract_rss(self, url, video_id, doc):
  389. playlist_title = doc.find('./channel/title').text
  390. playlist_desc_el = doc.find('./channel/description')
  391. playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
  392. entries = [{
  393. '_type': 'url',
  394. 'url': e.find('link').text,
  395. 'title': e.find('title').text,
  396. } for e in doc.findall('./channel/item')]
  397. return {
  398. '_type': 'playlist',
  399. 'id': url,
  400. 'title': playlist_title,
  401. 'description': playlist_desc,
  402. 'entries': entries,
  403. }
  404. def _extract_camtasia(self, url, video_id, webpage):
  405. """ Returns None if no camtasia video can be found. """
  406. camtasia_cfg = self._search_regex(
  407. r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
  408. webpage, 'camtasia configuration file', default=None)
  409. if camtasia_cfg is None:
  410. return None
  411. title = self._html_search_meta('DC.title', webpage, fatal=True)
  412. camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
  413. camtasia_cfg = self._download_xml(
  414. camtasia_url, video_id,
  415. note='Downloading camtasia configuration',
  416. errnote='Failed to download camtasia configuration')
  417. fileset_node = camtasia_cfg.find('./playlist/array/fileset')
  418. entries = []
  419. for n in fileset_node.getchildren():
  420. url_n = n.find('./uri')
  421. if url_n is None:
  422. continue
  423. entries.append({
  424. 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
  425. 'title': '%s - %s' % (title, n.tag),
  426. 'url': compat_urlparse.urljoin(url, url_n.text),
  427. 'duration': float_or_none(n.find('./duration').text),
  428. })
  429. return {
  430. '_type': 'playlist',
  431. 'entries': entries,
  432. 'title': title,
  433. }
  434. def _real_extract(self, url):
  435. if url.startswith('//'):
  436. return {
  437. '_type': 'url',
  438. 'url': self.http_scheme() + url,
  439. }
  440. parsed_url = compat_urlparse.urlparse(url)
  441. if not parsed_url.scheme:
  442. default_search = self._downloader.params.get('default_search')
  443. if default_search is None:
  444. default_search = 'fixup_error'
  445. if default_search in ('auto', 'auto_warning', 'fixup_error'):
  446. if '/' in url:
  447. self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
  448. return self.url_result('http://' + url)
  449. elif default_search != 'fixup_error':
  450. if default_search == 'auto_warning':
  451. if re.match(r'^(?:url|URL)$', url):
  452. raise ExtractorError(
  453. 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url,
  454. expected=True)
  455. else:
  456. self._downloader.report_warning(
  457. 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url)
  458. return self.url_result('ytsearch:' + url)
  459. if default_search in ('error', 'fixup_error'):
  460. raise ExtractorError(
  461. ('%r is not a valid URL. '
  462. 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
  463. ) % (url, url), expected=True)
  464. else:
  465. assert ':' in default_search
  466. return self.url_result(default_search + url)
  467. url, smuggled_data = unsmuggle_url(url)
  468. force_videoid = None
  469. if smuggled_data and 'force_videoid' in smuggled_data:
  470. force_videoid = smuggled_data['force_videoid']
  471. video_id = force_videoid
  472. else:
  473. video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
  474. self.to_screen('%s: Requesting header' % video_id)
  475. head_req = HEADRequest(url)
  476. response = self._request_webpage(
  477. head_req, video_id,
  478. note=False, errnote='Could not send HEAD request to %s' % url,
  479. fatal=False)
  480. if response is not False:
  481. # Check for redirect
  482. new_url = response.geturl()
  483. if url != new_url:
  484. self.report_following_redirect(new_url)
  485. if force_videoid:
  486. new_url = smuggle_url(
  487. new_url, {'force_videoid': force_videoid})
  488. return self.url_result(new_url)
  489. # Check for direct link to a video
  490. content_type = response.headers.get('Content-Type', '')
  491. m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
  492. if m:
  493. upload_date = response.headers.get('Last-Modified')
  494. if upload_date:
  495. upload_date = unified_strdate(upload_date)
  496. return {
  497. 'id': video_id,
  498. 'title': os.path.splitext(url_basename(url))[0],
  499. 'formats': [{
  500. 'format_id': m.group('format_id'),
  501. 'url': url,
  502. 'vcodec': 'none' if m.group('type') == 'audio' else None
  503. }],
  504. 'upload_date': upload_date,
  505. }
  506. try:
  507. webpage = self._download_webpage(url, video_id)
  508. except ValueError:
  509. # since this is the last-resort InfoExtractor, if
  510. # this error is thrown, it'll be thrown here
  511. raise ExtractorError('Failed to download URL: %s' % url)
  512. self.report_extraction(video_id)
  513. # Is it an RSS feed?
  514. try:
  515. doc = parse_xml(webpage)
  516. if doc.tag == 'rss':
  517. return self._extract_rss(url, video_id, doc)
  518. except compat_xml_parse_error:
  519. pass
  520. # Is it a Camtasia project?
  521. camtasia_res = self._extract_camtasia(url, video_id, webpage)
  522. if camtasia_res is not None:
  523. return camtasia_res
  524. # Sometimes embedded video player is hidden behind percent encoding
  525. # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
  526. # Unescaping the whole page allows to handle those cases in a generic way
  527. webpage = compat_urllib_parse.unquote(webpage)
  528. # it's tempting to parse this further, but you would
  529. # have to take into account all the variations like
  530. # Video Title - Site Name
  531. # Site Name | Video Title
  532. # Video Title - Tagline | Site Name
  533. # and so on and so forth; it's just not practical
  534. video_title = self._html_search_regex(
  535. r'(?s)<title>(.*?)</title>', webpage, 'video title',
  536. default='video')
  537. # Try to detect age limit automatically
  538. age_limit = self._rta_search(webpage)
  539. # And then there are the jokers who advertise that they use RTA,
  540. # but actually don't.
  541. AGE_LIMIT_MARKERS = [
  542. r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
  543. ]
  544. if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
  545. age_limit = 18
  546. # video uploader is domain name
  547. video_uploader = self._search_regex(
  548. r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
  549. # Helper method
  550. def _playlist_from_matches(matches, getter, ie=None):
  551. urlrs = orderedSet(self.url_result(getter(m), ie) for m in matches)
  552. return self.playlist_result(
  553. urlrs, playlist_id=video_id, playlist_title=video_title)
  554. # Look for BrightCove:
  555. bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
  556. if bc_urls:
  557. self.to_screen('Brightcove video detected.')
  558. entries = [{
  559. '_type': 'url',
  560. 'url': smuggle_url(bc_url, {'Referer': url}),
  561. 'ie_key': 'Brightcove'
  562. } for bc_url in bc_urls]
  563. return {
  564. '_type': 'playlist',
  565. 'title': video_title,
  566. 'id': video_id,
  567. 'entries': entries,
  568. }
  569. # Look for embedded (iframe) Vimeo player
  570. mobj = re.search(
  571. r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
  572. if mobj:
  573. player_url = unescapeHTML(mobj.group('url'))
  574. surl = smuggle_url(player_url, {'Referer': url})
  575. return self.url_result(surl, 'Vimeo')
  576. # Look for embedded (swf embed) Vimeo player
  577. mobj = re.search(
  578. r'<embed[^>]+?src="(https?://(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
  579. if mobj:
  580. return self.url_result(mobj.group(1), 'Vimeo')
  581. # Look for embedded YouTube player
  582. matches = re.findall(r'''(?x)
  583. (?:
  584. <iframe[^>]+?src=|
  585. data-video-url=|
  586. <embed[^>]+?src=|
  587. embedSWF\(?:\s*
  588. )
  589. (["\'])
  590. (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
  591. (?:embed|v)/.+?)
  592. \1''', webpage)
  593. if matches:
  594. return _playlist_from_matches(
  595. matches, lambda m: unescapeHTML(m[1]), ie='Youtube')
  596. # Look for embedded Dailymotion player
  597. matches = re.findall(
  598. r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
  599. if matches:
  600. return _playlist_from_matches(
  601. matches, lambda m: unescapeHTML(m[1]))
  602. # Look for embedded Wistia player
  603. match = re.search(
  604. r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
  605. if match:
  606. return {
  607. '_type': 'url_transparent',
  608. 'url': unescapeHTML(match.group('url')),
  609. 'ie_key': 'Wistia',
  610. 'uploader': video_uploader,
  611. 'title': video_title,
  612. 'id': video_id,
  613. }
  614. # Look for embedded blip.tv player
  615. mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
  616. if mobj:
  617. return self.url_result('http://blip.tv/a/a-'+mobj.group(1), 'BlipTV')
  618. mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
  619. if mobj:
  620. return self.url_result(mobj.group(1), 'BlipTV')
  621. # Look for embedded condenast player
  622. matches = re.findall(
  623. r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
  624. webpage)
  625. if matches:
  626. return {
  627. '_type': 'playlist',
  628. 'entries': [{
  629. '_type': 'url',
  630. 'ie_key': 'CondeNast',
  631. 'url': ma,
  632. } for ma in matches],
  633. 'title': video_title,
  634. 'id': video_id,
  635. }
  636. # Look for Bandcamp pages with custom domain
  637. mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
  638. if mobj is not None:
  639. burl = unescapeHTML(mobj.group(1))
  640. # Don't set the extractor because it can be a track url or an album
  641. return self.url_result(burl)
  642. # Look for embedded Vevo player
  643. mobj = re.search(
  644. r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
  645. if mobj is not None:
  646. return self.url_result(mobj.group('url'))
  647. # Look for Ooyala videos
  648. mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
  649. re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
  650. if mobj is not None:
  651. return OoyalaIE._build_url_result(mobj.group('ec'))
  652. # Look for Aparat videos
  653. mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
  654. if mobj is not None:
  655. return self.url_result(mobj.group(1), 'Aparat')
  656. # Look for MPORA videos
  657. mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
  658. if mobj is not None:
  659. return self.url_result(mobj.group(1), 'Mpora')
  660. # Look for embedded NovaMov-based player
  661. mobj = re.search(
  662. r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
  663. (?P<url>http://(?:(?:embed|www)\.)?
  664. (?:novamov\.com|
  665. nowvideo\.(?:ch|sx|eu|at|ag|co)|
  666. videoweed\.(?:es|com)|
  667. movshare\.(?:net|sx|ag)|
  668. divxstage\.(?:eu|net|ch|co|at|ag))
  669. /embed\.php.+?)\1''', webpage)
  670. if mobj is not None:
  671. return self.url_result(mobj.group('url'))
  672. # Look for embedded Facebook player
  673. mobj = re.search(
  674. r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
  675. if mobj is not None:
  676. return self.url_result(mobj.group('url'), 'Facebook')
  677. # Look for embedded VK player
  678. mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
  679. if mobj is not None:
  680. return self.url_result(mobj.group('url'), 'VK')
  681. # Look for embedded ivi player
  682. mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
  683. if mobj is not None:
  684. return self.url_result(mobj.group('url'), 'Ivi')
  685. # Look for embedded Huffington Post player
  686. mobj = re.search(
  687. r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
  688. if mobj is not None:
  689. return self.url_result(mobj.group('url'), 'HuffPost')
  690. # Look for embed.ly
  691. mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
  692. if mobj is not None:
  693. return self.url_result(mobj.group('url'))
  694. mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
  695. if mobj is not None:
  696. return self.url_result(compat_urllib_parse.unquote(mobj.group('url')))
  697. # Look for funnyordie embed
  698. matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
  699. if matches:
  700. return _playlist_from_matches(
  701. matches, getter=unescapeHTML, ie='FunnyOrDie')
  702. # Look for embedded RUTV player
  703. rutv_url = RUTVIE._extract_url(webpage)
  704. if rutv_url:
  705. return self.url_result(rutv_url, 'RUTV')
  706. # Look for embedded TED player
  707. mobj = re.search(
  708. r'<iframe[^>]+?src=(["\'])(?P<url>http://embed\.ted\.com/.+?)\1', webpage)
  709. if mobj is not None:
  710. return self.url_result(mobj.group('url'), 'TED')
  711. # Look for embedded Ustream videos
  712. mobj = re.search(
  713. r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
  714. if mobj is not None:
  715. return self.url_result(mobj.group('url'), 'Ustream')
  716. # Look for embedded arte.tv player
  717. mobj = re.search(
  718. r'<script [^>]*?src="(?P<url>http://www\.arte\.tv/playerv2/embed[^"]+)"',
  719. webpage)
  720. if mobj is not None:
  721. return self.url_result(mobj.group('url'), 'ArteTVEmbed')
  722. # Look for embedded smotri.com player
  723. smotri_url = SmotriIE._extract_url(webpage)
  724. if smotri_url:
  725. return self.url_result(smotri_url, 'Smotri')
  726. # Look for embeded soundcloud player
  727. mobj = re.search(
  728. r'<iframe src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
  729. webpage)
  730. if mobj is not None:
  731. url = unescapeHTML(mobj.group('url'))
  732. return self.url_result(url)
  733. # Look for embedded vulture.com player
  734. mobj = re.search(
  735. r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
  736. webpage)
  737. if mobj is not None:
  738. url = unescapeHTML(mobj.group('url'))
  739. return self.url_result(url, ie='Vulture')
  740. # Look for embedded mtvservices player
  741. mobj = re.search(
  742. r'<iframe src="(?P<url>https?://media\.mtvnservices\.com/embed/[^"]+)"',
  743. webpage)
  744. if mobj is not None:
  745. url = unescapeHTML(mobj.group('url'))
  746. return self.url_result(url, ie='MTVServicesEmbedded')
  747. # Look for embedded yahoo player
  748. mobj = re.search(
  749. r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
  750. webpage)
  751. if mobj is not None:
  752. return self.url_result(mobj.group('url'), 'Yahoo')
  753. # Look for embedded sbs.com.au player
  754. mobj = re.search(
  755. r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)sbs\.com\.au/ondemand/video/single/.+?)\1',
  756. webpage)
  757. if mobj is not None:
  758. return self.url_result(mobj.group('url'), 'SBS')
  759. mobj = re.search(
  760. r'<iframe[^>]+?src=(["\'])(?P<url>https?://m\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
  761. webpage)
  762. if mobj is not None:
  763. return self.url_result(mobj.group('url'), 'MLB')
  764. # Start with something easy: JW Player in SWFObject
  765. found = re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
  766. if not found:
  767. # Look for gorilla-vid style embedding
  768. found = re.findall(r'''(?sx)
  769. (?:
  770. jw_plugins|
  771. JWPlayerOptions|
  772. jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
  773. )
  774. .*?file\s*:\s*["\'](.*?)["\']''', webpage)
  775. if not found:
  776. # Broaden the search a little bit
  777. found = re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
  778. if not found:
  779. # Broaden the findall a little bit: JWPlayer JS loader
  780. found = re.findall(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage)
  781. if not found:
  782. # Flow player
  783. found = re.findall(r'''(?xs)
  784. flowplayer\("[^"]+",\s*
  785. \{[^}]+?\}\s*,
  786. \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
  787. ["']?url["']?\s*:\s*["']([^"']+)["']
  788. ''', webpage)
  789. if not found:
  790. # Try to find twitter cards info
  791. found = re.findall(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
  792. if not found:
  793. # We look for Open Graph info:
  794. # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
  795. m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
  796. # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
  797. if m_video_type is not None:
  798. def check_video(vurl):
  799. vpath = compat_urlparse.urlparse(vurl).path
  800. vext = determine_ext(vpath)
  801. return '.' in vpath and vext not in ('swf', 'png', 'jpg')
  802. found = list(filter(
  803. check_video,
  804. re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)))
  805. if not found:
  806. # HTML5 video
  807. found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]+)? src="([^"]+)"', webpage)
  808. if not found:
  809. found = re.search(
  810. r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
  811. r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};url=\'?([^\'"]+)',
  812. webpage)
  813. if found:
  814. new_url = found.group(1)
  815. self.report_following_redirect(new_url)
  816. return {
  817. '_type': 'url',
  818. 'url': new_url,
  819. }
  820. if not found:
  821. raise ExtractorError('Unsupported URL: %s' % url)
  822. entries = []
  823. for video_url in found:
  824. video_url = compat_urlparse.urljoin(url, video_url)
  825. video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
  826. # Sometimes, jwplayer extraction will result in a YouTube URL
  827. if YoutubeIE.suitable(video_url):
  828. entries.append(self.url_result(video_url, 'Youtube'))
  829. continue
  830. # here's a fun little line of code for you:
  831. video_id = os.path.splitext(video_id)[0]
  832. entries.append({
  833. 'id': video_id,
  834. 'url': video_url,
  835. 'uploader': video_uploader,
  836. 'title': video_title,
  837. 'age_limit': age_limit,
  838. })
  839. if len(entries) == 1:
  840. return entries[0]
  841. else:
  842. for num, e in enumerate(entries, start=1):
  843. e['title'] = '%s (%d)' % (e['title'], num)
  844. return {
  845. '_type': 'playlist',
  846. 'entries': entries,
  847. }