You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1091 lines
43 KiB

10 years ago
11 years ago
11 years ago
11 years ago
10 years ago
10 years ago
  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import os
  4. import re
  5. from .common import InfoExtractor
  6. from .youtube import YoutubeIE
  7. from ..compat import (
  8. compat_urllib_parse,
  9. compat_urlparse,
  10. compat_xml_parse_error,
  11. )
  12. from ..utils import (
  13. determine_ext,
  14. ExtractorError,
  15. float_or_none,
  16. HEADRequest,
  17. orderedSet,
  18. parse_xml,
  19. smuggle_url,
  20. unescapeHTML,
  21. unified_strdate,
  22. unsmuggle_url,
  23. url_basename,
  24. )
  25. from .brightcove import BrightcoveIE
  26. from .ooyala import OoyalaIE
  27. from .rutv import RUTVIE
  28. from .smotri import SmotriIE
  29. from .condenast import CondeNastIE
  30. class GenericIE(InfoExtractor):
  31. IE_DESC = 'Generic downloader that works on some sites'
  32. _VALID_URL = r'.*'
  33. IE_NAME = 'generic'
  34. _TESTS = [
  35. {
  36. 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
  37. 'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
  38. 'info_dict': {
  39. 'id': '13601338388002',
  40. 'ext': 'mp4',
  41. 'uploader': 'www.hodiho.fr',
  42. 'title': 'R\u00e9gis plante sa Jeep',
  43. }
  44. },
  45. # bandcamp page with custom domain
  46. {
  47. 'add_ie': ['Bandcamp'],
  48. 'url': 'http://bronyrock.com/track/the-pony-mash',
  49. 'info_dict': {
  50. 'id': '3235767654',
  51. 'ext': 'mp3',
  52. 'title': 'The Pony Mash',
  53. 'uploader': 'M_Pallante',
  54. },
  55. 'skip': 'There is a limit of 200 free downloads / month for the test song',
  56. },
  57. # embedded brightcove video
  58. # it also tests brightcove videos that need to set the 'Referer' in the
  59. # http requests
  60. {
  61. 'add_ie': ['Brightcove'],
  62. 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
  63. 'info_dict': {
  64. 'id': '2765128793001',
  65. 'ext': 'mp4',
  66. 'title': 'Le cours de bourse : l’analyse technique',
  67. 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
  68. 'uploader': 'BFM BUSINESS',
  69. },
  70. 'params': {
  71. 'skip_download': True,
  72. },
  73. },
  74. {
  75. # https://github.com/rg3/youtube-dl/issues/2253
  76. 'url': 'http://bcove.me/i6nfkrc3',
  77. 'md5': '0ba9446db037002366bab3b3eb30c88c',
  78. 'info_dict': {
  79. 'id': '3101154703001',
  80. 'ext': 'mp4',
  81. 'title': 'Still no power',
  82. 'uploader': 'thestar.com',
  83. 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
  84. },
  85. 'add_ie': ['Brightcove'],
  86. },
  87. {
  88. 'url': 'http://www.championat.com/video/football/v/87/87499.html',
  89. 'md5': 'fb973ecf6e4a78a67453647444222983',
  90. 'info_dict': {
  91. 'id': '3414141473001',
  92. 'ext': 'mp4',
  93. 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
  94. 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
  95. 'uploader': 'Championat',
  96. },
  97. },
  98. {
  99. # https://github.com/rg3/youtube-dl/issues/3541
  100. 'add_ie': ['Brightcove'],
  101. 'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
  102. 'info_dict': {
  103. 'id': '3866516442001',
  104. 'ext': 'mp4',
  105. 'title': 'Leer mij vrouwen kennen: Aflevering 1',
  106. 'description': 'Leer mij vrouwen kennen: Aflevering 1',
  107. 'uploader': 'SBS Broadcasting',
  108. },
  109. 'skip': 'Restricted to Netherlands',
  110. 'params': {
  111. 'skip_download': True, # m3u8 download
  112. },
  113. },
  114. # Direct link to a video
  115. {
  116. 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
  117. 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
  118. 'info_dict': {
  119. 'id': 'trailer',
  120. 'ext': 'mp4',
  121. 'title': 'trailer',
  122. 'upload_date': '20100513',
  123. }
  124. },
  125. # ooyala video
  126. {
  127. 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
  128. 'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
  129. 'info_dict': {
  130. 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
  131. 'ext': 'mp4',
  132. 'title': '2cc213299525360.mov', # that's what we get
  133. },
  134. },
  135. # google redirect
  136. {
  137. 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
  138. 'info_dict': {
  139. 'id': 'cmQHVoWB5FY',
  140. 'ext': 'mp4',
  141. 'upload_date': '20130224',
  142. 'uploader_id': 'TheVerge',
  143. 'description': 'Chris Ziegler takes a look at the Alcatel OneTouch Fire and the ZTE Open; two of the first Firefox OS handsets to be officially announced.',
  144. 'uploader': 'The Verge',
  145. 'title': 'First Firefox OS phones side-by-side',
  146. },
  147. 'params': {
  148. 'skip_download': False,
  149. }
  150. },
  151. # embed.ly video
  152. {
  153. 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
  154. 'info_dict': {
  155. 'id': '9ODmcdjQcHQ',
  156. 'ext': 'mp4',
  157. 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
  158. 'upload_date': '20140225',
  159. 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
  160. 'uploader': 'Tested',
  161. 'uploader_id': 'testedcom',
  162. },
  163. # No need to test YoutubeIE here
  164. 'params': {
  165. 'skip_download': True,
  166. },
  167. },
  168. # funnyordie embed
  169. {
  170. 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
  171. 'info_dict': {
  172. 'id': '18e820ec3f',
  173. 'ext': 'mp4',
  174. 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
  175. 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
  176. },
  177. },
  178. # RUTV embed
  179. {
  180. 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
  181. 'info_dict': {
  182. 'id': '776940',
  183. 'ext': 'mp4',
  184. 'title': 'Охотское море стало целиком российским',
  185. 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
  186. },
  187. 'params': {
  188. # m3u8 download
  189. 'skip_download': True,
  190. },
  191. },
  192. # Embedded TED video
  193. {
  194. 'url': 'http://en.support.wordpress.com/videos/ted-talks/',
  195. 'md5': '65fdff94098e4a607385a60c5177c638',
  196. 'info_dict': {
  197. 'id': '1969',
  198. 'ext': 'mp4',
  199. 'title': 'Hidden miracles of the natural world',
  200. 'uploader': 'Louie Schwartzberg',
  201. 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
  202. }
  203. },
  204. # Embeded Ustream video
  205. {
  206. 'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
  207. 'md5': '27b99cdb639c9b12a79bca876a073417',
  208. 'info_dict': {
  209. 'id': '45734260',
  210. 'ext': 'flv',
  211. 'uploader': 'AU SPA: The NSA and Privacy',
  212. 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
  213. }
  214. },
  215. # nowvideo embed hidden behind percent encoding
  216. {
  217. 'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
  218. 'md5': '2baf4ddd70f697d94b1c18cf796d5107',
  219. 'info_dict': {
  220. 'id': '06e53103ca9aa',
  221. 'ext': 'flv',
  222. 'title': 'Macross Episode 001 Watch Macross Episode 001 onl',
  223. 'description': 'No description',
  224. },
  225. },
  226. # arte embed
  227. {
  228. 'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
  229. 'md5': '7653032cbb25bf6c80d80f217055fa43',
  230. 'info_dict': {
  231. 'id': '048195-004_PLUS7-F',
  232. 'ext': 'flv',
  233. 'title': 'X:enius',
  234. 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
  235. 'upload_date': '20140320',
  236. },
  237. 'params': {
  238. 'skip_download': 'Requires rtmpdump'
  239. }
  240. },
  241. # Condé Nast embed
  242. {
  243. 'url': 'http://www.wired.com/2014/04/honda-asimo/',
  244. 'md5': 'ba0dfe966fa007657bd1443ee672db0f',
  245. 'info_dict': {
  246. 'id': '53501be369702d3275860000',
  247. 'ext': 'mp4',
  248. 'title': 'Honda’s New Asimo Robot Is More Human Than Ever',
  249. }
  250. },
  251. # Dailymotion embed
  252. {
  253. 'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
  254. 'md5': '441aeeb82eb72c422c7f14ec533999cd',
  255. 'info_dict': {
  256. 'id': 'k2mm4bCdJ6CQ2i7c8o2',
  257. 'ext': 'mp4',
  258. 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
  259. 'uploader': 'Spi0n',
  260. },
  261. 'add_ie': ['Dailymotion'],
  262. },
  263. # YouTube embed
  264. {
  265. 'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
  266. 'info_dict': {
  267. 'id': 'FXRb4ykk4S0',
  268. 'ext': 'mp4',
  269. 'title': 'The NBL Auction 2014',
  270. 'uploader': 'BADMINTON England',
  271. 'uploader_id': 'BADMINTONEvents',
  272. 'upload_date': '20140603',
  273. 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
  274. },
  275. 'add_ie': ['Youtube'],
  276. 'params': {
  277. 'skip_download': True,
  278. }
  279. },
  280. # MTVSercices embed
  281. {
  282. 'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
  283. 'md5': '35727f82f58c76d996fc188f9755b0d5',
  284. 'info_dict': {
  285. 'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
  286. 'ext': 'mp4',
  287. 'title': 'Review',
  288. 'description': 'Mario\'s life in the fast lane has never looked so good.',
  289. },
  290. },
  291. # YouTube embed via <data-embed-url="">
  292. {
  293. 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
  294. 'info_dict': {
  295. 'id': '4vAffPZIT44',
  296. 'ext': 'mp4',
  297. 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
  298. 'uploader': 'Gameloft',
  299. 'uploader_id': 'gameloft',
  300. 'upload_date': '20140828',
  301. 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
  302. },
  303. 'params': {
  304. 'skip_download': True,
  305. }
  306. },
  307. # Camtasia studio
  308. {
  309. 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
  310. 'playlist': [{
  311. 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
  312. 'info_dict': {
  313. 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
  314. 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
  315. 'ext': 'flv',
  316. 'duration': 2235.90,
  317. }
  318. }, {
  319. 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
  320. 'info_dict': {
  321. 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
  322. 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
  323. 'ext': 'flv',
  324. 'duration': 2235.93,
  325. }
  326. }],
  327. 'info_dict': {
  328. 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
  329. }
  330. },
  331. # Flowplayer
  332. {
  333. 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
  334. 'md5': '9d65602bf31c6e20014319c7d07fba27',
  335. 'info_dict': {
  336. 'id': '5123ea6d5e5a7',
  337. 'ext': 'mp4',
  338. 'age_limit': 18,
  339. 'uploader': 'www.handjobhub.com',
  340. 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
  341. }
  342. },
  343. # RSS feed
  344. {
  345. 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
  346. 'info_dict': {
  347. 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
  348. 'title': 'Zero Punctuation',
  349. 'description': 're:'
  350. },
  351. 'playlist_mincount': 11,
  352. },
  353. # Multiple brightcove videos
  354. # https://github.com/rg3/youtube-dl/issues/2283
  355. {
  356. 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
  357. 'info_dict': {
  358. 'id': 'always-never',
  359. 'title': 'Always / Never - The New Yorker',
  360. },
  361. 'playlist_count': 3,
  362. 'params': {
  363. 'extract_flat': False,
  364. 'skip_download': True,
  365. }
  366. },
  367. # MLB embed
  368. {
  369. 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
  370. 'md5': '96f09a37e44da40dd083e12d9a683327',
  371. 'info_dict': {
  372. 'id': '33322633',
  373. 'ext': 'mp4',
  374. 'title': 'Ump changes call to ball',
  375. 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
  376. 'duration': 48,
  377. 'timestamp': 1401537900,
  378. 'upload_date': '20140531',
  379. 'thumbnail': 're:^https?://.*\.jpg$',
  380. },
  381. },
  382. # Wistia embed
  383. {
  384. 'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
  385. 'md5': '8788b683c777a5cf25621eaf286d0c23',
  386. 'info_dict': {
  387. 'id': '1cfaf6b7ea',
  388. 'ext': 'mov',
  389. 'title': 'md5:51364a8d3d009997ba99656004b5e20d',
  390. 'duration': 643.0,
  391. 'filesize': 182808282,
  392. 'uploader': 'education-portal.com',
  393. },
  394. },
  395. {
  396. 'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
  397. 'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
  398. 'info_dict': {
  399. 'id': 'uxjb0lwrcz',
  400. 'ext': 'mp4',
  401. 'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
  402. 'duration': 1715.0,
  403. 'uploader': 'thoughtworks.wistia.com',
  404. },
  405. },
  406. # Direct download with broken HEAD
  407. {
  408. 'url': 'http://ai-radio.org:8000/radio.opus',
  409. 'info_dict': {
  410. 'id': 'radio',
  411. 'ext': 'opus',
  412. 'title': 'radio',
  413. },
  414. 'params': {
  415. 'skip_download': True, # infinite live stream
  416. },
  417. 'expected_warnings': [
  418. r'501.*Not Implemented'
  419. ],
  420. },
  421. # Soundcloud embed
  422. {
  423. 'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
  424. 'info_dict': {
  425. 'id': '174391317',
  426. 'ext': 'mp3',
  427. 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
  428. 'uploader': 'Sophos Security',
  429. 'title': 'Chet Chat 171 - Oct 29, 2014',
  430. 'upload_date': '20141029',
  431. }
  432. },
  433. # Livestream embed
  434. {
  435. 'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
  436. 'info_dict': {
  437. 'id': '67864563',
  438. 'ext': 'flv',
  439. 'upload_date': '20141112',
  440. 'title': 'Rosetta #CometLanding webcast HL 10',
  441. }
  442. },
  443. # LazyYT
  444. {
  445. 'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
  446. 'info_dict': {
  447. 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
  448. },
  449. 'playlist_mincount': 2,
  450. },
  451. # Direct link with incorrect MIME type
  452. {
  453. 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
  454. 'md5': '4ccbebe5f36706d85221f204d7eb5913',
  455. 'info_dict': {
  456. 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
  457. 'id': '5_Lennart_Poettering_-_Systemd',
  458. 'ext': 'webm',
  459. 'title': '5_Lennart_Poettering_-_Systemd',
  460. 'upload_date': '20141120',
  461. },
  462. 'expected_warnings': [
  463. 'URL could be a direct video link, returning it as such.'
  464. ]
  465. },
  466. # Cinchcast embed
  467. {
  468. 'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
  469. 'info_dict': {
  470. 'id': '7141703',
  471. 'ext': 'mp3',
  472. 'upload_date': '20141126',
  473. 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
  474. }
  475. },
  476. ]
  477. def report_following_redirect(self, new_url):
  478. """Report information extraction."""
  479. self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
  480. def _extract_rss(self, url, video_id, doc):
  481. playlist_title = doc.find('./channel/title').text
  482. playlist_desc_el = doc.find('./channel/description')
  483. playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
  484. entries = [{
  485. '_type': 'url',
  486. 'url': e.find('link').text,
  487. 'title': e.find('title').text,
  488. } for e in doc.findall('./channel/item')]
  489. return {
  490. '_type': 'playlist',
  491. 'id': url,
  492. 'title': playlist_title,
  493. 'description': playlist_desc,
  494. 'entries': entries,
  495. }
  496. def _extract_camtasia(self, url, video_id, webpage):
  497. """ Returns None if no camtasia video can be found. """
  498. camtasia_cfg = self._search_regex(
  499. r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
  500. webpage, 'camtasia configuration file', default=None)
  501. if camtasia_cfg is None:
  502. return None
  503. title = self._html_search_meta('DC.title', webpage, fatal=True)
  504. camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
  505. camtasia_cfg = self._download_xml(
  506. camtasia_url, video_id,
  507. note='Downloading camtasia configuration',
  508. errnote='Failed to download camtasia configuration')
  509. fileset_node = camtasia_cfg.find('./playlist/array/fileset')
  510. entries = []
  511. for n in fileset_node.getchildren():
  512. url_n = n.find('./uri')
  513. if url_n is None:
  514. continue
  515. entries.append({
  516. 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
  517. 'title': '%s - %s' % (title, n.tag),
  518. 'url': compat_urlparse.urljoin(url, url_n.text),
  519. 'duration': float_or_none(n.find('./duration').text),
  520. })
  521. return {
  522. '_type': 'playlist',
  523. 'entries': entries,
  524. 'title': title,
  525. }
  526. def _real_extract(self, url):
  527. if url.startswith('//'):
  528. return {
  529. '_type': 'url',
  530. 'url': self.http_scheme() + url,
  531. }
  532. parsed_url = compat_urlparse.urlparse(url)
  533. if not parsed_url.scheme:
  534. default_search = self._downloader.params.get('default_search')
  535. if default_search is None:
  536. default_search = 'fixup_error'
  537. if default_search in ('auto', 'auto_warning', 'fixup_error'):
  538. if '/' in url:
  539. self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
  540. return self.url_result('http://' + url)
  541. elif default_search != 'fixup_error':
  542. if default_search == 'auto_warning':
  543. if re.match(r'^(?:url|URL)$', url):
  544. raise ExtractorError(
  545. 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url,
  546. expected=True)
  547. else:
  548. self._downloader.report_warning(
  549. 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url)
  550. return self.url_result('ytsearch:' + url)
  551. if default_search in ('error', 'fixup_error'):
  552. raise ExtractorError(
  553. '%r is not a valid URL. '
  554. 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
  555. % (url, url), expected=True)
  556. else:
  557. if ':' not in default_search:
  558. default_search += ':'
  559. return self.url_result(default_search + url)
  560. url, smuggled_data = unsmuggle_url(url)
  561. force_videoid = None
  562. is_intentional = smuggled_data and smuggled_data.get('to_generic')
  563. if smuggled_data and 'force_videoid' in smuggled_data:
  564. force_videoid = smuggled_data['force_videoid']
  565. video_id = force_videoid
  566. else:
  567. video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
  568. self.to_screen('%s: Requesting header' % video_id)
  569. head_req = HEADRequest(url)
  570. head_response = self._request_webpage(
  571. head_req, video_id,
  572. note=False, errnote='Could not send HEAD request to %s' % url,
  573. fatal=False)
  574. if head_response is not False:
  575. # Check for redirect
  576. new_url = head_response.geturl()
  577. if url != new_url:
  578. self.report_following_redirect(new_url)
  579. if force_videoid:
  580. new_url = smuggle_url(
  581. new_url, {'force_videoid': force_videoid})
  582. return self.url_result(new_url)
  583. full_response = None
  584. if head_response is False:
  585. full_response = self._request_webpage(url, video_id)
  586. head_response = full_response
  587. # Check for direct link to a video
  588. content_type = head_response.headers.get('Content-Type', '')
  589. m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
  590. if m:
  591. upload_date = unified_strdate(
  592. head_response.headers.get('Last-Modified'))
  593. return {
  594. 'id': video_id,
  595. 'title': os.path.splitext(url_basename(url))[0],
  596. 'direct': True,
  597. 'formats': [{
  598. 'format_id': m.group('format_id'),
  599. 'url': url,
  600. 'vcodec': 'none' if m.group('type') == 'audio' else None
  601. }],
  602. 'upload_date': upload_date,
  603. }
  604. if not self._downloader.params.get('test', False) and not is_intentional:
  605. self._downloader.report_warning('Falling back on generic information extractor.')
  606. if not full_response:
  607. full_response = self._request_webpage(url, video_id)
  608. # Maybe it's a direct link to a video?
  609. # Be careful not to download the whole thing!
  610. first_bytes = full_response.read(512)
  611. if not re.match(r'^\s*<', first_bytes.decode('utf-8', 'replace')):
  612. self._downloader.report_warning(
  613. 'URL could be a direct video link, returning it as such.')
  614. upload_date = unified_strdate(
  615. head_response.headers.get('Last-Modified'))
  616. return {
  617. 'id': video_id,
  618. 'title': os.path.splitext(url_basename(url))[0],
  619. 'direct': True,
  620. 'url': url,
  621. 'upload_date': upload_date,
  622. }
  623. webpage = self._webpage_read_content(
  624. full_response, url, video_id, prefix=first_bytes)
  625. self.report_extraction(video_id)
  626. # Is it an RSS feed?
  627. try:
  628. doc = parse_xml(webpage)
  629. if doc.tag == 'rss':
  630. return self._extract_rss(url, video_id, doc)
  631. except compat_xml_parse_error:
  632. pass
  633. # Is it a Camtasia project?
  634. camtasia_res = self._extract_camtasia(url, video_id, webpage)
  635. if camtasia_res is not None:
  636. return camtasia_res
  637. # Sometimes embedded video player is hidden behind percent encoding
  638. # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
  639. # Unescaping the whole page allows to handle those cases in a generic way
  640. webpage = compat_urllib_parse.unquote(webpage)
  641. # it's tempting to parse this further, but you would
  642. # have to take into account all the variations like
  643. # Video Title - Site Name
  644. # Site Name | Video Title
  645. # Video Title - Tagline | Site Name
  646. # and so on and so forth; it's just not practical
  647. video_title = self._html_search_regex(
  648. r'(?s)<title>(.*?)</title>', webpage, 'video title',
  649. default='video')
  650. # Try to detect age limit automatically
  651. age_limit = self._rta_search(webpage)
  652. # And then there are the jokers who advertise that they use RTA,
  653. # but actually don't.
  654. AGE_LIMIT_MARKERS = [
  655. r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
  656. ]
  657. if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
  658. age_limit = 18
  659. # video uploader is domain name
  660. video_uploader = self._search_regex(
  661. r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
  662. # Helper method
  663. def _playlist_from_matches(matches, getter, ie=None):
  664. urlrs = orderedSet(
  665. self.url_result(self._proto_relative_url(getter(m)), ie)
  666. for m in matches)
  667. return self.playlist_result(
  668. urlrs, playlist_id=video_id, playlist_title=video_title)
  669. # Look for BrightCove:
  670. bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
  671. if bc_urls:
  672. self.to_screen('Brightcove video detected.')
  673. entries = [{
  674. '_type': 'url',
  675. 'url': smuggle_url(bc_url, {'Referer': url}),
  676. 'ie_key': 'Brightcove'
  677. } for bc_url in bc_urls]
  678. return {
  679. '_type': 'playlist',
  680. 'title': video_title,
  681. 'id': video_id,
  682. 'entries': entries,
  683. }
  684. # Look for embedded (iframe) Vimeo player
  685. mobj = re.search(
  686. r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
  687. if mobj:
  688. player_url = unescapeHTML(mobj.group('url'))
  689. surl = smuggle_url(player_url, {'Referer': url})
  690. return self.url_result(surl)
  691. # Look for embedded (swf embed) Vimeo player
  692. mobj = re.search(
  693. r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
  694. if mobj:
  695. return self.url_result(mobj.group(1))
  696. # Look for embedded YouTube player
  697. matches = re.findall(r'''(?x)
  698. (?:
  699. <iframe[^>]+?src=|
  700. data-video-url=|
  701. <embed[^>]+?src=|
  702. embedSWF\(?:\s*|
  703. new\s+SWFObject\(
  704. )
  705. (["\'])
  706. (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
  707. (?:embed|v|p)/.+?)
  708. \1''', webpage)
  709. if matches:
  710. return _playlist_from_matches(
  711. matches, lambda m: unescapeHTML(m[1]))
  712. # Look for lazyYT YouTube embed
  713. matches = re.findall(
  714. r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
  715. if matches:
  716. return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
  717. # Look for embedded Dailymotion player
  718. matches = re.findall(
  719. r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
  720. if matches:
  721. return _playlist_from_matches(
  722. matches, lambda m: unescapeHTML(m[1]))
  723. # Look for embedded Dailymotion playlist player (#3822)
  724. m = re.search(
  725. r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
  726. if m:
  727. playlists = re.findall(
  728. r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
  729. if playlists:
  730. return _playlist_from_matches(
  731. playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
  732. # Look for embedded Wistia player
  733. match = re.search(
  734. r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
  735. if match:
  736. embed_url = self._proto_relative_url(
  737. unescapeHTML(match.group('url')))
  738. return {
  739. '_type': 'url_transparent',
  740. 'url': embed_url,
  741. 'ie_key': 'Wistia',
  742. 'uploader': video_uploader,
  743. 'title': video_title,
  744. 'id': video_id,
  745. }
  746. match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
  747. if match:
  748. return {
  749. '_type': 'url_transparent',
  750. 'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
  751. 'ie_key': 'Wistia',
  752. 'uploader': video_uploader,
  753. 'title': video_title,
  754. 'id': match.group('id')
  755. }
  756. # Look for embedded blip.tv player
  757. mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
  758. if mobj:
  759. return self.url_result('http://blip.tv/a/a-' + mobj.group(1), 'BlipTV')
  760. mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
  761. if mobj:
  762. return self.url_result(mobj.group(1), 'BlipTV')
  763. # Look for embedded condenast player
  764. matches = re.findall(
  765. r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
  766. webpage)
  767. if matches:
  768. return {
  769. '_type': 'playlist',
  770. 'entries': [{
  771. '_type': 'url',
  772. 'ie_key': 'CondeNast',
  773. 'url': ma,
  774. } for ma in matches],
  775. 'title': video_title,
  776. 'id': video_id,
  777. }
  778. # Look for Bandcamp pages with custom domain
  779. mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
  780. if mobj is not None:
  781. burl = unescapeHTML(mobj.group(1))
  782. # Don't set the extractor because it can be a track url or an album
  783. return self.url_result(burl)
  784. # Look for embedded Vevo player
  785. mobj = re.search(
  786. r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
  787. if mobj is not None:
  788. return self.url_result(mobj.group('url'))
  789. # Look for Ooyala videos
  790. mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
  791. re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
  792. if mobj is not None:
  793. return OoyalaIE._build_url_result(mobj.group('ec'))
  794. # Look for Aparat videos
  795. mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
  796. if mobj is not None:
  797. return self.url_result(mobj.group(1), 'Aparat')
  798. # Look for MPORA videos
  799. mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
  800. if mobj is not None:
  801. return self.url_result(mobj.group(1), 'Mpora')
  802. # Look for embedded NovaMov-based player
  803. mobj = re.search(
  804. r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
  805. (?P<url>http://(?:(?:embed|www)\.)?
  806. (?:novamov\.com|
  807. nowvideo\.(?:ch|sx|eu|at|ag|co)|
  808. videoweed\.(?:es|com)|
  809. movshare\.(?:net|sx|ag)|
  810. divxstage\.(?:eu|net|ch|co|at|ag))
  811. /embed\.php.+?)\1''', webpage)
  812. if mobj is not None:
  813. return self.url_result(mobj.group('url'))
  814. # Look for embedded Facebook player
  815. mobj = re.search(
  816. r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
  817. if mobj is not None:
  818. return self.url_result(mobj.group('url'), 'Facebook')
  819. # Look for embedded VK player
  820. mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
  821. if mobj is not None:
  822. return self.url_result(mobj.group('url'), 'VK')
  823. # Look for embedded ivi player
  824. mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
  825. if mobj is not None:
  826. return self.url_result(mobj.group('url'), 'Ivi')
  827. # Look for embedded Huffington Post player
  828. mobj = re.search(
  829. r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
  830. if mobj is not None:
  831. return self.url_result(mobj.group('url'), 'HuffPost')
  832. # Look for embed.ly
  833. mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
  834. if mobj is not None:
  835. return self.url_result(mobj.group('url'))
  836. mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
  837. if mobj is not None:
  838. return self.url_result(compat_urllib_parse.unquote(mobj.group('url')))
  839. # Look for funnyordie embed
  840. matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
  841. if matches:
  842. return _playlist_from_matches(
  843. matches, getter=unescapeHTML, ie='FunnyOrDie')
  844. # Look for embedded RUTV player
  845. rutv_url = RUTVIE._extract_url(webpage)
  846. if rutv_url:
  847. return self.url_result(rutv_url, 'RUTV')
  848. # Look for embedded TED player
  849. mobj = re.search(
  850. r'<iframe[^>]+?src=(["\'])(?P<url>http://embed\.ted\.com/.+?)\1', webpage)
  851. if mobj is not None:
  852. return self.url_result(mobj.group('url'), 'TED')
  853. # Look for embedded Ustream videos
  854. mobj = re.search(
  855. r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
  856. if mobj is not None:
  857. return self.url_result(mobj.group('url'), 'Ustream')
  858. # Look for embedded arte.tv player
  859. mobj = re.search(
  860. r'<script [^>]*?src="(?P<url>http://www\.arte\.tv/playerv2/embed[^"]+)"',
  861. webpage)
  862. if mobj is not None:
  863. return self.url_result(mobj.group('url'), 'ArteTVEmbed')
  864. # Look for embedded smotri.com player
  865. smotri_url = SmotriIE._extract_url(webpage)
  866. if smotri_url:
  867. return self.url_result(smotri_url, 'Smotri')
  868. # Look for embeded soundcloud player
  869. mobj = re.search(
  870. r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
  871. webpage)
  872. if mobj is not None:
  873. url = unescapeHTML(mobj.group('url'))
  874. return self.url_result(url)
  875. # Look for embedded vulture.com player
  876. mobj = re.search(
  877. r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
  878. webpage)
  879. if mobj is not None:
  880. url = unescapeHTML(mobj.group('url'))
  881. return self.url_result(url, ie='Vulture')
  882. # Look for embedded mtvservices player
  883. mobj = re.search(
  884. r'<iframe src="(?P<url>https?://media\.mtvnservices\.com/embed/[^"]+)"',
  885. webpage)
  886. if mobj is not None:
  887. url = unescapeHTML(mobj.group('url'))
  888. return self.url_result(url, ie='MTVServicesEmbedded')
  889. # Look for embedded yahoo player
  890. mobj = re.search(
  891. r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
  892. webpage)
  893. if mobj is not None:
  894. return self.url_result(mobj.group('url'), 'Yahoo')
  895. # Look for embedded sbs.com.au player
  896. mobj = re.search(
  897. r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)sbs\.com\.au/ondemand/video/single/.+?)\1',
  898. webpage)
  899. if mobj is not None:
  900. return self.url_result(mobj.group('url'), 'SBS')
  901. # Look for embedded Cinchcast player
  902. mobj = re.search(
  903. r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
  904. webpage)
  905. if mobj is not None:
  906. return self.url_result(mobj.group('url'), 'Cinchcast')
  907. mobj = re.search(
  908. r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
  909. webpage)
  910. if mobj is not None:
  911. return self.url_result(mobj.group('url'), 'MLB')
  912. mobj = re.search(
  913. r'<iframe[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
  914. webpage)
  915. if mobj is not None:
  916. return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
  917. mobj = re.search(
  918. r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
  919. webpage)
  920. if mobj is not None:
  921. return self.url_result(mobj.group('url'), 'Livestream')
  922. def check_video(vurl):
  923. vpath = compat_urlparse.urlparse(vurl).path
  924. vext = determine_ext(vpath)
  925. return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
  926. def filter_video(urls):
  927. return list(filter(check_video, urls))
  928. # Start with something easy: JW Player in SWFObject
  929. found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
  930. if not found:
  931. # Look for gorilla-vid style embedding
  932. found = filter_video(re.findall(r'''(?sx)
  933. (?:
  934. jw_plugins|
  935. JWPlayerOptions|
  936. jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
  937. )
  938. .*?file\s*:\s*["\'](.*?)["\']''', webpage))
  939. if not found:
  940. # Broaden the search a little bit
  941. found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
  942. if not found:
  943. # Broaden the findall a little bit: JWPlayer JS loader
  944. found = filter_video(re.findall(
  945. r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
  946. if not found:
  947. # Flow player
  948. found = filter_video(re.findall(r'''(?xs)
  949. flowplayer\("[^"]+",\s*
  950. \{[^}]+?\}\s*,
  951. \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
  952. ["']?url["']?\s*:\s*["']([^"']+)["']
  953. ''', webpage))
  954. if not found:
  955. # Try to find twitter cards info
  956. found = filter_video(re.findall(
  957. r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
  958. if not found:
  959. # We look for Open Graph info:
  960. # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
  961. m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
  962. # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
  963. if m_video_type is not None:
  964. found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
  965. if not found:
  966. # HTML5 video
  967. found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
  968. if not found:
  969. found = re.search(
  970. r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
  971. r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};url=\'?([^\'"]+)',
  972. webpage)
  973. if found:
  974. new_url = found.group(1)
  975. self.report_following_redirect(new_url)
  976. return {
  977. '_type': 'url',
  978. 'url': new_url,
  979. }
  980. if not found:
  981. raise ExtractorError('Unsupported URL: %s' % url)
  982. entries = []
  983. for video_url in found:
  984. video_url = compat_urlparse.urljoin(url, video_url)
  985. video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
  986. # Sometimes, jwplayer extraction will result in a YouTube URL
  987. if YoutubeIE.suitable(video_url):
  988. entries.append(self.url_result(video_url, 'Youtube'))
  989. continue
  990. # here's a fun little line of code for you:
  991. video_id = os.path.splitext(video_id)[0]
  992. entries.append({
  993. 'id': video_id,
  994. 'url': video_url,
  995. 'uploader': video_uploader,
  996. 'title': video_title,
  997. 'age_limit': age_limit,
  998. })
  999. if len(entries) == 1:
  1000. return entries[0]
  1001. else:
  1002. for num, e in enumerate(entries, start=1):
  1003. e['title'] = '%s (%d)' % (e['title'], num)
  1004. return {
  1005. '_type': 'playlist',
  1006. 'entries': entries,
  1007. }