You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1344 lines
58 KiB

8 years ago
10 years ago
8 years ago
9 years ago
8 years ago
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import itertools
  4. import re
  5. from .common import InfoExtractor
  6. from ..utils import (
  7. clean_html,
  8. dict_get,
  9. ExtractorError,
  10. float_or_none,
  11. get_element_by_class,
  12. int_or_none,
  13. js_to_json,
  14. parse_duration,
  15. parse_iso8601,
  16. try_get,
  17. unescapeHTML,
  18. url_or_none,
  19. urlencode_postdata,
  20. urljoin,
  21. )
  22. from ..compat import (
  23. compat_etree_Element,
  24. compat_HTTPError,
  25. compat_urlparse,
  26. )
  27. class BBCCoUkIE(InfoExtractor):
  28. IE_NAME = 'bbc.co.uk'
  29. IE_DESC = 'BBC iPlayer'
  30. _ID_REGEX = r'(?:[pbm][\da-z]{7}|w[\da-z]{7,14})'
  31. _VALID_URL = r'''(?x)
  32. https?://
  33. (?:www\.)?bbc\.co\.uk/
  34. (?:
  35. programmes/(?!articles/)|
  36. iplayer(?:/[^/]+)?/(?:episode/|playlist/)|
  37. music/(?:clips|audiovideo/popular)[/#]|
  38. radio/player/|
  39. events/[^/]+/play/[^/]+/
  40. )
  41. (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
  42. ''' % _ID_REGEX
  43. _LOGIN_URL = 'https://account.bbc.com/signin'
  44. _NETRC_MACHINE = 'bbc'
  45. _MEDIASELECTOR_URLS = [
  46. # Provides HQ HLS streams with even better quality that pc mediaset but fails
  47. # with geolocation in some cases when it's even not geo restricted at all (e.g.
  48. # http://www.bbc.co.uk/programmes/b06bp7lf). Also may fail with selectionunavailable.
  49. 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
  50. 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
  51. ]
  52. _MEDIASELECTION_NS = 'http://bbc.co.uk/2008/mp/mediaselection'
  53. _EMP_PLAYLIST_NS = 'http://bbc.co.uk/2008/emp/playlist'
  54. _NAMESPACES = (
  55. _MEDIASELECTION_NS,
  56. _EMP_PLAYLIST_NS,
  57. )
  58. _TESTS = [
  59. {
  60. 'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
  61. 'info_dict': {
  62. 'id': 'b039d07m',
  63. 'ext': 'flv',
  64. 'title': 'Leonard Cohen, Kaleidoscope - BBC Radio 4',
  65. 'description': 'The Canadian poet and songwriter reflects on his musical career.',
  66. },
  67. 'params': {
  68. # rtmp download
  69. 'skip_download': True,
  70. }
  71. },
  72. {
  73. 'url': 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/',
  74. 'info_dict': {
  75. 'id': 'b00yng1d',
  76. 'ext': 'flv',
  77. 'title': 'The Man in Black: Series 3: The Printed Name',
  78. 'description': "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey.",
  79. 'duration': 1800,
  80. },
  81. 'params': {
  82. # rtmp download
  83. 'skip_download': True,
  84. },
  85. 'skip': 'Episode is no longer available on BBC iPlayer Radio',
  86. },
  87. {
  88. 'url': 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/',
  89. 'info_dict': {
  90. 'id': 'b00yng1d',
  91. 'ext': 'flv',
  92. 'title': 'The Voice UK: Series 3: Blind Auditions 5',
  93. 'description': 'Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.',
  94. 'duration': 5100,
  95. },
  96. 'params': {
  97. # rtmp download
  98. 'skip_download': True,
  99. },
  100. 'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
  101. },
  102. {
  103. 'url': 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion',
  104. 'info_dict': {
  105. 'id': 'b03k3pb7',
  106. 'ext': 'flv',
  107. 'title': "Tomorrow's Worlds: The Unearthly History of Science Fiction",
  108. 'description': '2. Invasion',
  109. 'duration': 3600,
  110. },
  111. 'params': {
  112. # rtmp download
  113. 'skip_download': True,
  114. },
  115. 'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
  116. }, {
  117. 'url': 'http://www.bbc.co.uk/programmes/b04v20dw',
  118. 'info_dict': {
  119. 'id': 'b04v209v',
  120. 'ext': 'flv',
  121. 'title': 'Pete Tong, The Essential New Tune Special',
  122. 'description': "Pete has a very special mix - all of 2014's Essential New Tunes!",
  123. 'duration': 10800,
  124. },
  125. 'params': {
  126. # rtmp download
  127. 'skip_download': True,
  128. },
  129. 'skip': 'Episode is no longer available on BBC iPlayer Radio',
  130. }, {
  131. 'url': 'http://www.bbc.co.uk/music/clips/p022h44b',
  132. 'note': 'Audio',
  133. 'info_dict': {
  134. 'id': 'p022h44j',
  135. 'ext': 'flv',
  136. 'title': 'BBC Proms Music Guides, Rachmaninov: Symphonic Dances',
  137. 'description': "In this Proms Music Guide, Andrew McGregor looks at Rachmaninov's Symphonic Dances.",
  138. 'duration': 227,
  139. },
  140. 'params': {
  141. # rtmp download
  142. 'skip_download': True,
  143. }
  144. }, {
  145. 'url': 'http://www.bbc.co.uk/music/clips/p025c0zz',
  146. 'note': 'Video',
  147. 'info_dict': {
  148. 'id': 'p025c103',
  149. 'ext': 'flv',
  150. 'title': 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)',
  151. 'description': 'Rae Morris performs Closer for BBC Three at Reading 2014',
  152. 'duration': 226,
  153. },
  154. 'params': {
  155. # rtmp download
  156. 'skip_download': True,
  157. }
  158. }, {
  159. 'url': 'http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls',
  160. 'info_dict': {
  161. 'id': 'p02n76xf',
  162. 'ext': 'flv',
  163. 'title': 'Natural World, 2015-2016: 2. Super Powered Owls',
  164. 'description': 'md5:e4db5c937d0e95a7c6b5e654d429183d',
  165. 'duration': 3540,
  166. },
  167. 'params': {
  168. # rtmp download
  169. 'skip_download': True,
  170. },
  171. 'skip': 'geolocation',
  172. }, {
  173. 'url': 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition',
  174. 'info_dict': {
  175. 'id': 'b05zmgw1',
  176. 'ext': 'flv',
  177. 'description': 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.',
  178. 'title': 'Royal Academy Summer Exhibition',
  179. 'duration': 3540,
  180. },
  181. 'params': {
  182. # rtmp download
  183. 'skip_download': True,
  184. },
  185. 'skip': 'geolocation',
  186. }, {
  187. # iptv-all mediaset fails with geolocation however there is no geo restriction
  188. # for this programme at all
  189. 'url': 'http://www.bbc.co.uk/programmes/b06rkn85',
  190. 'info_dict': {
  191. 'id': 'b06rkms3',
  192. 'ext': 'flv',
  193. 'title': "Best of the Mini-Mixes 2015: Part 3, Annie Mac's Friday Night - BBC Radio 1",
  194. 'description': "Annie has part three in the Best of the Mini-Mixes 2015, plus the year's Most Played!",
  195. },
  196. 'params': {
  197. # rtmp download
  198. 'skip_download': True,
  199. },
  200. 'skip': 'Now it\'s really geo-restricted',
  201. }, {
  202. # compact player (https://github.com/ytdl-org/youtube-dl/issues/8147)
  203. 'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
  204. 'info_dict': {
  205. 'id': 'p028bfkj',
  206. 'ext': 'flv',
  207. 'title': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
  208. 'description': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
  209. },
  210. 'params': {
  211. # rtmp download
  212. 'skip_download': True,
  213. },
  214. }, {
  215. 'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
  216. 'only_matching': True,
  217. }, {
  218. 'url': 'http://www.bbc.co.uk/music/clips#p02frcc3',
  219. 'only_matching': True,
  220. }, {
  221. 'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo',
  222. 'only_matching': True,
  223. }, {
  224. 'url': 'http://www.bbc.co.uk/radio/player/p03cchwf',
  225. 'only_matching': True,
  226. }, {
  227. 'url': 'https://www.bbc.co.uk/music/audiovideo/popular#p055bc55',
  228. 'only_matching': True,
  229. }, {
  230. 'url': 'http://www.bbc.co.uk/programmes/w3csv1y9',
  231. 'only_matching': True,
  232. }, {
  233. 'url': 'https://www.bbc.co.uk/programmes/m00005xn',
  234. 'only_matching': True,
  235. }, {
  236. 'url': 'https://www.bbc.co.uk/programmes/w172w4dww1jqt5s',
  237. 'only_matching': True,
  238. }]
  239. _USP_RE = r'/([^/]+?)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'
  240. def _login(self):
  241. username, password = self._get_login_info()
  242. if username is None:
  243. return
  244. login_page = self._download_webpage(
  245. self._LOGIN_URL, None, 'Downloading signin page')
  246. login_form = self._hidden_inputs(login_page)
  247. login_form.update({
  248. 'username': username,
  249. 'password': password,
  250. })
  251. post_url = urljoin(self._LOGIN_URL, self._search_regex(
  252. r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
  253. 'post url', default=self._LOGIN_URL, group='url'))
  254. response, urlh = self._download_webpage_handle(
  255. post_url, None, 'Logging in', data=urlencode_postdata(login_form),
  256. headers={'Referer': self._LOGIN_URL})
  257. if self._LOGIN_URL in urlh.geturl():
  258. error = clean_html(get_element_by_class('form-message', response))
  259. if error:
  260. raise ExtractorError(
  261. 'Unable to login: %s' % error, expected=True)
  262. raise ExtractorError('Unable to log in')
  263. def _real_initialize(self):
  264. self._login()
  265. class MediaSelectionError(Exception):
  266. def __init__(self, id):
  267. self.id = id
  268. def _extract_asx_playlist(self, connection, programme_id):
  269. asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
  270. return [ref.get('href') for ref in asx.findall('./Entry/ref')]
  271. def _extract_items(self, playlist):
  272. return playlist.findall('./{%s}item' % self._EMP_PLAYLIST_NS)
  273. def _findall_ns(self, element, xpath):
  274. elements = []
  275. for ns in self._NAMESPACES:
  276. elements.extend(element.findall(xpath % ns))
  277. return elements
  278. def _extract_medias(self, media_selection):
  279. error = media_selection.find('./{%s}error' % self._MEDIASELECTION_NS)
  280. if error is None:
  281. media_selection.find('./{%s}error' % self._EMP_PLAYLIST_NS)
  282. if error is not None:
  283. raise BBCCoUkIE.MediaSelectionError(error.get('id'))
  284. return self._findall_ns(media_selection, './{%s}media')
  285. def _extract_connections(self, media):
  286. return self._findall_ns(media, './{%s}connection')
  287. def _get_subtitles(self, media, programme_id):
  288. subtitles = {}
  289. for connection in self._extract_connections(media):
  290. cc_url = url_or_none(connection.get('href'))
  291. if not cc_url:
  292. continue
  293. captions = self._download_xml(
  294. cc_url, programme_id, 'Downloading captions', fatal=False)
  295. if not isinstance(captions, compat_etree_Element):
  296. continue
  297. lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
  298. subtitles[lang] = [
  299. {
  300. 'url': connection.get('href'),
  301. 'ext': 'ttml',
  302. },
  303. ]
  304. return subtitles
  305. def _raise_extractor_error(self, media_selection_error):
  306. raise ExtractorError(
  307. '%s returned error: %s' % (self.IE_NAME, media_selection_error.id),
  308. expected=True)
  309. def _download_media_selector(self, programme_id):
  310. last_exception = None
  311. for mediaselector_url in self._MEDIASELECTOR_URLS:
  312. try:
  313. return self._download_media_selector_url(
  314. mediaselector_url % programme_id, programme_id)
  315. except BBCCoUkIE.MediaSelectionError as e:
  316. if e.id in ('notukerror', 'geolocation', 'selectionunavailable'):
  317. last_exception = e
  318. continue
  319. self._raise_extractor_error(e)
  320. self._raise_extractor_error(last_exception)
  321. def _download_media_selector_url(self, url, programme_id=None):
  322. media_selection = self._download_xml(
  323. url, programme_id, 'Downloading media selection XML',
  324. expected_status=(403, 404))
  325. return self._process_media_selector(media_selection, programme_id)
  326. def _process_media_selector(self, media_selection, programme_id):
  327. formats = []
  328. subtitles = None
  329. urls = []
  330. for media in self._extract_medias(media_selection):
  331. kind = media.get('kind')
  332. if kind in ('video', 'audio'):
  333. bitrate = int_or_none(media.get('bitrate'))
  334. encoding = media.get('encoding')
  335. service = media.get('service')
  336. width = int_or_none(media.get('width'))
  337. height = int_or_none(media.get('height'))
  338. file_size = int_or_none(media.get('media_file_size'))
  339. for connection in self._extract_connections(media):
  340. href = connection.get('href')
  341. if href in urls:
  342. continue
  343. if href:
  344. urls.append(href)
  345. conn_kind = connection.get('kind')
  346. protocol = connection.get('protocol')
  347. supplier = connection.get('supplier')
  348. transfer_format = connection.get('transferFormat')
  349. format_id = supplier or conn_kind or protocol
  350. if service:
  351. format_id = '%s_%s' % (service, format_id)
  352. # ASX playlist
  353. if supplier == 'asx':
  354. for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
  355. formats.append({
  356. 'url': ref,
  357. 'format_id': 'ref%s_%s' % (i, format_id),
  358. })
  359. elif transfer_format == 'dash':
  360. formats.extend(self._extract_mpd_formats(
  361. href, programme_id, mpd_id=format_id, fatal=False))
  362. elif transfer_format == 'hls':
  363. formats.extend(self._extract_m3u8_formats(
  364. href, programme_id, ext='mp4', entry_protocol='m3u8_native',
  365. m3u8_id=format_id, fatal=False))
  366. if re.search(self._USP_RE, href):
  367. usp_formats = self._extract_m3u8_formats(
  368. re.sub(self._USP_RE, r'/\1.ism/\1.m3u8', href),
  369. programme_id, ext='mp4', entry_protocol='m3u8_native',
  370. m3u8_id=format_id, fatal=False)
  371. for f in usp_formats:
  372. if f.get('height') and f['height'] > 720:
  373. continue
  374. formats.append(f)
  375. elif transfer_format == 'hds':
  376. formats.extend(self._extract_f4m_formats(
  377. href, programme_id, f4m_id=format_id, fatal=False))
  378. else:
  379. if not service and not supplier and bitrate:
  380. format_id += '-%d' % bitrate
  381. fmt = {
  382. 'format_id': format_id,
  383. 'filesize': file_size,
  384. }
  385. if kind == 'video':
  386. fmt.update({
  387. 'width': width,
  388. 'height': height,
  389. 'tbr': bitrate,
  390. 'vcodec': encoding,
  391. })
  392. else:
  393. fmt.update({
  394. 'abr': bitrate,
  395. 'acodec': encoding,
  396. 'vcodec': 'none',
  397. })
  398. if protocol in ('http', 'https'):
  399. # Direct link
  400. fmt.update({
  401. 'url': href,
  402. })
  403. elif protocol == 'rtmp':
  404. application = connection.get('application', 'ondemand')
  405. auth_string = connection.get('authString')
  406. identifier = connection.get('identifier')
  407. server = connection.get('server')
  408. fmt.update({
  409. 'url': '%s://%s/%s?%s' % (protocol, server, application, auth_string),
  410. 'play_path': identifier,
  411. 'app': '%s?%s' % (application, auth_string),
  412. 'page_url': 'http://www.bbc.co.uk',
  413. 'player_url': 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf',
  414. 'rtmp_live': False,
  415. 'ext': 'flv',
  416. })
  417. else:
  418. continue
  419. formats.append(fmt)
  420. elif kind == 'captions':
  421. subtitles = self.extract_subtitles(media, programme_id)
  422. return formats, subtitles
  423. def _download_playlist(self, playlist_id):
  424. try:
  425. playlist = self._download_json(
  426. 'http://www.bbc.co.uk/programmes/%s/playlist.json' % playlist_id,
  427. playlist_id, 'Downloading playlist JSON')
  428. version = playlist.get('defaultAvailableVersion')
  429. if version:
  430. smp_config = version['smpConfig']
  431. title = smp_config['title']
  432. description = smp_config['summary']
  433. for item in smp_config['items']:
  434. kind = item['kind']
  435. if kind not in ('programme', 'radioProgramme'):
  436. continue
  437. programme_id = item.get('vpid')
  438. duration = int_or_none(item.get('duration'))
  439. formats, subtitles = self._download_media_selector(programme_id)
  440. return programme_id, title, description, duration, formats, subtitles
  441. except ExtractorError as ee:
  442. if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
  443. raise
  444. # fallback to legacy playlist
  445. return self._process_legacy_playlist(playlist_id)
  446. def _process_legacy_playlist_url(self, url, display_id):
  447. playlist = self._download_legacy_playlist_url(url, display_id)
  448. return self._extract_from_legacy_playlist(playlist, display_id)
  449. def _process_legacy_playlist(self, playlist_id):
  450. return self._process_legacy_playlist_url(
  451. 'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id, playlist_id)
  452. def _download_legacy_playlist_url(self, url, playlist_id=None):
  453. return self._download_xml(
  454. url, playlist_id, 'Downloading legacy playlist XML')
  455. def _extract_from_legacy_playlist(self, playlist, playlist_id):
  456. no_items = playlist.find('./{%s}noItems' % self._EMP_PLAYLIST_NS)
  457. if no_items is not None:
  458. reason = no_items.get('reason')
  459. if reason == 'preAvailability':
  460. msg = 'Episode %s is not yet available' % playlist_id
  461. elif reason == 'postAvailability':
  462. msg = 'Episode %s is no longer available' % playlist_id
  463. elif reason == 'noMedia':
  464. msg = 'Episode %s is not currently available' % playlist_id
  465. else:
  466. msg = 'Episode %s is not available: %s' % (playlist_id, reason)
  467. raise ExtractorError(msg, expected=True)
  468. for item in self._extract_items(playlist):
  469. kind = item.get('kind')
  470. if kind not in ('programme', 'radioProgramme'):
  471. continue
  472. title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
  473. description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
  474. description = description_el.text if description_el is not None else None
  475. def get_programme_id(item):
  476. def get_from_attributes(item):
  477. for p in('identifier', 'group'):
  478. value = item.get(p)
  479. if value and re.match(r'^[pb][\da-z]{7}$', value):
  480. return value
  481. get_from_attributes(item)
  482. mediator = item.find('./{%s}mediator' % self._EMP_PLAYLIST_NS)
  483. if mediator is not None:
  484. return get_from_attributes(mediator)
  485. programme_id = get_programme_id(item)
  486. duration = int_or_none(item.get('duration'))
  487. if programme_id:
  488. formats, subtitles = self._download_media_selector(programme_id)
  489. else:
  490. formats, subtitles = self._process_media_selector(item, playlist_id)
  491. programme_id = playlist_id
  492. return programme_id, title, description, duration, formats, subtitles
  493. def _real_extract(self, url):
  494. group_id = self._match_id(url)
  495. webpage = self._download_webpage(url, group_id, 'Downloading video page')
  496. error = self._search_regex(
  497. r'<div\b[^>]+\bclass=["\']smp__message delta["\'][^>]*>([^<]+)<',
  498. webpage, 'error', default=None)
  499. if error:
  500. raise ExtractorError(error, expected=True)
  501. programme_id = None
  502. duration = None
  503. tviplayer = self._search_regex(
  504. r'mediator\.bind\(({.+?})\s*,\s*document\.getElementById',
  505. webpage, 'player', default=None)
  506. if tviplayer:
  507. player = self._parse_json(tviplayer, group_id).get('player', {})
  508. duration = int_or_none(player.get('duration'))
  509. programme_id = player.get('vpid')
  510. if not programme_id:
  511. programme_id = self._search_regex(
  512. r'"vpid"\s*:\s*"(%s)"' % self._ID_REGEX, webpage, 'vpid', fatal=False, default=None)
  513. if programme_id:
  514. formats, subtitles = self._download_media_selector(programme_id)
  515. title = self._og_search_title(webpage, default=None) or self._html_search_regex(
  516. (r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>',
  517. r'<div[^>]+class="info"[^>]*>\s*<h1>(.+?)</h1>'), webpage, 'title')
  518. description = self._search_regex(
  519. (r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
  520. r'<div[^>]+class="info_+synopsis"[^>]*>([^<]+)</div>'),
  521. webpage, 'description', default=None)
  522. if not description:
  523. description = self._html_search_meta('description', webpage)
  524. else:
  525. programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
  526. self._sort_formats(formats)
  527. return {
  528. 'id': programme_id,
  529. 'title': title,
  530. 'description': description,
  531. 'thumbnail': self._og_search_thumbnail(webpage, default=None),
  532. 'duration': duration,
  533. 'formats': formats,
  534. 'subtitles': subtitles,
  535. }
  536. class BBCIE(BBCCoUkIE):
  537. IE_NAME = 'bbc'
  538. IE_DESC = 'BBC'
  539. _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
  540. _MEDIASELECTOR_URLS = [
  541. # Provides HQ HLS streams but fails with geolocation in some cases when it's
  542. # even not geo restricted at all
  543. 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
  544. # Provides more formats, namely direct mp4 links, but fails on some videos with
  545. # notukerror for non UK (?) users (e.g.
  546. # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
  547. 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s',
  548. # Provides fewer formats, but works everywhere for everybody (hopefully)
  549. 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s',
  550. ]
  551. _TESTS = [{
  552. # article with multiple videos embedded with data-playable containing vpids
  553. 'url': 'http://www.bbc.com/news/world-europe-32668511',
  554. 'info_dict': {
  555. 'id': 'world-europe-32668511',
  556. 'title': 'Russia stages massive WW2 parade despite Western boycott',
  557. 'description': 'md5:00ff61976f6081841f759a08bf78cc9c',
  558. },
  559. 'playlist_count': 2,
  560. }, {
  561. # article with multiple videos embedded with data-playable (more videos)
  562. 'url': 'http://www.bbc.com/news/business-28299555',
  563. 'info_dict': {
  564. 'id': 'business-28299555',
  565. 'title': 'Farnborough Airshow: Video highlights',
  566. 'description': 'BBC reports and video highlights at the Farnborough Airshow.',
  567. },
  568. 'playlist_count': 9,
  569. 'skip': 'Save time',
  570. }, {
  571. # article with multiple videos embedded with `new SMP()`
  572. # broken
  573. 'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
  574. 'info_dict': {
  575. 'id': '3662a707-0af9-3149-963f-47bea720b460',
  576. 'title': 'BUGGER',
  577. },
  578. 'playlist_count': 18,
  579. }, {
  580. # single video embedded with data-playable containing vpid
  581. 'url': 'http://www.bbc.com/news/world-europe-32041533',
  582. 'info_dict': {
  583. 'id': 'p02mprgb',
  584. 'ext': 'mp4',
  585. 'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
  586. 'description': 'md5:2868290467291b37feda7863f7a83f54',
  587. 'duration': 47,
  588. 'timestamp': 1427219242,
  589. 'upload_date': '20150324',
  590. },
  591. 'params': {
  592. # rtmp download
  593. 'skip_download': True,
  594. }
  595. }, {
  596. # article with single video embedded with data-playable containing XML playlist
  597. # with direct video links as progressiveDownloadUrl (for now these are extracted)
  598. # and playlist with f4m and m3u8 as streamingUrl
  599. 'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
  600. 'info_dict': {
  601. 'id': '150615_telabyad_kentin_cogu',
  602. 'ext': 'mp4',
  603. 'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
  604. 'description': 'md5:33a4805a855c9baf7115fcbde57e7025',
  605. 'timestamp': 1434397334,
  606. 'upload_date': '20150615',
  607. },
  608. 'params': {
  609. 'skip_download': True,
  610. }
  611. }, {
  612. # single video embedded with data-playable containing XML playlists (regional section)
  613. 'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
  614. 'info_dict': {
  615. 'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
  616. 'ext': 'mp4',
  617. 'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
  618. 'description': 'md5:1525f17448c4ee262b64b8f0c9ce66c8',
  619. 'timestamp': 1434713142,
  620. 'upload_date': '20150619',
  621. },
  622. 'params': {
  623. 'skip_download': True,
  624. }
  625. }, {
  626. # single video from video playlist embedded with vxp-playlist-data JSON
  627. 'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
  628. 'info_dict': {
  629. 'id': 'p02w6qjc',
  630. 'ext': 'mp4',
  631. 'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
  632. 'duration': 56,
  633. 'description': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
  634. },
  635. 'params': {
  636. 'skip_download': True,
  637. }
  638. }, {
  639. # single video story with digitalData
  640. 'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
  641. 'info_dict': {
  642. 'id': 'p02q6gc4',
  643. 'ext': 'flv',
  644. 'title': 'Sri Lanka’s spicy secret',
  645. 'description': 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.',
  646. 'timestamp': 1437674293,
  647. 'upload_date': '20150723',
  648. },
  649. 'params': {
  650. # rtmp download
  651. 'skip_download': True,
  652. }
  653. }, {
  654. # single video story without digitalData
  655. 'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
  656. 'info_dict': {
  657. 'id': 'p018zqqg',
  658. 'ext': 'mp4',
  659. 'title': 'Hyundai Santa Fe Sport: Rock star',
  660. 'description': 'md5:b042a26142c4154a6e472933cf20793d',
  661. 'timestamp': 1415867444,
  662. 'upload_date': '20141113',
  663. },
  664. 'params': {
  665. # rtmp download
  666. 'skip_download': True,
  667. }
  668. }, {
  669. # single video embedded with Morph
  670. 'url': 'http://www.bbc.co.uk/sport/live/olympics/36895975',
  671. 'info_dict': {
  672. 'id': 'p041vhd0',
  673. 'ext': 'mp4',
  674. 'title': "Nigeria v Japan - Men's First Round",
  675. 'description': 'Live coverage of the first round from Group B at the Amazonia Arena.',
  676. 'duration': 7980,
  677. 'uploader': 'BBC Sport',
  678. 'uploader_id': 'bbc_sport',
  679. },
  680. 'params': {
  681. # m3u8 download
  682. 'skip_download': True,
  683. },
  684. 'skip': 'Georestricted to UK',
  685. }, {
  686. # single video with playlist.sxml URL in playlist param
  687. 'url': 'http://www.bbc.com/sport/0/football/33653409',
  688. 'info_dict': {
  689. 'id': 'p02xycnp',
  690. 'ext': 'mp4',
  691. 'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
  692. 'description': 'BBC Sport\'s David Ornstein has the latest transfer gossip, including rumours of a Manchester United return for Cristiano Ronaldo.',
  693. 'duration': 140,
  694. },
  695. 'params': {
  696. # rtmp download
  697. 'skip_download': True,
  698. }
  699. }, {
  700. # article with multiple videos embedded with playlist.sxml in playlist param
  701. 'url': 'http://www.bbc.com/sport/0/football/34475836',
  702. 'info_dict': {
  703. 'id': '34475836',
  704. 'title': 'Jurgen Klopp: Furious football from a witty and winning coach',
  705. 'description': 'Fast-paced football, wit, wisdom and a ready smile - why Liverpool fans should come to love new boss Jurgen Klopp.',
  706. },
  707. 'playlist_count': 3,
  708. }, {
  709. # school report article with single video
  710. 'url': 'http://www.bbc.co.uk/schoolreport/35744779',
  711. 'info_dict': {
  712. 'id': '35744779',
  713. 'title': 'School which breaks down barriers in Jerusalem',
  714. },
  715. 'playlist_count': 1,
  716. }, {
  717. # single video with playlist URL from weather section
  718. 'url': 'http://www.bbc.com/weather/features/33601775',
  719. 'only_matching': True,
  720. }, {
  721. # custom redirection to www.bbc.com
  722. 'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
  723. 'only_matching': True,
  724. }, {
  725. # single video article embedded with data-media-vpid
  726. 'url': 'http://www.bbc.co.uk/sport/rowing/35908187',
  727. 'only_matching': True,
  728. }, {
  729. 'url': 'https://www.bbc.co.uk/bbcthree/clip/73d0bbd0-abc3-4cea-b3c0-cdae21905eb1',
  730. 'info_dict': {
  731. 'id': 'p06556y7',
  732. 'ext': 'mp4',
  733. 'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
  734. 'description': 'md5:4b7dfd063d5a789a1512e99662be3ddd',
  735. },
  736. 'params': {
  737. 'skip_download': True,
  738. }
  739. }, {
  740. # window.__PRELOADED_STATE__
  741. 'url': 'https://www.bbc.co.uk/radio/play/b0b9z4yl',
  742. 'info_dict': {
  743. 'id': 'b0b9z4vz',
  744. 'ext': 'mp4',
  745. 'title': 'Prom 6: An American in Paris and Turangalila',
  746. 'description': 'md5:51cf7d6f5c8553f197e58203bc78dff8',
  747. 'uploader': 'Radio 3',
  748. 'uploader_id': 'bbc_radio_three',
  749. },
  750. }, {
  751. 'url': 'http://www.bbc.co.uk/learningenglish/chinese/features/lingohack/ep-181227',
  752. 'info_dict': {
  753. 'id': 'p06w9tws',
  754. 'ext': 'mp4',
  755. 'title': 'md5:2fabf12a726603193a2879a055f72514',
  756. 'description': 'Learn English words and phrases from this story',
  757. },
  758. 'add_ie': [BBCCoUkIE.ie_key()],
  759. }]
  760. @classmethod
  761. def suitable(cls, url):
  762. EXCLUDE_IE = (BBCCoUkIE, BBCCoUkArticleIE, BBCCoUkIPlayerPlaylistIE, BBCCoUkPlaylistIE)
  763. return (False if any(ie.suitable(url) for ie in EXCLUDE_IE)
  764. else super(BBCIE, cls).suitable(url))
  765. def _extract_from_media_meta(self, media_meta, video_id):
  766. # Direct links to media in media metadata (e.g.
  767. # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
  768. # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
  769. source_files = media_meta.get('sourceFiles')
  770. if source_files:
  771. return [{
  772. 'url': f['url'],
  773. 'format_id': format_id,
  774. 'ext': f.get('encoding'),
  775. 'tbr': float_or_none(f.get('bitrate'), 1000),
  776. 'filesize': int_or_none(f.get('filesize')),
  777. } for format_id, f in source_files.items() if f.get('url')], []
  778. programme_id = media_meta.get('externalId')
  779. if programme_id:
  780. return self._download_media_selector(programme_id)
  781. # Process playlist.sxml as legacy playlist
  782. href = media_meta.get('href')
  783. if href:
  784. playlist = self._download_legacy_playlist_url(href)
  785. _, _, _, _, formats, subtitles = self._extract_from_legacy_playlist(playlist, video_id)
  786. return formats, subtitles
  787. return [], []
  788. def _extract_from_playlist_sxml(self, url, playlist_id, timestamp):
  789. programme_id, title, description, duration, formats, subtitles = \
  790. self._process_legacy_playlist_url(url, playlist_id)
  791. self._sort_formats(formats)
  792. return {
  793. 'id': programme_id,
  794. 'title': title,
  795. 'description': description,
  796. 'duration': duration,
  797. 'timestamp': timestamp,
  798. 'formats': formats,
  799. 'subtitles': subtitles,
  800. }
  801. def _real_extract(self, url):
  802. playlist_id = self._match_id(url)
  803. webpage = self._download_webpage(url, playlist_id)
  804. json_ld_info = self._search_json_ld(webpage, playlist_id, default={})
  805. timestamp = json_ld_info.get('timestamp')
  806. playlist_title = json_ld_info.get('title')
  807. if not playlist_title:
  808. playlist_title = self._og_search_title(
  809. webpage, default=None) or self._html_search_regex(
  810. r'<title>(.+?)</title>', webpage, 'playlist title', default=None)
  811. if playlist_title:
  812. playlist_title = re.sub(r'(.+)\s*-\s*BBC.*?$', r'\1', playlist_title).strip()
  813. playlist_description = json_ld_info.get(
  814. 'description') or self._og_search_description(webpage, default=None)
  815. if not timestamp:
  816. timestamp = parse_iso8601(self._search_regex(
  817. [r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
  818. r'itemprop="datePublished"[^>]+datetime="([^"]+)"',
  819. r'"datePublished":\s*"([^"]+)'],
  820. webpage, 'date', default=None))
  821. entries = []
  822. # article with multiple videos embedded with playlist.sxml (e.g.
  823. # http://www.bbc.com/sport/0/football/34475836)
  824. playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
  825. playlists.extend(re.findall(r'data-media-id="([^"]+/playlist\.sxml)"', webpage))
  826. if playlists:
  827. entries = [
  828. self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)
  829. for playlist_url in playlists]
  830. # news article with multiple videos embedded with data-playable
  831. data_playables = re.findall(r'data-playable=(["\'])({.+?})\1', webpage)
  832. if data_playables:
  833. for _, data_playable_json in data_playables:
  834. data_playable = self._parse_json(
  835. unescapeHTML(data_playable_json), playlist_id, fatal=False)
  836. if not data_playable:
  837. continue
  838. settings = data_playable.get('settings', {})
  839. if settings:
  840. # data-playable with video vpid in settings.playlistObject.items (e.g.
  841. # http://www.bbc.com/news/world-us-canada-34473351)
  842. playlist_object = settings.get('playlistObject', {})
  843. if playlist_object:
  844. items = playlist_object.get('items')
  845. if items and isinstance(items, list):
  846. title = playlist_object['title']
  847. description = playlist_object.get('summary')
  848. duration = int_or_none(items[0].get('duration'))
  849. programme_id = items[0].get('vpid')
  850. formats, subtitles = self._download_media_selector(programme_id)
  851. self._sort_formats(formats)
  852. entries.append({
  853. 'id': programme_id,
  854. 'title': title,
  855. 'description': description,
  856. 'timestamp': timestamp,
  857. 'duration': duration,
  858. 'formats': formats,
  859. 'subtitles': subtitles,
  860. })
  861. else:
  862. # data-playable without vpid but with a playlist.sxml URLs
  863. # in otherSettings.playlist (e.g.
  864. # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
  865. playlist = data_playable.get('otherSettings', {}).get('playlist', {})
  866. if playlist:
  867. entry = None
  868. for key in ('streaming', 'progressiveDownload'):
  869. playlist_url = playlist.get('%sUrl' % key)
  870. if not playlist_url:
  871. continue
  872. try:
  873. info = self._extract_from_playlist_sxml(
  874. playlist_url, playlist_id, timestamp)
  875. if not entry:
  876. entry = info
  877. else:
  878. entry['title'] = info['title']
  879. entry['formats'].extend(info['formats'])
  880. except Exception as e:
  881. # Some playlist URL may fail with 500, at the same time
  882. # the other one may work fine (e.g.
  883. # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
  884. if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
  885. continue
  886. raise
  887. if entry:
  888. self._sort_formats(entry['formats'])
  889. entries.append(entry)
  890. if entries:
  891. return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
  892. # http://www.bbc.co.uk/learningenglish/chinese/features/lingohack/ep-181227
  893. group_id = self._search_regex(
  894. r'<div[^>]+\bclass=["\']video["\'][^>]+\bdata-pid=["\'](%s)' % self._ID_REGEX,
  895. webpage, 'group id', default=None)
  896. if playlist_id:
  897. return self.url_result(
  898. 'https://www.bbc.co.uk/programmes/%s' % group_id,
  899. ie=BBCCoUkIE.ie_key())
  900. # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
  901. programme_id = self._search_regex(
  902. [r'data-(?:video-player|media)-vpid="(%s)"' % self._ID_REGEX,
  903. r'<param[^>]+name="externalIdentifier"[^>]+value="(%s)"' % self._ID_REGEX,
  904. r'videoId\s*:\s*["\'](%s)["\']' % self._ID_REGEX],
  905. webpage, 'vpid', default=None)
  906. if programme_id:
  907. formats, subtitles = self._download_media_selector(programme_id)
  908. self._sort_formats(formats)
  909. # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
  910. digital_data = self._parse_json(
  911. self._search_regex(
  912. r'var\s+digitalData\s*=\s*({.+?});?\n', webpage, 'digital data', default='{}'),
  913. programme_id, fatal=False)
  914. page_info = digital_data.get('page', {}).get('pageInfo', {})
  915. title = page_info.get('pageName') or self._og_search_title(webpage)
  916. description = page_info.get('description') or self._og_search_description(webpage)
  917. timestamp = parse_iso8601(page_info.get('publicationDate')) or timestamp
  918. return {
  919. 'id': programme_id,
  920. 'title': title,
  921. 'description': description,
  922. 'timestamp': timestamp,
  923. 'formats': formats,
  924. 'subtitles': subtitles,
  925. }
  926. # Morph based embed (e.g. http://www.bbc.co.uk/sport/live/olympics/36895975)
  927. # There are several setPayload calls may be present but the video
  928. # seems to be always related to the first one
  929. morph_payload = self._parse_json(
  930. self._search_regex(
  931. r'Morph\.setPayload\([^,]+,\s*({.+?})\);',
  932. webpage, 'morph payload', default='{}'),
  933. playlist_id, fatal=False)
  934. if morph_payload:
  935. components = try_get(morph_payload, lambda x: x['body']['components'], list) or []
  936. for component in components:
  937. if not isinstance(component, dict):
  938. continue
  939. lead_media = try_get(component, lambda x: x['props']['leadMedia'], dict)
  940. if not lead_media:
  941. continue
  942. identifiers = lead_media.get('identifiers')
  943. if not identifiers or not isinstance(identifiers, dict):
  944. continue
  945. programme_id = identifiers.get('vpid') or identifiers.get('playablePid')
  946. if not programme_id:
  947. continue
  948. title = lead_media.get('title') or self._og_search_title(webpage)
  949. formats, subtitles = self._download_media_selector(programme_id)
  950. self._sort_formats(formats)
  951. description = lead_media.get('summary')
  952. uploader = lead_media.get('masterBrand')
  953. uploader_id = lead_media.get('mid')
  954. duration = None
  955. duration_d = lead_media.get('duration')
  956. if isinstance(duration_d, dict):
  957. duration = parse_duration(dict_get(
  958. duration_d, ('rawDuration', 'formattedDuration', 'spokenDuration')))
  959. return {
  960. 'id': programme_id,
  961. 'title': title,
  962. 'description': description,
  963. 'duration': duration,
  964. 'uploader': uploader,
  965. 'uploader_id': uploader_id,
  966. 'formats': formats,
  967. 'subtitles': subtitles,
  968. }
  969. preload_state = self._parse_json(self._search_regex(
  970. r'window\.__PRELOADED_STATE__\s*=\s*({.+?});', webpage,
  971. 'preload state', default='{}'), playlist_id, fatal=False)
  972. if preload_state:
  973. current_programme = preload_state.get('programmes', {}).get('current') or {}
  974. programme_id = current_programme.get('id')
  975. if current_programme and programme_id and current_programme.get('type') == 'playable_item':
  976. title = current_programme.get('titles', {}).get('tertiary') or playlist_title
  977. formats, subtitles = self._download_media_selector(programme_id)
  978. self._sort_formats(formats)
  979. synopses = current_programme.get('synopses') or {}
  980. network = current_programme.get('network') or {}
  981. duration = int_or_none(
  982. current_programme.get('duration', {}).get('value'))
  983. thumbnail = None
  984. image_url = current_programme.get('image_url')
  985. if image_url:
  986. thumbnail = image_url.replace('{recipe}', '1920x1920')
  987. return {
  988. 'id': programme_id,
  989. 'title': title,
  990. 'description': dict_get(synopses, ('long', 'medium', 'short')),
  991. 'thumbnail': thumbnail,
  992. 'duration': duration,
  993. 'uploader': network.get('short_title'),
  994. 'uploader_id': network.get('id'),
  995. 'formats': formats,
  996. 'subtitles': subtitles,
  997. }
  998. bbc3_config = self._parse_json(
  999. self._search_regex(
  1000. r'(?s)bbcthreeConfig\s*=\s*({.+?})\s*;\s*<', webpage,
  1001. 'bbcthree config', default='{}'),
  1002. playlist_id, transform_source=js_to_json, fatal=False)
  1003. if bbc3_config:
  1004. bbc3_playlist = try_get(
  1005. bbc3_config, lambda x: x['payload']['content']['bbcMedia']['playlist'],
  1006. dict)
  1007. if bbc3_playlist:
  1008. playlist_title = bbc3_playlist.get('title') or playlist_title
  1009. thumbnail = bbc3_playlist.get('holdingImageURL')
  1010. entries = []
  1011. for bbc3_item in bbc3_playlist['items']:
  1012. programme_id = bbc3_item.get('versionID')
  1013. if not programme_id:
  1014. continue
  1015. formats, subtitles = self._download_media_selector(programme_id)
  1016. self._sort_formats(formats)
  1017. entries.append({
  1018. 'id': programme_id,
  1019. 'title': playlist_title,
  1020. 'thumbnail': thumbnail,
  1021. 'timestamp': timestamp,
  1022. 'formats': formats,
  1023. 'subtitles': subtitles,
  1024. })
  1025. return self.playlist_result(
  1026. entries, playlist_id, playlist_title, playlist_description)
  1027. def extract_all(pattern):
  1028. return list(filter(None, map(
  1029. lambda s: self._parse_json(s, playlist_id, fatal=False),
  1030. re.findall(pattern, webpage))))
  1031. # Multiple video article (e.g.
  1032. # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
  1033. EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+%s(?:\b[^"]+)?' % self._ID_REGEX
  1034. entries = []
  1035. for match in extract_all(r'new\s+SMP\(({.+?})\)'):
  1036. embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')
  1037. if embed_url and re.match(EMBED_URL, embed_url):
  1038. entries.append(embed_url)
  1039. entries.extend(re.findall(
  1040. r'setPlaylist\("(%s)"\)' % EMBED_URL, webpage))
  1041. if entries:
  1042. return self.playlist_result(
  1043. [self.url_result(entry_, 'BBCCoUk') for entry_ in entries],
  1044. playlist_id, playlist_title, playlist_description)
  1045. # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
  1046. medias = extract_all(r"data-media-meta='({[^']+})'")
  1047. if not medias:
  1048. # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
  1049. media_asset = self._search_regex(
  1050. r'mediaAssetPage\.init\(\s*({.+?}), "/',
  1051. webpage, 'media asset', default=None)
  1052. if media_asset:
  1053. media_asset_page = self._parse_json(media_asset, playlist_id, fatal=False)
  1054. medias = []
  1055. for video in media_asset_page.get('videos', {}).values():
  1056. medias.extend(video.values())
  1057. if not medias:
  1058. # Multiple video playlist with single `now playing` entry (e.g.
  1059. # http://www.bbc.com/news/video_and_audio/must_see/33767813)
  1060. vxp_playlist = self._parse_json(
  1061. self._search_regex(
  1062. r'<script[^>]+class="vxp-playlist-data"[^>]+type="application/json"[^>]*>([^<]+)</script>',
  1063. webpage, 'playlist data'),
  1064. playlist_id)
  1065. playlist_medias = []
  1066. for item in vxp_playlist:
  1067. media = item.get('media')
  1068. if not media:
  1069. continue
  1070. playlist_medias.append(media)
  1071. # Download single video if found media with asset id matching the video id from URL
  1072. if item.get('advert', {}).get('assetId') == playlist_id:
  1073. medias = [media]
  1074. break
  1075. # Fallback to the whole playlist
  1076. if not medias:
  1077. medias = playlist_medias
  1078. entries = []
  1079. for num, media_meta in enumerate(medias, start=1):
  1080. formats, subtitles = self._extract_from_media_meta(media_meta, playlist_id)
  1081. if not formats:
  1082. continue
  1083. self._sort_formats(formats)
  1084. video_id = media_meta.get('externalId')
  1085. if not video_id:
  1086. video_id = playlist_id if len(medias) == 1 else '%s-%s' % (playlist_id, num)
  1087. title = media_meta.get('caption')
  1088. if not title:
  1089. title = playlist_title if len(medias) == 1 else '%s - Video %s' % (playlist_title, num)
  1090. duration = int_or_none(media_meta.get('durationInSeconds')) or parse_duration(media_meta.get('duration'))
  1091. images = []
  1092. for image in media_meta.get('images', {}).values():
  1093. images.extend(image.values())
  1094. if 'image' in media_meta:
  1095. images.append(media_meta['image'])
  1096. thumbnails = [{
  1097. 'url': image.get('href'),
  1098. 'width': int_or_none(image.get('width')),
  1099. 'height': int_or_none(image.get('height')),
  1100. } for image in images]
  1101. entries.append({
  1102. 'id': video_id,
  1103. 'title': title,
  1104. 'thumbnails': thumbnails,
  1105. 'duration': duration,
  1106. 'timestamp': timestamp,
  1107. 'formats': formats,
  1108. 'subtitles': subtitles,
  1109. })
  1110. return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
  1111. class BBCCoUkArticleIE(InfoExtractor):
  1112. _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
  1113. IE_NAME = 'bbc.co.uk:article'
  1114. IE_DESC = 'BBC articles'
  1115. _TEST = {
  1116. 'url': 'http://www.bbc.co.uk/programmes/articles/3jNQLTMrPlYGTBn0WV6M2MS/not-your-typical-role-model-ada-lovelace-the-19th-century-programmer',
  1117. 'info_dict': {
  1118. 'id': '3jNQLTMrPlYGTBn0WV6M2MS',
  1119. 'title': 'Calculating Ada: The Countess of Computing - Not your typical role model: Ada Lovelace the 19th century programmer - BBC Four',
  1120. 'description': 'Hannah Fry reveals some of her surprising discoveries about Ada Lovelace during filming.',
  1121. },
  1122. 'playlist_count': 4,
  1123. 'add_ie': ['BBCCoUk'],
  1124. }
  1125. def _real_extract(self, url):
  1126. playlist_id = self._match_id(url)
  1127. webpage = self._download_webpage(url, playlist_id)
  1128. title = self._og_search_title(webpage)
  1129. description = self._og_search_description(webpage).strip()
  1130. entries = [self.url_result(programme_url) for programme_url in re.findall(
  1131. r'<div[^>]+typeof="Clip"[^>]+resource="([^"]+)"', webpage)]
  1132. return self.playlist_result(entries, playlist_id, title, description)
  1133. class BBCCoUkPlaylistBaseIE(InfoExtractor):
  1134. def _entries(self, webpage, url, playlist_id):
  1135. single_page = 'page' in compat_urlparse.parse_qs(
  1136. compat_urlparse.urlparse(url).query)
  1137. for page_num in itertools.count(2):
  1138. for video_id in re.findall(
  1139. self._VIDEO_ID_TEMPLATE % BBCCoUkIE._ID_REGEX, webpage):
  1140. yield self.url_result(
  1141. self._URL_TEMPLATE % video_id, BBCCoUkIE.ie_key())
  1142. if single_page:
  1143. return
  1144. next_page = self._search_regex(
  1145. r'<li[^>]+class=(["\'])pagination_+next\1[^>]*><a[^>]+href=(["\'])(?P<url>(?:(?!\2).)+)\2',
  1146. webpage, 'next page url', default=None, group='url')
  1147. if not next_page:
  1148. break
  1149. webpage = self._download_webpage(
  1150. compat_urlparse.urljoin(url, next_page), playlist_id,
  1151. 'Downloading page %d' % page_num, page_num)
  1152. def _real_extract(self, url):
  1153. playlist_id = self._match_id(url)
  1154. webpage = self._download_webpage(url, playlist_id)
  1155. title, description = self._extract_title_and_description(webpage)
  1156. return self.playlist_result(
  1157. self._entries(webpage, url, playlist_id),
  1158. playlist_id, title, description)
  1159. class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
  1160. IE_NAME = 'bbc.co.uk:iplayer:playlist'
  1161. _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/(?:episodes|group)/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
  1162. _URL_TEMPLATE = 'http://www.bbc.co.uk/iplayer/episode/%s'
  1163. _VIDEO_ID_TEMPLATE = r'data-ip-id=["\'](%s)'
  1164. _TESTS = [{
  1165. 'url': 'http://www.bbc.co.uk/iplayer/episodes/b05rcz9v',
  1166. 'info_dict': {
  1167. 'id': 'b05rcz9v',
  1168. 'title': 'The Disappearance',
  1169. 'description': 'French thriller serial about a missing teenager.',
  1170. },
  1171. 'playlist_mincount': 6,
  1172. 'skip': 'This programme is not currently available on BBC iPlayer',
  1173. }, {
  1174. # Available for over a year unlike 30 days for most other programmes
  1175. 'url': 'http://www.bbc.co.uk/iplayer/group/p02tcc32',
  1176. 'info_dict': {
  1177. 'id': 'p02tcc32',
  1178. 'title': 'Bohemian Icons',
  1179. 'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
  1180. },
  1181. 'playlist_mincount': 10,
  1182. }]
  1183. def _extract_title_and_description(self, webpage):
  1184. title = self._search_regex(r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)
  1185. description = self._search_regex(
  1186. r'<p[^>]+class=(["\'])subtitle\1[^>]*>(?P<value>[^<]+)</p>',
  1187. webpage, 'description', fatal=False, group='value')
  1188. return title, description
  1189. class BBCCoUkPlaylistIE(BBCCoUkPlaylistBaseIE):
  1190. IE_NAME = 'bbc.co.uk:playlist'
  1191. _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/programmes/(?P<id>%s)/(?:episodes|broadcasts|clips)' % BBCCoUkIE._ID_REGEX
  1192. _URL_TEMPLATE = 'http://www.bbc.co.uk/programmes/%s'
  1193. _VIDEO_ID_TEMPLATE = r'data-pid=["\'](%s)'
  1194. _TESTS = [{
  1195. 'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/clips',
  1196. 'info_dict': {
  1197. 'id': 'b05rcz9v',
  1198. 'title': 'The Disappearance - Clips - BBC Four',
  1199. 'description': 'French thriller serial about a missing teenager.',
  1200. },
  1201. 'playlist_mincount': 7,
  1202. }, {
  1203. # multipage playlist, explicit page
  1204. 'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips?page=1',
  1205. 'info_dict': {
  1206. 'id': 'b00mfl7n',
  1207. 'title': 'Frozen Planet - Clips - BBC One',
  1208. 'description': 'md5:65dcbf591ae628dafe32aa6c4a4a0d8c',
  1209. },
  1210. 'playlist_mincount': 24,
  1211. }, {
  1212. # multipage playlist, all pages
  1213. 'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips',
  1214. 'info_dict': {
  1215. 'id': 'b00mfl7n',
  1216. 'title': 'Frozen Planet - Clips - BBC One',
  1217. 'description': 'md5:65dcbf591ae628dafe32aa6c4a4a0d8c',
  1218. },
  1219. 'playlist_mincount': 142,
  1220. }, {
  1221. 'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/broadcasts/2016/06',
  1222. 'only_matching': True,
  1223. }, {
  1224. 'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/clips',
  1225. 'only_matching': True,
  1226. }, {
  1227. 'url': 'http://www.bbc.co.uk/programmes/b055jkys/episodes/player',
  1228. 'only_matching': True,
  1229. }]
  1230. def _extract_title_and_description(self, webpage):
  1231. title = self._og_search_title(webpage, fatal=False)
  1232. description = self._og_search_description(webpage)
  1233. return title, description