You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

299 lines
12 KiB

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import uuid
  4. import xml.etree.ElementTree as etree
  5. import json
  6. import re
  7. from .common import InfoExtractor
  8. from .brightcove import BrightcoveNewIE
  9. from ..compat import (
  10. compat_str,
  11. compat_etree_register_namespace,
  12. )
  13. from ..utils import (
  14. extract_attributes,
  15. xpath_with_ns,
  16. xpath_element,
  17. xpath_text,
  18. int_or_none,
  19. parse_duration,
  20. smuggle_url,
  21. ExtractorError,
  22. determine_ext,
  23. )
  24. class ITVIE(InfoExtractor):
  25. _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
  26. _GEO_COUNTRIES = ['GB']
  27. _TESTS = [{
  28. 'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
  29. 'info_dict': {
  30. 'id': '2a2936a0053',
  31. 'ext': 'flv',
  32. 'title': 'Home Movie',
  33. },
  34. 'params': {
  35. # rtmp download
  36. 'skip_download': True,
  37. },
  38. }, {
  39. # unavailable via data-playlist-url
  40. 'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',
  41. 'only_matching': True,
  42. }, {
  43. # InvalidVodcrid
  44. 'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034',
  45. 'only_matching': True,
  46. }, {
  47. # ContentUnavailable
  48. 'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024',
  49. 'only_matching': True,
  50. }]
  51. def _real_extract(self, url):
  52. video_id = self._match_id(url)
  53. webpage = self._download_webpage(url, video_id)
  54. params = extract_attributes(self._search_regex(
  55. r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
  56. ns_map = {
  57. 'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',
  58. 'tem': 'http://tempuri.org/',
  59. 'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',
  60. 'com': 'http://schemas.itv.com/2009/05/Common',
  61. }
  62. for ns, full_ns in ns_map.items():
  63. compat_etree_register_namespace(ns, full_ns)
  64. def _add_ns(name):
  65. return xpath_with_ns(name, ns_map)
  66. def _add_sub_element(element, name):
  67. return etree.SubElement(element, _add_ns(name))
  68. production_id = (
  69. params.get('data-video-autoplay-id') or
  70. '%s#001' % (
  71. params.get('data-video-episode-id') or
  72. video_id.replace('a', '/')))
  73. req_env = etree.Element(_add_ns('soapenv:Envelope'))
  74. _add_sub_element(req_env, 'soapenv:Header')
  75. body = _add_sub_element(req_env, 'soapenv:Body')
  76. get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
  77. request = _add_sub_element(get_playlist, 'tem:request')
  78. _add_sub_element(request, 'itv:ProductionId').text = production_id
  79. _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
  80. vodcrid = _add_sub_element(request, 'itv:Vodcrid')
  81. _add_sub_element(vodcrid, 'com:Id')
  82. _add_sub_element(request, 'itv:Partition')
  83. user_info = _add_sub_element(get_playlist, 'tem:userInfo')
  84. _add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'
  85. _add_sub_element(user_info, 'itv:DM')
  86. _add_sub_element(user_info, 'itv:RevenueScienceValue')
  87. _add_sub_element(user_info, 'itv:SessionId')
  88. _add_sub_element(user_info, 'itv:SsoToken')
  89. _add_sub_element(user_info, 'itv:UserToken')
  90. site_info = _add_sub_element(get_playlist, 'tem:siteInfo')
  91. _add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'
  92. _add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'
  93. _add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'
  94. _add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'
  95. _add_sub_element(site_info, 'itv:Category')
  96. _add_sub_element(site_info, 'itv:Platform').text = 'DotCom'
  97. _add_sub_element(site_info, 'itv:Site').text = 'ItvCom'
  98. device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')
  99. _add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'
  100. player_info = _add_sub_element(get_playlist, 'tem:playerInfo')
  101. _add_sub_element(player_info, 'itv:Version').text = '2'
  102. headers = self.geo_verification_headers()
  103. headers.update({
  104. 'Content-Type': 'text/xml; charset=utf-8',
  105. 'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
  106. })
  107. info = self._search_json_ld(webpage, video_id, default={})
  108. formats = []
  109. subtitles = {}
  110. def extract_subtitle(sub_url):
  111. ext = determine_ext(sub_url, 'ttml')
  112. subtitles.setdefault('en', []).append({
  113. 'url': sub_url,
  114. 'ext': 'ttml' if ext == 'xml' else ext,
  115. })
  116. resp_env = self._download_xml(
  117. params['data-playlist-url'], video_id,
  118. headers=headers, data=etree.tostring(req_env))
  119. playlist = xpath_element(resp_env, './/Playlist')
  120. if playlist is None:
  121. fault_code = xpath_text(resp_env, './/faultcode')
  122. fault_string = xpath_text(resp_env, './/faultstring')
  123. if fault_code == 'InvalidGeoRegion':
  124. self.raise_geo_restricted(
  125. msg=fault_string, countries=self._GEO_COUNTRIES)
  126. elif fault_code not in (
  127. 'InvalidEntity', 'InvalidVodcrid', 'ContentUnavailable'):
  128. raise ExtractorError(
  129. '%s said: %s' % (self.IE_NAME, fault_string), expected=True)
  130. info.update({
  131. 'title': self._og_search_title(webpage),
  132. 'episode_title': params.get('data-video-episode'),
  133. 'series': params.get('data-video-title'),
  134. })
  135. else:
  136. title = xpath_text(playlist, 'EpisodeTitle', default=None)
  137. info.update({
  138. 'title': title,
  139. 'episode_title': title,
  140. 'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
  141. 'series': xpath_text(playlist, 'ProgrammeTitle'),
  142. 'duration': parse_duration(xpath_text(playlist, 'Duration')),
  143. })
  144. video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
  145. media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
  146. rtmp_url = media_files.attrib['base']
  147. for media_file in media_files.findall('MediaFile'):
  148. play_path = xpath_text(media_file, 'URL')
  149. if not play_path:
  150. continue
  151. tbr = int_or_none(media_file.get('bitrate'), 1000)
  152. f = {
  153. 'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
  154. 'play_path': play_path,
  155. # Providing this swfVfy allows to avoid truncated downloads
  156. 'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
  157. 'page_url': url,
  158. 'tbr': tbr,
  159. 'ext': 'flv',
  160. }
  161. app = self._search_regex(
  162. 'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
  163. if app:
  164. f.update({
  165. 'url': rtmp_url.split('?', 1)[0],
  166. 'app': app,
  167. })
  168. else:
  169. f['url'] = rtmp_url
  170. formats.append(f)
  171. for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
  172. if caption_url.text:
  173. extract_subtitle(caption_url.text)
  174. ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
  175. hmac = params.get('data-video-hmac')
  176. if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url):
  177. headers = self.geo_verification_headers()
  178. headers.update({
  179. 'Accept': 'application/vnd.itv.vod.playlist.v2+json',
  180. 'Content-Type': 'application/json',
  181. 'hmac': hmac.upper(),
  182. })
  183. ios_playlist = self._download_json(
  184. ios_playlist_url, video_id, data=json.dumps({
  185. 'user': {
  186. 'itvUserId': '',
  187. 'entitlements': [],
  188. 'token': ''
  189. },
  190. 'device': {
  191. 'manufacturer': 'Safari',
  192. 'model': '5',
  193. 'os': {
  194. 'name': 'Windows NT',
  195. 'version': '6.1',
  196. 'type': 'desktop'
  197. }
  198. },
  199. 'client': {
  200. 'version': '4.1',
  201. 'id': 'browser'
  202. },
  203. 'variantAvailability': {
  204. 'featureset': {
  205. 'min': ['hls', 'aes', 'outband-webvtt'],
  206. 'max': ['hls', 'aes', 'outband-webvtt']
  207. },
  208. 'platformTag': 'dotcom'
  209. }
  210. }).encode(), headers=headers, fatal=False)
  211. if ios_playlist:
  212. video_data = ios_playlist.get('Playlist', {}).get('Video', {})
  213. ios_base_url = video_data.get('Base')
  214. for media_file in video_data.get('MediaFiles', []):
  215. href = media_file.get('Href')
  216. if not href:
  217. continue
  218. if ios_base_url:
  219. href = ios_base_url + href
  220. ext = determine_ext(href)
  221. if ext == 'm3u8':
  222. formats.extend(self._extract_m3u8_formats(
  223. href, video_id, 'mp4', entry_protocol='m3u8_native',
  224. m3u8_id='hls', fatal=False))
  225. else:
  226. formats.append({
  227. 'url': href,
  228. })
  229. subs = video_data.get('Subtitles')
  230. if isinstance(subs, list):
  231. for sub in subs:
  232. if not isinstance(sub, dict):
  233. continue
  234. href = sub.get('Href')
  235. if isinstance(href, compat_str):
  236. extract_subtitle(href)
  237. if not info.get('duration'):
  238. info['duration'] = parse_duration(video_data.get('Duration'))
  239. self._sort_formats(formats)
  240. info.update({
  241. 'id': video_id,
  242. 'formats': formats,
  243. 'subtitles': subtitles,
  244. })
  245. return info
  246. class ITVBTCCIE(InfoExtractor):
  247. _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'
  248. _TEST = {
  249. 'url': 'http://www.itv.com/btcc/races/btcc-2018-all-the-action-from-brands-hatch',
  250. 'info_dict': {
  251. 'id': 'btcc-2018-all-the-action-from-brands-hatch',
  252. 'title': 'BTCC 2018: All the action from Brands Hatch',
  253. },
  254. 'playlist_mincount': 9,
  255. }
  256. BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'
  257. def _real_extract(self, url):
  258. playlist_id = self._match_id(url)
  259. webpage = self._download_webpage(url, playlist_id)
  260. entries = [
  261. self.url_result(
  262. smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {
  263. # ITV does not like some GB IP ranges, so here are some
  264. # IP blocks it accepts
  265. 'geo_ip_blocks': [
  266. '193.113.0.0/16', '54.36.162.0/23', '159.65.16.0/21'
  267. ],
  268. 'referrer': url,
  269. }),
  270. ie=BrightcoveNewIE.ie_key(), video_id=video_id)
  271. for video_id in re.findall(r'data-video-id=["\'](\d+)', webpage)]
  272. title = self._og_search_title(webpage, fatal=False)
  273. return self.playlist_result(entries, playlist_id, title)