You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

359 lines
14 KiB

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import base64
  5. from .common import InfoExtractor
  6. from ..compat import (
  7. compat_urlparse,
  8. compat_parse_qs,
  9. )
  10. from ..utils import (
  11. clean_html,
  12. ExtractorError,
  13. int_or_none,
  14. unsmuggle_url,
  15. smuggle_url,
  16. )
  17. class KalturaIE(InfoExtractor):
  18. _VALID_URL = r'''(?x)
  19. (?:
  20. kaltura:(?P<partner_id>\d+):(?P<id>[0-9a-z_]+)|
  21. https?://
  22. (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com(?::\d+)?/
  23. (?:
  24. (?:
  25. # flash player
  26. index\.php/(?:kwidget|extwidget/preview)|
  27. # html5 player
  28. html5/html5lib/[^/]+/mwEmbedFrame\.php
  29. )
  30. )(?:/(?P<path>[^?]+))?(?:\?(?P<query>.*))?
  31. )
  32. '''
  33. _SERVICE_URL = 'http://cdnapi.kaltura.com'
  34. _SERVICE_BASE = '/api_v3/index.php'
  35. # See https://github.com/kaltura/server/blob/master/plugins/content/caption/base/lib/model/enums/CaptionType.php
  36. _CAPTION_TYPES = {
  37. 1: 'srt',
  38. 2: 'ttml',
  39. 3: 'vtt',
  40. }
  41. _TESTS = [
  42. {
  43. 'url': 'kaltura:269692:1_1jc2y3e4',
  44. 'md5': '3adcbdb3dcc02d647539e53f284ba171',
  45. 'info_dict': {
  46. 'id': '1_1jc2y3e4',
  47. 'ext': 'mp4',
  48. 'title': 'Straight from the Heart',
  49. 'upload_date': '20131219',
  50. 'uploader_id': 'mlundberg@wolfgangsvault.com',
  51. 'description': 'The Allman Brothers Band, 12/16/1981',
  52. 'thumbnail': 're:^https?://.*/thumbnail/.*',
  53. 'timestamp': int,
  54. },
  55. },
  56. {
  57. 'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4',
  58. 'only_matching': True,
  59. },
  60. {
  61. 'url': 'https://cdnapisec.kaltura.com/index.php/kwidget/wid/_557781/uiconf_id/22845202/entry_id/1_plr1syf3',
  62. 'only_matching': True,
  63. },
  64. {
  65. 'url': 'https://cdnapisec.kaltura.com/html5/html5lib/v2.30.2/mwEmbedFrame.php/p/1337/uiconf_id/20540612/entry_id/1_sf5ovm7u?wid=_243342',
  66. 'only_matching': True,
  67. },
  68. {
  69. # video with subtitles
  70. 'url': 'kaltura:111032:1_cw786r8q',
  71. 'only_matching': True,
  72. },
  73. {
  74. # video with ttml subtitles (no fileExt)
  75. 'url': 'kaltura:1926081:0_l5ye1133',
  76. 'info_dict': {
  77. 'id': '0_l5ye1133',
  78. 'ext': 'mp4',
  79. 'title': 'What Can You Do With Python?',
  80. 'upload_date': '20160221',
  81. 'uploader_id': 'stork',
  82. 'thumbnail': 're:^https?://.*/thumbnail/.*',
  83. 'timestamp': int,
  84. 'subtitles': {
  85. 'en': [{
  86. 'ext': 'ttml',
  87. }],
  88. },
  89. },
  90. 'skip': 'Gone. Maybe https://www.safaribooksonline.com/library/tutorials/introduction-to-python-anon/3469/',
  91. 'params': {
  92. 'skip_download': True,
  93. },
  94. },
  95. {
  96. 'url': 'https://www.kaltura.com/index.php/extwidget/preview/partner_id/1770401/uiconf_id/37307382/entry_id/0_58u8kme7/embed/iframe?&flashvars[streamerType]=auto',
  97. 'only_matching': True,
  98. },
  99. {
  100. 'url': 'https://www.kaltura.com:443/index.php/extwidget/preview/partner_id/1770401/uiconf_id/37307382/entry_id/0_58u8kme7/embed/iframe?&flashvars[streamerType]=auto',
  101. 'only_matching': True,
  102. }
  103. ]
  104. @staticmethod
  105. def _extract_url(webpage):
  106. # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
  107. mobj = (
  108. re.search(
  109. r"""(?xs)
  110. kWidget\.(?:thumb)?[Ee]mbed\(
  111. \{.*?
  112. (?P<q1>['"])wid(?P=q1)\s*:\s*
  113. (?P<q2>['"])_?(?P<partner_id>(?:(?!(?P=q2)).)+)(?P=q2),.*?
  114. (?P<q3>['"])entry_?[Ii]d(?P=q3)\s*:\s*
  115. (?P<q4>['"])(?P<id>(?:(?!(?P=q4)).)+)(?P=q4)(?:,|\s*\})
  116. """, webpage) or
  117. re.search(
  118. r'''(?xs)
  119. (?P<q1>["'])
  120. (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?::\d+)?/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
  121. (?P=q1).*?
  122. (?:
  123. (?:
  124. entry_?[Ii]d|
  125. (?P<q2>["'])entry_?[Ii]d(?P=q2)
  126. )\s*:\s*|
  127. \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]\s*=\s*
  128. )
  129. (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
  130. ''', webpage) or
  131. re.search(
  132. r'''(?xs)
  133. <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["'])
  134. (?:https?:)?//(?:(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
  135. (?:(?!(?P=q1)).)*
  136. [?&;]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
  137. (?:(?!(?P=q1)).)*
  138. (?P=q1)
  139. ''', webpage)
  140. )
  141. if mobj:
  142. embed_info = mobj.groupdict()
  143. for k, v in embed_info.items():
  144. embed_info[k] = v.strip()
  145. url = 'kaltura:%(partner_id)s:%(id)s' % embed_info
  146. escaped_pid = re.escape(embed_info['partner_id'])
  147. service_url = re.search(
  148. r'<script[^>]+src=["\']((?:https?:)?//.+?)/p/%s/sp/%s00/embedIframeJs' % (escaped_pid, escaped_pid),
  149. webpage)
  150. if service_url:
  151. url = smuggle_url(url, {'service_url': service_url.group(1)})
  152. return url
  153. def _kaltura_api_call(self, video_id, actions, service_url=None, *args, **kwargs):
  154. params = actions[0]
  155. if len(actions) > 1:
  156. for i, a in enumerate(actions[1:], start=1):
  157. for k, v in a.items():
  158. params['%d:%s' % (i, k)] = v
  159. data = self._download_json(
  160. (service_url or self._SERVICE_URL) + self._SERVICE_BASE,
  161. video_id, query=params, *args, **kwargs)
  162. status = data if len(actions) == 1 else data[0]
  163. if status.get('objectType') == 'KalturaAPIException':
  164. raise ExtractorError(
  165. '%s said: %s' % (self.IE_NAME, status['message']))
  166. return data
  167. def _get_video_info(self, video_id, partner_id, service_url=None):
  168. actions = [
  169. {
  170. 'action': 'null',
  171. 'apiVersion': '3.1.5',
  172. 'clientTag': 'kdp:v3.8.5',
  173. 'format': 1, # JSON, 2 = XML, 3 = PHP
  174. 'service': 'multirequest',
  175. },
  176. {
  177. 'expiry': 86400,
  178. 'service': 'session',
  179. 'action': 'startWidgetSession',
  180. 'widgetId': '_%s' % partner_id,
  181. },
  182. {
  183. 'action': 'get',
  184. 'entryId': video_id,
  185. 'service': 'baseentry',
  186. 'ks': '{1:result:ks}',
  187. 'responseProfile:fields': 'createdAt,dataUrl,duration,name,plays,thumbnailUrl,userId',
  188. 'responseProfile:type': 1,
  189. },
  190. {
  191. 'action': 'getbyentryid',
  192. 'entryId': video_id,
  193. 'service': 'flavorAsset',
  194. 'ks': '{1:result:ks}',
  195. },
  196. {
  197. 'action': 'list',
  198. 'filter:entryIdEqual': video_id,
  199. 'service': 'caption_captionasset',
  200. 'ks': '{1:result:ks}',
  201. },
  202. ]
  203. return self._kaltura_api_call(
  204. video_id, actions, service_url, note='Downloading video info JSON')
  205. def _real_extract(self, url):
  206. url, smuggled_data = unsmuggle_url(url, {})
  207. mobj = re.match(self._VALID_URL, url)
  208. partner_id, entry_id = mobj.group('partner_id', 'id')
  209. ks = None
  210. captions = None
  211. if partner_id and entry_id:
  212. _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id, smuggled_data.get('service_url'))
  213. else:
  214. path, query = mobj.group('path', 'query')
  215. if not path and not query:
  216. raise ExtractorError('Invalid URL', expected=True)
  217. params = {}
  218. if query:
  219. params = compat_parse_qs(query)
  220. if path:
  221. splitted_path = path.split('/')
  222. params.update(dict((zip(splitted_path[::2], [[v] for v in splitted_path[1::2]]))))
  223. if 'wid' in params:
  224. partner_id = params['wid'][0][1:]
  225. elif 'p' in params:
  226. partner_id = params['p'][0]
  227. elif 'partner_id' in params:
  228. partner_id = params['partner_id'][0]
  229. else:
  230. raise ExtractorError('Invalid URL', expected=True)
  231. if 'entry_id' in params:
  232. entry_id = params['entry_id'][0]
  233. _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id)
  234. elif 'uiconf_id' in params and 'flashvars[referenceId]' in params:
  235. reference_id = params['flashvars[referenceId]'][0]
  236. webpage = self._download_webpage(url, reference_id)
  237. entry_data = self._parse_json(self._search_regex(
  238. r'window\.kalturaIframePackageData\s*=\s*({.*});',
  239. webpage, 'kalturaIframePackageData'),
  240. reference_id)['entryResult']
  241. info, flavor_assets = entry_data['meta'], entry_data['contextData']['flavorAssets']
  242. entry_id = info['id']
  243. # Unfortunately, data returned in kalturaIframePackageData lacks
  244. # captions so we will try requesting the complete data using
  245. # regular approach since we now know the entry_id
  246. try:
  247. _, info, flavor_assets, captions = self._get_video_info(
  248. entry_id, partner_id)
  249. except ExtractorError:
  250. # Regular scenario failed but we already have everything
  251. # extracted apart from captions and can process at least
  252. # with this
  253. pass
  254. else:
  255. raise ExtractorError('Invalid URL', expected=True)
  256. ks = params.get('flashvars[ks]', [None])[0]
  257. source_url = smuggled_data.get('source_url')
  258. if source_url:
  259. referrer = base64.b64encode(
  260. '://'.join(compat_urlparse.urlparse(source_url)[:2])
  261. .encode('utf-8')).decode('utf-8')
  262. else:
  263. referrer = None
  264. def sign_url(unsigned_url):
  265. if ks:
  266. unsigned_url += '/ks/%s' % ks
  267. if referrer:
  268. unsigned_url += '?referrer=%s' % referrer
  269. return unsigned_url
  270. data_url = info['dataUrl']
  271. if '/flvclipper/' in data_url:
  272. data_url = re.sub(r'/flvclipper/.*', '/serveFlavor', data_url)
  273. formats = []
  274. for f in flavor_assets:
  275. # Continue if asset is not ready
  276. if f.get('status') != 2:
  277. continue
  278. # Original format that's not available (e.g. kaltura:1926081:0_c03e1b5g)
  279. # skip for now.
  280. if f.get('fileExt') == 'chun':
  281. continue
  282. # DRM-protected video, cannot be decrypted
  283. if f.get('fileExt') == 'wvm':
  284. continue
  285. if not f.get('fileExt'):
  286. # QT indicates QuickTime; some videos have broken fileExt
  287. if f.get('containerFormat') == 'qt':
  288. f['fileExt'] = 'mov'
  289. else:
  290. f['fileExt'] = 'mp4'
  291. video_url = sign_url(
  292. '%s/flavorId/%s' % (data_url, f['id']))
  293. # audio-only has no videoCodecId (e.g. kaltura:1926081:0_c03e1b5g
  294. # -f mp4-56)
  295. vcodec = 'none' if 'videoCodecId' not in f and f.get(
  296. 'frameRate') == 0 else f.get('videoCodecId')
  297. formats.append({
  298. 'format_id': '%(fileExt)s-%(bitrate)s' % f,
  299. 'ext': f.get('fileExt'),
  300. 'tbr': int_or_none(f['bitrate']),
  301. 'fps': int_or_none(f.get('frameRate')),
  302. 'filesize_approx': int_or_none(f.get('size'), invscale=1024),
  303. 'container': f.get('containerFormat'),
  304. 'vcodec': vcodec,
  305. 'height': int_or_none(f.get('height')),
  306. 'width': int_or_none(f.get('width')),
  307. 'url': video_url,
  308. })
  309. if '/playManifest/' in data_url:
  310. m3u8_url = sign_url(data_url.replace(
  311. 'format/url', 'format/applehttp'))
  312. formats.extend(self._extract_m3u8_formats(
  313. m3u8_url, entry_id, 'mp4', 'm3u8_native',
  314. m3u8_id='hls', fatal=False))
  315. self._sort_formats(formats)
  316. subtitles = {}
  317. if captions:
  318. for caption in captions.get('objects', []):
  319. # Continue if caption is not ready
  320. if caption.get('status') != 2:
  321. continue
  322. if not caption.get('id'):
  323. continue
  324. caption_format = int_or_none(caption.get('format'))
  325. subtitles.setdefault(caption.get('languageCode') or caption.get('language'), []).append({
  326. 'url': '%s/api_v3/service/caption_captionasset/action/serve/captionAssetId/%s' % (self._SERVICE_URL, caption['id']),
  327. 'ext': caption.get('fileExt') or self._CAPTION_TYPES.get(caption_format) or 'ttml',
  328. })
  329. return {
  330. 'id': entry_id,
  331. 'title': info['name'],
  332. 'formats': formats,
  333. 'subtitles': subtitles,
  334. 'description': clean_html(info.get('description')),
  335. 'thumbnail': info.get('thumbnailUrl'),
  336. 'duration': info.get('duration'),
  337. 'timestamp': info.get('createdAt'),
  338. 'uploader_id': info.get('userId') if info.get('userId') != 'None' else None,
  339. 'view_count': info.get('plays'),
  340. }