You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

284 lines
10 KiB

  1. from __future__ import unicode_literals
  2. from .subtitles import SubtitlesInfoExtractor
  3. from ..utils import (
  4. fix_xml_ampersands,
  5. parse_duration,
  6. qualities,
  7. strip_jsonp,
  8. unified_strdate,
  9. url_basename,
  10. )
  11. class NPOBaseIE(SubtitlesInfoExtractor):
  12. def _get_token(self, video_id):
  13. token_page = self._download_webpage(
  14. 'http://ida.omroep.nl/npoplayer/i.js',
  15. video_id, note='Downloading token')
  16. return self._search_regex(
  17. r'npoplayer\.token = "(.+?)"', token_page, 'token')
  18. class NPOIE(NPOBaseIE):
  19. IE_NAME = 'npo.nl'
  20. _VALID_URL = r'https?://www\.npo\.nl/[^/]+/[^/]+/(?P<id>[^/?]+)'
  21. _TESTS = [
  22. {
  23. 'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719',
  24. 'md5': '4b3f9c429157ec4775f2c9cb7b911016',
  25. 'info_dict': {
  26. 'id': 'VPWON_1220719',
  27. 'ext': 'm4v',
  28. 'title': 'Nieuwsuur',
  29. 'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.',
  30. 'upload_date': '20140622',
  31. },
  32. },
  33. {
  34. 'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800',
  35. 'md5': 'da50a5787dbfc1603c4ad80f31c5120b',
  36. 'info_dict': {
  37. 'id': 'VARA_101191800',
  38. 'ext': 'm4v',
  39. 'title': 'De Mega Mike & Mega Thomas show',
  40. 'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
  41. 'upload_date': '20090227',
  42. 'duration': 2400,
  43. },
  44. },
  45. {
  46. 'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289',
  47. 'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
  48. 'info_dict': {
  49. 'id': 'VPWON_1169289',
  50. 'ext': 'm4v',
  51. 'title': 'Tegenlicht',
  52. 'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
  53. 'upload_date': '20130225',
  54. 'duration': 3000,
  55. },
  56. },
  57. {
  58. 'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
  59. 'info_dict': {
  60. 'id': 'WO_VPRO_043706',
  61. 'ext': 'wmv',
  62. 'title': 'De nieuwe mens - Deel 1',
  63. 'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
  64. 'duration': 4680,
  65. },
  66. 'params': {
  67. # mplayer mms download
  68. 'skip_download': True,
  69. }
  70. },
  71. # non asf in streams
  72. {
  73. 'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771',
  74. 'md5': 'b3da13de374cbe2d5332a7e910bef97f',
  75. 'info_dict': {
  76. 'id': 'WO_NOS_762771',
  77. 'ext': 'mp4',
  78. 'title': 'Hoe gaat Europa verder na Parijs?',
  79. },
  80. },
  81. ]
  82. def _real_extract(self, url):
  83. video_id = self._match_id(url)
  84. return self._get_info(video_id)
  85. def _get_info(self, video_id):
  86. metadata = self._download_json(
  87. 'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
  88. video_id,
  89. # We have to remove the javascript callback
  90. transform_source=strip_jsonp,
  91. )
  92. token = self._get_token(video_id)
  93. formats = []
  94. pubopties = metadata.get('pubopties')
  95. if pubopties:
  96. quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
  97. for format_id in pubopties:
  98. format_info = self._download_json(
  99. 'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s'
  100. % (video_id, format_id, token),
  101. video_id, 'Downloading %s JSON' % format_id)
  102. if format_info.get('error_code', 0) or format_info.get('errorcode', 0):
  103. continue
  104. streams = format_info.get('streams')
  105. if streams:
  106. video_info = self._download_json(
  107. streams[0] + '&type=json',
  108. video_id, 'Downloading %s stream JSON' % format_id)
  109. else:
  110. video_info = format_info
  111. video_url = video_info.get('url')
  112. if not video_url:
  113. continue
  114. if format_id == 'adaptive':
  115. formats.extend(self._extract_m3u8_formats(video_url, video_id))
  116. else:
  117. formats.append({
  118. 'url': video_url,
  119. 'format_id': format_id,
  120. 'quality': quality(format_id),
  121. })
  122. streams = metadata.get('streams')
  123. if streams:
  124. for i, stream in enumerate(streams):
  125. stream_url = stream.get('url')
  126. if not stream_url:
  127. continue
  128. if '.asf' not in stream_url:
  129. formats.append({
  130. 'url': stream_url,
  131. 'quality': stream.get('kwaliteit'),
  132. })
  133. continue
  134. asx = self._download_xml(
  135. stream_url, video_id,
  136. 'Downloading stream %d ASX playlist' % i,
  137. transform_source=fix_xml_ampersands)
  138. ref = asx.find('./ENTRY/Ref')
  139. if ref is None:
  140. continue
  141. video_url = ref.get('href')
  142. if not video_url:
  143. continue
  144. formats.append({
  145. 'url': video_url,
  146. 'ext': stream.get('formaat', 'asf'),
  147. 'quality': stream.get('kwaliteit'),
  148. })
  149. subtitles = {}
  150. tt888 = metadata.get('tt888')
  151. if self._have_to_download_any_subtitles and tt888 == 'ja':
  152. subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
  153. subtitles = self.extract_subtitles(video_id, subtitles)
  154. self._sort_formats(formats)
  155. return {
  156. 'id': video_id,
  157. 'title': metadata['titel'],
  158. 'description': metadata['info'],
  159. 'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
  160. 'upload_date': unified_strdate(metadata.get('gidsdatum')),
  161. 'duration': parse_duration(metadata.get('tijdsduur')),
  162. 'formats': formats,
  163. 'subtitles': subtitles,
  164. }
  165. class NPOLiveIE(NPOBaseIE):
  166. IE_NAME = 'npo.nl:live'
  167. _VALID_URL = r'https?://www\.npo\.nl/live/(?P<id>.+)'
  168. _TEST = {
  169. 'url': 'http://www.npo.nl/live/npo-1',
  170. 'info_dict': {
  171. 'id': 'LI_NEDERLAND1_136692',
  172. 'display_id': 'npo-1',
  173. 'ext': 'mp4',
  174. 'title': 're:^Nederland 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
  175. 'description': 'Livestream',
  176. 'is_live': True,
  177. },
  178. 'params': {
  179. 'skip_download': True,
  180. }
  181. }
  182. def _real_extract(self, url):
  183. display_id = self._match_id(url)
  184. webpage = self._download_webpage(url, display_id)
  185. live_id = self._search_regex(
  186. r'data-prid="([^"]+)"', webpage, 'live id')
  187. metadata = self._download_json(
  188. 'http://e.omroep.nl/metadata/%s' % live_id,
  189. display_id, transform_source=strip_jsonp)
  190. token = self._get_token(display_id)
  191. formats = []
  192. streams = metadata.get('streams')
  193. if streams:
  194. for stream in streams:
  195. stream_type = stream.get('type').lower()
  196. if stream_type == 'ss':
  197. continue
  198. stream_info = self._download_json(
  199. 'http://ida.omroep.nl/aapi/?stream=%s&token=%s&type=jsonp'
  200. % (stream.get('url'), token),
  201. display_id, 'Downloading %s JSON' % stream_type)
  202. if stream_info.get('error_code', 0) or stream_info.get('errorcode', 0):
  203. continue
  204. stream_url = self._download_json(
  205. stream_info['stream'], display_id,
  206. 'Downloading %s URL' % stream_type,
  207. transform_source=strip_jsonp)
  208. if stream_type == 'hds':
  209. f4m_formats = self._extract_f4m_formats(stream_url, display_id)
  210. # f4m downloader downloads only piece of live stream
  211. for f4m_format in f4m_formats:
  212. f4m_format['preference'] = -1
  213. formats.extend(f4m_formats)
  214. elif stream_type == 'hls':
  215. formats.extend(self._extract_m3u8_formats(stream_url, display_id, 'mp4'))
  216. else:
  217. formats.append({
  218. 'url': stream_url,
  219. })
  220. self._sort_formats(formats)
  221. return {
  222. 'id': live_id,
  223. 'display_id': display_id,
  224. 'title': self._live_title(metadata['titel']),
  225. 'description': metadata['info'],
  226. 'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
  227. 'formats': formats,
  228. 'is_live': True,
  229. }
  230. class TegenlichtVproIE(NPOIE):
  231. IE_NAME = 'tegenlicht.vpro.nl'
  232. _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
  233. _TESTS = [
  234. {
  235. 'url': 'http://tegenlicht.vpro.nl/afleveringen/2012-2013/de-toekomst-komt-uit-afrika.html',
  236. 'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
  237. 'info_dict': {
  238. 'id': 'VPWON_1169289',
  239. 'ext': 'm4v',
  240. 'title': 'Tegenlicht',
  241. 'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
  242. 'upload_date': '20130225',
  243. },
  244. },
  245. ]
  246. def _real_extract(self, url):
  247. name = url_basename(url)
  248. webpage = self._download_webpage(url, name)
  249. urn = self._html_search_meta('mediaurn', webpage)
  250. info_page = self._download_json(
  251. 'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
  252. return self._get_info(info_page['mid'])