You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

285 lines
11 KiB

  1. from __future__ import unicode_literals
  2. import re
  3. from .common import InfoExtractor
  4. from ..compat import (
  5. compat_parse_qs,
  6. compat_urllib_parse_unquote,
  7. )
  8. from ..utils import (
  9. determine_ext,
  10. ExtractorError,
  11. int_or_none,
  12. urlencode_postdata,
  13. get_element_by_attribute,
  14. mimetype2ext,
  15. )
  16. class MetacafeIE(InfoExtractor):
  17. _VALID_URL = r'https?://(?:www\.)?metacafe\.com/watch/(?P<video_id>[^/]+)/(?P<display_id>[^/?#]+)'
  18. _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
  19. _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
  20. IE_NAME = 'metacafe'
  21. _TESTS = [
  22. # Youtube video
  23. {
  24. 'add_ie': ['Youtube'],
  25. 'url': 'http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/',
  26. 'info_dict': {
  27. 'id': '_aUehQsCQtM',
  28. 'ext': 'mp4',
  29. 'upload_date': '20090102',
  30. 'title': 'The Electric Company | "Short I" | PBS KIDS GO!',
  31. 'description': 'md5:2439a8ef6d5a70e380c22f5ad323e5a8',
  32. 'uploader': 'PBS',
  33. 'uploader_id': 'PBS'
  34. }
  35. },
  36. # Normal metacafe video
  37. {
  38. 'url': 'http://www.metacafe.com/watch/11121940/news_stuff_you_wont_do_with_your_playstation_4/',
  39. 'md5': '6e0bca200eaad2552e6915ed6fd4d9ad',
  40. 'info_dict': {
  41. 'id': '11121940',
  42. 'ext': 'mp4',
  43. 'title': 'News: Stuff You Won\'t Do with Your PlayStation 4',
  44. 'uploader': 'ign',
  45. 'description': 'Sony released a massive FAQ on the PlayStation Blog detailing the PS4\'s capabilities and limitations.',
  46. },
  47. 'skip': 'Page is temporarily unavailable.',
  48. },
  49. # AnyClip video
  50. {
  51. 'url': 'http://www.metacafe.com/watch/an-dVVXnuY7Jh77J/the_andromeda_strain_1971_stop_the_bomb_part_3/',
  52. 'info_dict': {
  53. 'id': 'an-dVVXnuY7Jh77J',
  54. 'ext': 'mp4',
  55. 'title': 'The Andromeda Strain (1971): Stop the Bomb Part 3',
  56. 'uploader': 'AnyClip',
  57. 'description': 'md5:cbef0460d31e3807f6feb4e7a5952e5b',
  58. },
  59. },
  60. # age-restricted video
  61. {
  62. 'url': 'http://www.metacafe.com/watch/5186653/bbc_internal_christmas_tape_79_uncensored_outtakes_etc/',
  63. 'md5': '98dde7c1a35d02178e8ab7560fe8bd09',
  64. 'info_dict': {
  65. 'id': '5186653',
  66. 'ext': 'mp4',
  67. 'title': 'BBC INTERNAL Christmas Tape \'79 - UNCENSORED Outtakes, Etc.',
  68. 'uploader': 'Dwayne Pipe',
  69. 'description': 'md5:950bf4c581e2c059911fa3ffbe377e4b',
  70. 'age_limit': 18,
  71. },
  72. },
  73. # cbs video
  74. {
  75. 'url': 'http://www.metacafe.com/watch/cb-8VD4r_Zws8VP/open_this_is_face_the_nation_february_9/',
  76. 'info_dict': {
  77. 'id': '8VD4r_Zws8VP',
  78. 'ext': 'flv',
  79. 'title': 'Open: This is Face the Nation, February 9',
  80. 'description': 'md5:8a9ceec26d1f7ed6eab610834cc1a476',
  81. 'duration': 96,
  82. 'uploader': 'CBSI-NEW',
  83. 'upload_date': '20140209',
  84. 'timestamp': 1391959800,
  85. },
  86. 'params': {
  87. # rtmp download
  88. 'skip_download': True,
  89. },
  90. },
  91. # Movieclips.com video
  92. {
  93. 'url': 'http://www.metacafe.com/watch/mv-Wy7ZU/my_week_with_marilyn_do_you_love_me/',
  94. 'info_dict': {
  95. 'id': 'mv-Wy7ZU',
  96. 'ext': 'mp4',
  97. 'title': 'My Week with Marilyn - Do You Love Me?',
  98. 'description': 'From the movie My Week with Marilyn - Colin (Eddie Redmayne) professes his love to Marilyn (Michelle Williams) and gets her to promise to return to set and finish the movie.',
  99. 'uploader': 'movie_trailers',
  100. 'duration': 176,
  101. },
  102. 'params': {
  103. 'skip_download': 'requires rtmpdump',
  104. }
  105. }
  106. ]
  107. def report_disclaimer(self):
  108. self.to_screen('Retrieving disclaimer')
  109. def _confirm_age(self):
  110. # Retrieve disclaimer
  111. self.report_disclaimer()
  112. self._download_webpage(self._DISCLAIMER, None, False, 'Unable to retrieve disclaimer')
  113. # Confirm age
  114. self.report_age_confirmation()
  115. self._download_webpage(
  116. self._FILTER_POST, None, False, 'Unable to confirm age',
  117. data=urlencode_postdata({
  118. 'filters': '0',
  119. 'submit': "Continue - I'm over 18",
  120. }), headers={
  121. 'Content-Type': 'application/x-www-form-urlencoded',
  122. })
  123. def _real_extract(self, url):
  124. # Extract id and simplified title from URL
  125. video_id, display_id = re.match(self._VALID_URL, url).groups()
  126. # the video may come from an external site
  127. m_external = re.match(r'^(\w{2})-(.*)$', video_id)
  128. if m_external is not None:
  129. prefix, ext_id = m_external.groups()
  130. # Check if video comes from YouTube
  131. if prefix == 'yt':
  132. return self.url_result('http://www.youtube.com/watch?v=%s' % ext_id, 'Youtube')
  133. # CBS videos use theplatform.com
  134. if prefix == 'cb':
  135. return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
  136. # self._confirm_age()
  137. # AnyClip videos require the flashversion cookie so that we get the link
  138. # to the mp4 file
  139. headers = {}
  140. if video_id.startswith('an-'):
  141. headers['Cookie'] = 'flashVersion=0;'
  142. # Retrieve video webpage to extract further information
  143. webpage = self._download_webpage(url, video_id, headers=headers)
  144. error = get_element_by_attribute(
  145. 'class', 'notfound-page-title', webpage)
  146. if error:
  147. raise ExtractorError(error, expected=True)
  148. video_title = self._html_search_meta(
  149. ['og:title', 'twitter:title'], webpage, 'title', default=None) or self._search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
  150. # Extract URL, uploader and title from webpage
  151. self.report_extraction(video_id)
  152. video_url = None
  153. mobj = re.search(r'(?m)&(?:media|video)URL=([^&]+)', webpage)
  154. if mobj is not None:
  155. mediaURL = compat_urllib_parse_unquote(mobj.group(1))
  156. video_ext = determine_ext(mediaURL)
  157. # Extract gdaKey if available
  158. mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
  159. if mobj is None:
  160. video_url = mediaURL
  161. else:
  162. gdaKey = mobj.group(1)
  163. video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
  164. if video_url is None:
  165. mobj = re.search(r'<video src="([^"]+)"', webpage)
  166. if mobj:
  167. video_url = mobj.group(1)
  168. video_ext = 'mp4'
  169. if video_url is None:
  170. flashvars = self._search_regex(
  171. r' name="flashvars" value="(.*?)"', webpage, 'flashvars',
  172. default=None)
  173. if flashvars:
  174. vardict = compat_parse_qs(flashvars)
  175. if 'mediaData' not in vardict:
  176. raise ExtractorError('Unable to extract media URL')
  177. mobj = re.search(
  178. r'"mediaURL":"(?P<mediaURL>http.*?)",(.*?)"key":"(?P<key>.*?)"', vardict['mediaData'][0])
  179. if mobj is None:
  180. raise ExtractorError('Unable to extract media URL')
  181. mediaURL = mobj.group('mediaURL').replace('\\/', '/')
  182. video_url = '%s?__gda__=%s' % (mediaURL, mobj.group('key'))
  183. video_ext = determine_ext(video_url)
  184. if video_url is None:
  185. player_url = self._search_regex(
  186. r"swfobject\.embedSWF\('([^']+)'",
  187. webpage, 'config URL', default=None)
  188. if player_url:
  189. config_url = self._search_regex(
  190. r'config=(.+)$', player_url, 'config URL')
  191. config_doc = self._download_xml(
  192. config_url, video_id,
  193. note='Downloading video config')
  194. smil_url = config_doc.find('.//properties').attrib['smil_file']
  195. smil_doc = self._download_xml(
  196. smil_url, video_id,
  197. note='Downloading SMIL document')
  198. base_url = smil_doc.find('./head/meta').attrib['base']
  199. video_url = []
  200. for vn in smil_doc.findall('.//video'):
  201. br = int(vn.attrib['system-bitrate'])
  202. play_path = vn.attrib['src']
  203. video_url.append({
  204. 'format_id': 'smil-%d' % br,
  205. 'url': base_url,
  206. 'play_path': play_path,
  207. 'page_url': url,
  208. 'player_url': player_url,
  209. 'ext': play_path.partition(':')[0],
  210. })
  211. if video_url is None:
  212. flashvars = self._parse_json(self._search_regex(
  213. r'flashvars\s*=\s*({.*});', webpage, 'flashvars',
  214. default=None), video_id, fatal=False)
  215. if flashvars:
  216. video_url = []
  217. for source in flashvars.get('sources'):
  218. source_url = source.get('src')
  219. if not source_url:
  220. continue
  221. ext = mimetype2ext(source.get('type')) or determine_ext(source_url)
  222. if ext == 'm3u8':
  223. video_url.extend(self._extract_m3u8_formats(
  224. source_url, video_id, 'mp4',
  225. 'm3u8_native', m3u8_id='hls', fatal=False))
  226. else:
  227. video_url.append({
  228. 'url': source_url,
  229. 'ext': ext,
  230. })
  231. if video_url is None:
  232. raise ExtractorError('Unsupported video type')
  233. description = self._html_search_meta(
  234. ['og:description', 'twitter:description', 'description'],
  235. webpage, 'title', fatal=False)
  236. thumbnail = self._html_search_meta(
  237. ['og:image', 'twitter:image'], webpage, 'title', fatal=False)
  238. video_uploader = self._html_search_regex(
  239. r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
  240. webpage, 'uploader nickname', fatal=False)
  241. duration = int_or_none(
  242. self._html_search_meta('video:duration', webpage, default=None))
  243. age_limit = (
  244. 18
  245. if re.search(r'(?:"contentRating":|"rating",)"restricted"', webpage)
  246. else 0)
  247. if isinstance(video_url, list):
  248. formats = video_url
  249. else:
  250. formats = [{
  251. 'url': video_url,
  252. 'ext': video_ext,
  253. }]
  254. self._sort_formats(formats)
  255. return {
  256. 'id': video_id,
  257. 'display_id': display_id,
  258. 'description': description,
  259. 'uploader': video_uploader,
  260. 'title': video_title,
  261. 'thumbnail': thumbnail,
  262. 'age_limit': age_limit,
  263. 'formats': formats,
  264. 'duration': duration,
  265. }