You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

297 lines
12 KiB

10 years ago
10 years ago
10 years ago
  1. from __future__ import unicode_literals
  2. import re
  3. from .subtitles import SubtitlesInfoExtractor
  4. from ..compat import (
  5. compat_urllib_parse,
  6. compat_urllib_request,
  7. compat_str,
  8. )
  9. from ..utils import (
  10. ExtractorError,
  11. find_xpath_attr,
  12. fix_xml_ampersands,
  13. HEADRequest,
  14. unescapeHTML,
  15. url_basename,
  16. RegexNotFoundError,
  17. )
  18. def _media_xml_tag(tag):
  19. return '{http://search.yahoo.com/mrss/}%s' % tag
  20. class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
  21. _MOBILE_TEMPLATE = None
  22. @staticmethod
  23. def _id_from_uri(uri):
  24. return uri.split(':')[-1]
  25. # This was originally implemented for ComedyCentral, but it also works here
  26. @staticmethod
  27. def _transform_rtmp_url(rtmp_video_url):
  28. m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
  29. if not m:
  30. return rtmp_video_url
  31. base = 'http://viacommtvstrmfs.fplive.net/'
  32. return base + m.group('finalid')
  33. def _get_feed_url(self, uri):
  34. return self._FEED_URL
  35. def _get_thumbnail_url(self, uri, itemdoc):
  36. search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail'))
  37. thumb_node = itemdoc.find(search_path)
  38. if thumb_node is None:
  39. return None
  40. else:
  41. return thumb_node.attrib['url']
  42. def _extract_mobile_video_formats(self, mtvn_id):
  43. webpage_url = self._MOBILE_TEMPLATE % mtvn_id
  44. req = compat_urllib_request.Request(webpage_url)
  45. # Otherwise we get a webpage that would execute some javascript
  46. req.add_header('User-Agent', 'curl/7')
  47. webpage = self._download_webpage(req, mtvn_id,
  48. 'Downloading mobile page')
  49. metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
  50. req = HEADRequest(metrics_url)
  51. response = self._request_webpage(req, mtvn_id, 'Resolving url')
  52. url = response.geturl()
  53. # Transform the url to get the best quality:
  54. url = re.sub(r'.+pxE=mp4', 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=0+_pxK=18639+_pxE=mp4', url, 1)
  55. return [{'url': url, 'ext': 'mp4'}]
  56. def _extract_video_formats(self, mdoc, mtvn_id):
  57. if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4)$', mdoc.find('.//src').text) is not None:
  58. if mtvn_id is not None and self._MOBILE_TEMPLATE is not None:
  59. self.to_screen('The normal version is not available from your '
  60. 'country, trying with the mobile version')
  61. return self._extract_mobile_video_formats(mtvn_id)
  62. raise ExtractorError('This video is not available from your country.',
  63. expected=True)
  64. formats = []
  65. for rendition in mdoc.findall('.//rendition'):
  66. try:
  67. _, _, ext = rendition.attrib['type'].partition('/')
  68. rtmp_video_url = rendition.find('./src').text
  69. formats.append({'ext': ext,
  70. 'url': self._transform_rtmp_url(rtmp_video_url),
  71. 'format_id': rendition.get('bitrate'),
  72. 'width': int(rendition.get('width')),
  73. 'height': int(rendition.get('height')),
  74. })
  75. except (KeyError, TypeError):
  76. raise ExtractorError('Invalid rendition field.')
  77. self._sort_formats(formats)
  78. return formats
  79. def _extract_subtitles(self, mdoc, mtvn_id):
  80. subtitles = {}
  81. FORMATS = {
  82. 'scc': 'cea-608',
  83. 'eia-608': 'cea-608',
  84. 'xml': 'ttml',
  85. }
  86. subtitles_format = FORMATS.get(
  87. self._downloader.params.get('subtitlesformat'), 'ttml')
  88. for transcript in mdoc.findall('.//transcript'):
  89. if transcript.get('kind') != 'captions':
  90. continue
  91. lang = transcript.get('srclang')
  92. for typographic in transcript.findall('./typographic'):
  93. captions_format = typographic.get('format')
  94. if captions_format == subtitles_format:
  95. subtitles[lang] = compat_str(typographic.get('src'))
  96. break
  97. if self._downloader.params.get('listsubtitles', False):
  98. self._list_available_subtitles(mtvn_id, subtitles)
  99. return self.extract_subtitles(mtvn_id, subtitles)
  100. def _get_video_info(self, itemdoc):
  101. uri = itemdoc.find('guid').text
  102. video_id = self._id_from_uri(uri)
  103. self.report_extraction(video_id)
  104. mediagen_url = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content'))).attrib['url']
  105. # Remove the templates, like &device={device}
  106. mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', mediagen_url)
  107. if 'acceptMethods' not in mediagen_url:
  108. mediagen_url += '&acceptMethods=fms'
  109. mediagen_doc = self._download_xml(mediagen_url, video_id,
  110. 'Downloading video urls')
  111. description_node = itemdoc.find('description')
  112. if description_node is not None:
  113. description = description_node.text.strip()
  114. else:
  115. description = None
  116. title_el = None
  117. if title_el is None:
  118. title_el = find_xpath_attr(
  119. itemdoc, './/{http://search.yahoo.com/mrss/}category',
  120. 'scheme', 'urn:mtvn:video_title')
  121. if title_el is None:
  122. title_el = itemdoc.find('.//{http://search.yahoo.com/mrss/}title')
  123. if title_el is None:
  124. title_el = itemdoc.find('.//title')
  125. if title_el.text is None:
  126. title_el = None
  127. title = title_el.text
  128. if title is None:
  129. raise ExtractorError('Could not find video title')
  130. title = title.strip()
  131. # This a short id that's used in the webpage urls
  132. mtvn_id = None
  133. mtvn_id_node = find_xpath_attr(itemdoc, './/{http://search.yahoo.com/mrss/}category',
  134. 'scheme', 'urn:mtvn:id')
  135. if mtvn_id_node is not None:
  136. mtvn_id = mtvn_id_node.text
  137. return {
  138. 'title': title,
  139. 'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
  140. 'subtitles': self._extract_subtitles(mediagen_doc, mtvn_id),
  141. 'id': video_id,
  142. 'thumbnail': self._get_thumbnail_url(uri, itemdoc),
  143. 'description': description,
  144. }
  145. def _get_videos_info(self, uri):
  146. video_id = self._id_from_uri(uri)
  147. feed_url = self._get_feed_url(uri)
  148. data = compat_urllib_parse.urlencode({'uri': uri})
  149. idoc = self._download_xml(
  150. feed_url + '?' + data, video_id,
  151. 'Downloading info', transform_source=fix_xml_ampersands)
  152. return self.playlist_result(
  153. [self._get_video_info(item) for item in idoc.findall('.//item')])
  154. def _real_extract(self, url):
  155. title = url_basename(url)
  156. webpage = self._download_webpage(url, title)
  157. try:
  158. # the url can be http://media.mtvnservices.com/fb/{mgid}.swf
  159. # or http://media.mtvnservices.com/{mgid}
  160. og_url = self._og_search_video_url(webpage)
  161. mgid = url_basename(og_url)
  162. if mgid.endswith('.swf'):
  163. mgid = mgid[:-4]
  164. except RegexNotFoundError:
  165. mgid = None
  166. if mgid is None or ':' not in mgid:
  167. mgid = self._search_regex(
  168. [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
  169. webpage, 'mgid')
  170. videos_info = self._get_videos_info(mgid)
  171. if self._downloader.params.get('listsubtitles', False):
  172. return
  173. return videos_info
  174. class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
  175. IE_NAME = 'mtvservices:embedded'
  176. _VALID_URL = r'https?://media\.mtvnservices\.com/embed/(?P<mgid>.+?)(\?|/|$)'
  177. _TEST = {
  178. # From http://www.thewrap.com/peter-dinklage-sums-up-game-of-thrones-in-45-seconds-video/
  179. 'url': 'http://media.mtvnservices.com/embed/mgid:uma:video:mtv.com:1043906/cp~vid%3D1043906%26uri%3Dmgid%3Auma%3Avideo%3Amtv.com%3A1043906',
  180. 'md5': 'cb349b21a7897164cede95bd7bf3fbb9',
  181. 'info_dict': {
  182. 'id': '1043906',
  183. 'ext': 'mp4',
  184. 'title': 'Peter Dinklage Sums Up \'Game Of Thrones\' In 45 Seconds',
  185. 'description': '"Sexy sexy sexy, stabby stabby stabby, beautiful language," says Peter Dinklage as he tries summarizing "Game of Thrones" in under a minute.',
  186. },
  187. }
  188. def _get_feed_url(self, uri):
  189. video_id = self._id_from_uri(uri)
  190. site_id = uri.replace(video_id, '')
  191. config_url = ('http://media.mtvnservices.com/pmt/e1/players/{0}/'
  192. 'context4/context5/config.xml'.format(site_id))
  193. config_doc = self._download_xml(config_url, video_id)
  194. feed_node = config_doc.find('.//feed')
  195. feed_url = feed_node.text.strip().split('?')[0]
  196. return feed_url
  197. def _real_extract(self, url):
  198. mobj = re.match(self._VALID_URL, url)
  199. mgid = mobj.group('mgid')
  200. return self._get_videos_info(mgid)
  201. class MTVIE(MTVServicesInfoExtractor):
  202. _VALID_URL = r'''(?x)^https?://
  203. (?:(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$|
  204. m\.mtv\.com/videos/video\.rbml\?.*?id=(?P<mgid>[^&]+))'''
  205. _FEED_URL = 'http://www.mtv.com/player/embed/AS3/rss/'
  206. _TESTS = [
  207. {
  208. 'url': 'http://www.mtv.com/videos/misc/853555/ours-vh1-storytellers.jhtml',
  209. 'file': '853555.mp4',
  210. 'md5': '850f3f143316b1e71fa56a4edfd6e0f8',
  211. 'info_dict': {
  212. 'title': 'Taylor Swift - "Ours (VH1 Storytellers)"',
  213. 'description': 'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.',
  214. },
  215. },
  216. {
  217. 'add_ie': ['Vevo'],
  218. 'url': 'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml',
  219. 'file': 'USCJY1331283.mp4',
  220. 'md5': '73b4e7fcadd88929292fe52c3ced8caf',
  221. 'info_dict': {
  222. 'title': 'Everything Has Changed',
  223. 'upload_date': '20130606',
  224. 'uploader': 'Taylor Swift',
  225. },
  226. 'skip': 'VEVO is only available in some countries',
  227. },
  228. ]
  229. def _get_thumbnail_url(self, uri, itemdoc):
  230. return 'http://mtv.mtvnimages.com/uri/' + uri
  231. def _real_extract(self, url):
  232. mobj = re.match(self._VALID_URL, url)
  233. video_id = mobj.group('videoid')
  234. uri = mobj.groupdict().get('mgid')
  235. if uri is None:
  236. webpage = self._download_webpage(url, video_id)
  237. # Some videos come from Vevo.com
  238. m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
  239. webpage, re.DOTALL)
  240. if m_vevo:
  241. vevo_id = m_vevo.group(1)
  242. self.to_screen('Vevo video detected: %s' % vevo_id)
  243. return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
  244. uri = self._html_search_regex(r'/uri/(.*?)\?', webpage, 'uri')
  245. return self._get_videos_info(uri)
  246. class MTVIggyIE(MTVServicesInfoExtractor):
  247. IE_NAME = 'mtviggy.com'
  248. _VALID_URL = r'https?://www\.mtviggy\.com/videos/.+'
  249. _TEST = {
  250. 'url': 'http://www.mtviggy.com/videos/arcade-fire-behind-the-scenes-at-the-biggest-music-experiment-yet/',
  251. 'info_dict': {
  252. 'id': '984696',
  253. 'ext': 'mp4',
  254. 'title': 'Arcade Fire: Behind the Scenes at the Biggest Music Experiment Yet',
  255. }
  256. }
  257. _FEED_URL = 'http://all.mtvworldverticals.com/feed-xml/'