You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

276 lines
11 KiB

11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import itertools
  4. import json
  5. import re
  6. from .common import InfoExtractor, SearchInfoExtractor
  7. from ..compat import (
  8. compat_urllib_parse,
  9. compat_urlparse,
  10. )
  11. from ..utils import (
  12. clean_html,
  13. unescapeHTML,
  14. ExtractorError,
  15. int_or_none,
  16. )
  17. from .nbc import NBCSportsVPlayerIE
  18. class YahooIE(InfoExtractor):
  19. IE_DESC = 'Yahoo screen and movies'
  20. _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+?)-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
  21. _TESTS = [
  22. {
  23. 'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
  24. 'info_dict': {
  25. 'id': '2d25e626-2378-391f-ada0-ddaf1417e588',
  26. 'ext': 'mp4',
  27. 'title': 'Julian Smith & Travis Legg Watch Julian Smith',
  28. 'description': 'Julian and Travis watch Julian Smith',
  29. 'duration': 6863,
  30. },
  31. },
  32. {
  33. 'url': 'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
  34. 'md5': 'd6e6fc6e1313c608f316ddad7b82b306',
  35. 'info_dict': {
  36. 'id': 'd1dedf8c-d58c-38c3-8963-e899929ae0a9',
  37. 'ext': 'mp4',
  38. 'title': 'Codefellas - The Cougar Lies with Spanish Moss',
  39. 'description': 'md5:66b627ab0a282b26352136ca96ce73c1',
  40. 'duration': 151,
  41. },
  42. },
  43. {
  44. 'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
  45. 'md5': '60e8ac193d8fb71997caa8fce54c6460',
  46. 'info_dict': {
  47. 'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb',
  48. 'ext': 'mp4',
  49. 'title': "Yahoo Saves 'Community'",
  50. 'description': 'md5:4d4145af2fd3de00cbb6c1d664105053',
  51. 'duration': 170,
  52. }
  53. },
  54. {
  55. 'url': 'https://tw.screen.yahoo.com/election-2014-askmayor/敢問市長-黃秀霜批賴清德-非常高傲-033009720.html',
  56. 'md5': '3a09cf59349cfaddae1797acc3c087fc',
  57. 'info_dict': {
  58. 'id': 'cac903b3-fcf4-3c14-b632-643ab541712f',
  59. 'ext': 'mp4',
  60. 'title': '敢問市長/黃秀霜批賴清德「非常高傲」',
  61. 'description': '直言台南沒捷運 交通居五都之末',
  62. 'duration': 396,
  63. }
  64. },
  65. {
  66. 'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
  67. 'md5': '0b51660361f0e27c9789e7037ef76f4b',
  68. 'info_dict': {
  69. 'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
  70. 'ext': 'mp4',
  71. 'title': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder',
  72. 'description': 'md5:f66c890e1490f4910a9953c941dee944',
  73. 'duration': 97,
  74. }
  75. },
  76. {
  77. 'url': 'https://ca.sports.yahoo.com/video/program-makes-hockey-more-affordable-013127711.html',
  78. 'md5': '57e06440778b1828a6079d2f744212c4',
  79. 'info_dict': {
  80. 'id': 'c9fa2a36-0d4d-3937-b8f6-cc0fb1881e73',
  81. 'ext': 'mp4',
  82. 'title': 'Program that makes hockey more affordable not offered in Manitoba',
  83. 'description': 'md5:c54a609f4c078d92b74ffb9bf1f496f4',
  84. 'duration': 121,
  85. }
  86. }, {
  87. 'url': 'https://ca.finance.yahoo.com/news/hackers-sony-more-trouble-well-154609075.html',
  88. 'md5': '226a895aae7e21b0129e2a2006fe9690',
  89. 'info_dict': {
  90. 'id': 'e624c4bc-3389-34de-9dfc-025f74943409',
  91. 'ext': 'mp4',
  92. 'title': '\'The Interview\' TV Spot: War',
  93. 'description': 'The Interview',
  94. 'duration': 30,
  95. }
  96. }, {
  97. 'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
  98. 'md5': '67010fdf3a08d290e060a4dd96baa07b',
  99. 'info_dict': {
  100. 'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
  101. 'ext': 'mp4',
  102. 'title': 'China Moses Is Crazy About the Blues',
  103. 'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
  104. 'duration': 128,
  105. }
  106. }, {
  107. 'url': 'https://in.lifestyle.yahoo.com/video/connect-dots-dark-side-virgo-090247395.html',
  108. 'md5': 'd9a083ccf1379127bf25699d67e4791b',
  109. 'info_dict': {
  110. 'id': '52aeeaa3-b3d1-30d8-9ef8-5d0cf05efb7c',
  111. 'ext': 'mp4',
  112. 'title': 'Connect the Dots: Dark Side of Virgo',
  113. 'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
  114. 'duration': 201,
  115. }
  116. }, {
  117. 'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
  118. 'md5': '989396ae73d20c6f057746fb226aa215',
  119. 'info_dict': {
  120. 'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
  121. 'ext': 'mp4',
  122. 'title': '\'True Story\' Trailer',
  123. 'description': 'True Story',
  124. 'duration': 150,
  125. },
  126. }, {
  127. 'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
  128. 'only_matching': True,
  129. }, {
  130. 'note': 'NBC Sports embeds',
  131. 'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
  132. 'info_dict': {
  133. 'id': '9CsDKds0kvHI',
  134. 'ext': 'flv',
  135. 'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
  136. 'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
  137. }
  138. }
  139. ]
  140. def _real_extract(self, url):
  141. mobj = re.match(self._VALID_URL, url)
  142. display_id = mobj.group('display_id')
  143. page_id = mobj.group('id')
  144. url = mobj.group('url')
  145. host = mobj.group('host')
  146. webpage = self._download_webpage(url, display_id)
  147. # Look for iframed media first
  148. iframe_m = re.search(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
  149. if iframe_m:
  150. iframepage = self._download_webpage(
  151. host + iframe_m.group(1), display_id, 'Downloading iframe webpage')
  152. items_json = self._search_regex(
  153. r'mediaItems: (\[.+?\])$', iframepage, 'items', flags=re.MULTILINE, default=None)
  154. if items_json:
  155. items = json.loads(items_json)
  156. video_id = items[0]['id']
  157. return self._get_info(video_id, display_id, webpage)
  158. # Look for NBCSports iframes
  159. nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
  160. if nbc_sports_url:
  161. return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
  162. items_json = self._search_regex(
  163. r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
  164. default=None)
  165. if items_json is None:
  166. CONTENT_ID_REGEXES = [
  167. r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
  168. r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
  169. r'"first_videoid"\s*:\s*"([^"]+)"',
  170. r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
  171. ]
  172. video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
  173. else:
  174. items = json.loads(items_json)
  175. info = items['mediaItems']['query']['results']['mediaObj'][0]
  176. # The 'meta' field is not always in the video webpage, we request it
  177. # from another page
  178. video_id = info['id']
  179. return self._get_info(video_id, display_id, webpage)
  180. def _get_info(self, video_id, display_id, webpage):
  181. region = self._search_regex(
  182. r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
  183. webpage, 'region', fatal=False, default='US')
  184. data = compat_urllib_parse.urlencode({
  185. 'protocol': 'http',
  186. 'region': region,
  187. })
  188. query_url = (
  189. 'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
  190. '{id}?{data}'.format(id=video_id, data=data))
  191. query_result = self._download_json(
  192. query_url, display_id, 'Downloading video info')
  193. info = query_result['query']['results']['mediaObj'][0]
  194. meta = info.get('meta')
  195. if not meta:
  196. msg = info['status'].get('msg')
  197. if msg:
  198. raise ExtractorError(
  199. '%s returned error: %s' % (self.IE_NAME, msg), expected=True)
  200. raise ExtractorError('Unable to extract media object meta')
  201. formats = []
  202. for s in info['streams']:
  203. format_info = {
  204. 'width': int_or_none(s.get('width')),
  205. 'height': int_or_none(s.get('height')),
  206. 'tbr': int_or_none(s.get('bitrate')),
  207. }
  208. host = s['host']
  209. path = s['path']
  210. if host.startswith('rtmp'):
  211. format_info.update({
  212. 'url': host,
  213. 'play_path': path,
  214. 'ext': 'flv',
  215. })
  216. else:
  217. format_url = compat_urlparse.urljoin(host, path)
  218. format_info['url'] = format_url
  219. formats.append(format_info)
  220. self._sort_formats(formats)
  221. return {
  222. 'id': video_id,
  223. 'display_id': display_id,
  224. 'title': unescapeHTML(meta['title']),
  225. 'formats': formats,
  226. 'description': clean_html(meta['description']),
  227. 'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
  228. 'duration': int_or_none(meta.get('duration')),
  229. }
  230. class YahooSearchIE(SearchInfoExtractor):
  231. IE_DESC = 'Yahoo screen search'
  232. _MAX_RESULTS = 1000
  233. IE_NAME = 'screen.yahoo:search'
  234. _SEARCH_KEY = 'yvsearch'
  235. def _get_n_results(self, query, n):
  236. """Get a specified number of results for a query"""
  237. entries = []
  238. for pagenum in itertools.count(0):
  239. result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
  240. info = self._download_json(result_url, query,
  241. note='Downloading results page ' + str(pagenum + 1))
  242. m = info['m']
  243. results = info['results']
  244. for (i, r) in enumerate(results):
  245. if (pagenum * 30) + i >= n:
  246. break
  247. mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
  248. e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
  249. entries.append(e)
  250. if (pagenum * 30 + i >= n) or (m['last'] >= (m['total'] - 1)):
  251. break
  252. return {
  253. '_type': 'playlist',
  254. 'id': query,
  255. 'entries': entries,
  256. }