You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

262 lines
10 KiB

11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import itertools
  4. import json
  5. import re
  6. from .common import InfoExtractor, SearchInfoExtractor
  7. from ..compat import (
  8. compat_urllib_parse,
  9. compat_urlparse,
  10. )
  11. from ..utils import (
  12. clean_html,
  13. unescapeHTML,
  14. ExtractorError,
  15. int_or_none,
  16. )
  17. class YahooIE(InfoExtractor):
  18. IE_DESC = 'Yahoo screen and movies'
  19. _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+?)-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
  20. _TESTS = [
  21. {
  22. 'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
  23. 'md5': '4962b075c08be8690a922ee026d05e69',
  24. 'info_dict': {
  25. 'id': '2d25e626-2378-391f-ada0-ddaf1417e588',
  26. 'ext': 'mp4',
  27. 'title': 'Julian Smith & Travis Legg Watch Julian Smith',
  28. 'description': 'Julian and Travis watch Julian Smith',
  29. 'duration': 6863,
  30. },
  31. },
  32. {
  33. 'url': 'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
  34. 'md5': 'd6e6fc6e1313c608f316ddad7b82b306',
  35. 'info_dict': {
  36. 'id': 'd1dedf8c-d58c-38c3-8963-e899929ae0a9',
  37. 'ext': 'mp4',
  38. 'title': 'Codefellas - The Cougar Lies with Spanish Moss',
  39. 'description': 'md5:66b627ab0a282b26352136ca96ce73c1',
  40. 'duration': 151,
  41. },
  42. },
  43. {
  44. 'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
  45. 'md5': '60e8ac193d8fb71997caa8fce54c6460',
  46. 'info_dict': {
  47. 'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb',
  48. 'ext': 'mp4',
  49. 'title': "Yahoo Saves 'Community'",
  50. 'description': 'md5:4d4145af2fd3de00cbb6c1d664105053',
  51. 'duration': 170,
  52. }
  53. },
  54. {
  55. 'url': 'https://tw.screen.yahoo.com/election-2014-askmayor/敢問市長-黃秀霜批賴清德-非常高傲-033009720.html',
  56. 'md5': '3a09cf59349cfaddae1797acc3c087fc',
  57. 'info_dict': {
  58. 'id': 'cac903b3-fcf4-3c14-b632-643ab541712f',
  59. 'ext': 'mp4',
  60. 'title': '敢問市長/黃秀霜批賴清德「非常高傲」',
  61. 'description': '直言台南沒捷運 交通居五都之末',
  62. 'duration': 396,
  63. }
  64. },
  65. {
  66. 'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
  67. 'md5': '0b51660361f0e27c9789e7037ef76f4b',
  68. 'info_dict': {
  69. 'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
  70. 'ext': 'mp4',
  71. 'title': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder',
  72. 'description': 'md5:f66c890e1490f4910a9953c941dee944',
  73. 'duration': 97,
  74. }
  75. },
  76. {
  77. 'url': 'https://ca.sports.yahoo.com/video/program-makes-hockey-more-affordable-013127711.html',
  78. 'md5': '57e06440778b1828a6079d2f744212c4',
  79. 'info_dict': {
  80. 'id': 'c9fa2a36-0d4d-3937-b8f6-cc0fb1881e73',
  81. 'ext': 'mp4',
  82. 'title': 'Program that makes hockey more affordable not offered in Manitoba',
  83. 'description': 'md5:c54a609f4c078d92b74ffb9bf1f496f4',
  84. 'duration': 121,
  85. }
  86. }, {
  87. 'url': 'https://ca.finance.yahoo.com/news/hackers-sony-more-trouble-well-154609075.html',
  88. 'md5': '226a895aae7e21b0129e2a2006fe9690',
  89. 'info_dict': {
  90. 'id': 'e624c4bc-3389-34de-9dfc-025f74943409',
  91. 'ext': 'mp4',
  92. 'title': '\'The Interview\' TV Spot: War',
  93. 'description': 'The Interview',
  94. 'duration': 30,
  95. }
  96. }, {
  97. 'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
  98. 'md5': '67010fdf3a08d290e060a4dd96baa07b',
  99. 'info_dict': {
  100. 'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
  101. 'ext': 'mp4',
  102. 'title': 'China Moses Is Crazy About the Blues',
  103. 'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
  104. 'duration': 128,
  105. }
  106. }, {
  107. 'url': 'https://in.lifestyle.yahoo.com/video/connect-dots-dark-side-virgo-090247395.html',
  108. 'md5': 'd9a083ccf1379127bf25699d67e4791b',
  109. 'info_dict': {
  110. 'id': '52aeeaa3-b3d1-30d8-9ef8-5d0cf05efb7c',
  111. 'ext': 'mp4',
  112. 'title': 'Connect the Dots: Dark Side of Virgo',
  113. 'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
  114. 'duration': 201,
  115. }
  116. }, {
  117. 'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
  118. 'md5': '989396ae73d20c6f057746fb226aa215',
  119. 'info_dict': {
  120. 'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
  121. 'ext': 'mp4',
  122. 'title': '\'True Story\' Trailer',
  123. 'description': 'True Story',
  124. 'duration': 150,
  125. },
  126. }, {
  127. 'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
  128. 'only_matching': True,
  129. }
  130. ]
  131. def _real_extract(self, url):
  132. mobj = re.match(self._VALID_URL, url)
  133. display_id = mobj.group('display_id')
  134. page_id = mobj.group('id')
  135. url = mobj.group('url')
  136. host = mobj.group('host')
  137. webpage = self._download_webpage(url, display_id)
  138. # Look for iframed media first
  139. iframe_m = re.search(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
  140. if iframe_m:
  141. iframepage = self._download_webpage(
  142. host + iframe_m.group(1), display_id, 'Downloading iframe webpage')
  143. items_json = self._search_regex(
  144. r'mediaItems: (\[.+?\])$', iframepage, 'items', flags=re.MULTILINE, default=None)
  145. if items_json:
  146. items = json.loads(items_json)
  147. video_id = items[0]['id']
  148. return self._get_info(video_id, display_id, webpage)
  149. items_json = self._search_regex(
  150. r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
  151. default=None)
  152. if items_json is None:
  153. CONTENT_ID_REGEXES = [
  154. r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
  155. r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
  156. r'"first_videoid"\s*:\s*"([^"]+)"',
  157. r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
  158. ]
  159. video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
  160. else:
  161. items = json.loads(items_json)
  162. info = items['mediaItems']['query']['results']['mediaObj'][0]
  163. # The 'meta' field is not always in the video webpage, we request it
  164. # from another page
  165. video_id = info['id']
  166. return self._get_info(video_id, display_id, webpage)
  167. def _get_info(self, video_id, display_id, webpage):
  168. region = self._search_regex(
  169. r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
  170. webpage, 'region', fatal=False, default='US')
  171. data = compat_urllib_parse.urlencode({
  172. 'protocol': 'http',
  173. 'region': region,
  174. })
  175. query_url = (
  176. 'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
  177. '{id}?{data}'.format(id=video_id, data=data))
  178. query_result = self._download_json(
  179. query_url, display_id, 'Downloading video info')
  180. info = query_result['query']['results']['mediaObj'][0]
  181. meta = info.get('meta')
  182. if not meta:
  183. msg = info['status'].get('msg')
  184. if msg:
  185. raise ExtractorError(
  186. '%s returned error: %s' % (self.IE_NAME, msg), expected=True)
  187. raise ExtractorError('Unable to extract media object meta')
  188. formats = []
  189. for s in info['streams']:
  190. format_info = {
  191. 'width': int_or_none(s.get('width')),
  192. 'height': int_or_none(s.get('height')),
  193. 'tbr': int_or_none(s.get('bitrate')),
  194. }
  195. host = s['host']
  196. path = s['path']
  197. if host.startswith('rtmp'):
  198. format_info.update({
  199. 'url': host,
  200. 'play_path': path,
  201. 'ext': 'flv',
  202. })
  203. else:
  204. format_url = compat_urlparse.urljoin(host, path)
  205. format_info['url'] = format_url
  206. formats.append(format_info)
  207. self._sort_formats(formats)
  208. return {
  209. 'id': video_id,
  210. 'display_id': display_id,
  211. 'title': unescapeHTML(meta['title']),
  212. 'formats': formats,
  213. 'description': clean_html(meta['description']),
  214. 'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
  215. 'duration': int_or_none(meta.get('duration')),
  216. }
  217. class YahooSearchIE(SearchInfoExtractor):
  218. IE_DESC = 'Yahoo screen search'
  219. _MAX_RESULTS = 1000
  220. IE_NAME = 'screen.yahoo:search'
  221. _SEARCH_KEY = 'yvsearch'
  222. def _get_n_results(self, query, n):
  223. """Get a specified number of results for a query"""
  224. entries = []
  225. for pagenum in itertools.count(0):
  226. result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
  227. info = self._download_json(result_url, query,
  228. note='Downloading results page ' + str(pagenum + 1))
  229. m = info['m']
  230. results = info['results']
  231. for (i, r) in enumerate(results):
  232. if (pagenum * 30) + i >= n:
  233. break
  234. mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
  235. e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
  236. entries.append(e)
  237. if (pagenum * 30 + i >= n) or (m['last'] >= (m['total'] - 1)):
  238. break
  239. return {
  240. '_type': 'playlist',
  241. 'id': query,
  242. 'entries': entries,
  243. }