You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

249 lines
10 KiB

11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import itertools
  4. import json
  5. import re
  6. from .common import InfoExtractor, SearchInfoExtractor
  7. from ..utils import (
  8. ExtractorError,
  9. compat_urllib_parse,
  10. compat_urlparse,
  11. clean_html,
  12. int_or_none,
  13. )
  14. class YahooIE(InfoExtractor):
  15. IE_DESC = 'Yahoo screen and movies'
  16. _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+?)-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
  17. _TESTS = [
  18. {
  19. 'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
  20. 'md5': '4962b075c08be8690a922ee026d05e69',
  21. 'info_dict': {
  22. 'id': '2d25e626-2378-391f-ada0-ddaf1417e588',
  23. 'ext': 'mp4',
  24. 'title': 'Julian Smith & Travis Legg Watch Julian Smith',
  25. 'description': 'Julian and Travis watch Julian Smith',
  26. 'duration': 6863,
  27. },
  28. },
  29. {
  30. 'url': 'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
  31. 'md5': 'd6e6fc6e1313c608f316ddad7b82b306',
  32. 'info_dict': {
  33. 'id': 'd1dedf8c-d58c-38c3-8963-e899929ae0a9',
  34. 'ext': 'mp4',
  35. 'title': 'Codefellas - The Cougar Lies with Spanish Moss',
  36. 'description': 'md5:66b627ab0a282b26352136ca96ce73c1',
  37. 'duration': 151,
  38. },
  39. },
  40. {
  41. 'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
  42. 'md5': '60e8ac193d8fb71997caa8fce54c6460',
  43. 'info_dict': {
  44. 'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb',
  45. 'ext': 'mp4',
  46. 'title': "Yahoo Saves 'Community'",
  47. 'description': 'md5:4d4145af2fd3de00cbb6c1d664105053',
  48. 'duration': 170,
  49. }
  50. },
  51. {
  52. 'url': 'https://tw.screen.yahoo.com/taipei-opinion-poll/選情站報-街頭民調-台北市篇-102823042.html',
  53. 'md5': '92a7fdd8a08783c68a174d7aa067dde8',
  54. 'info_dict': {
  55. 'id': '7a23b569-7bea-36cb-85b9-bd5301a0a1fb',
  56. 'ext': 'mp4',
  57. 'title': '選情站報 街頭民調 台北市篇',
  58. 'description': '選情站報 街頭民調 台北市篇',
  59. 'duration': 429,
  60. }
  61. },
  62. {
  63. 'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
  64. 'md5': '0b51660361f0e27c9789e7037ef76f4b',
  65. 'info_dict': {
  66. 'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
  67. 'ext': 'mp4',
  68. 'title': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder',
  69. 'description': 'md5:f66c890e1490f4910a9953c941dee944',
  70. 'duration': 97,
  71. }
  72. },
  73. {
  74. 'url': 'https://ca.sports.yahoo.com/video/program-makes-hockey-more-affordable-013127711.html',
  75. 'md5': '57e06440778b1828a6079d2f744212c4',
  76. 'info_dict': {
  77. 'id': 'c9fa2a36-0d4d-3937-b8f6-cc0fb1881e73',
  78. 'ext': 'mp4',
  79. 'title': 'Program that makes hockey more affordable not offered in Manitoba',
  80. 'description': 'md5:c54a609f4c078d92b74ffb9bf1f496f4',
  81. 'duration': 121,
  82. }
  83. }, {
  84. 'url': 'https://ca.finance.yahoo.com/news/20-most-valuable-brands-world-112600775.html',
  85. 'md5': '3e401e4eed6325aa29d9b96125fd5b4f',
  86. 'info_dict': {
  87. 'id': 'c1b4c09c-8ed8-3b65-8b05-169c55358a83',
  88. 'ext': 'mp4',
  89. 'title': "Apple Is The World's Most Valuable Brand",
  90. 'description': 'md5:73eabc1a11c6f59752593b2ceefa1262',
  91. 'duration': 21,
  92. }
  93. }, {
  94. 'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
  95. 'md5': '67010fdf3a08d290e060a4dd96baa07b',
  96. 'info_dict': {
  97. 'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
  98. 'ext': 'mp4',
  99. 'title': 'China Moses Is Crazy About the Blues',
  100. 'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
  101. 'duration': 128,
  102. }
  103. }, {
  104. 'url': 'https://in.lifestyle.yahoo.com/video/connect-dots-dark-side-virgo-090247395.html',
  105. 'md5': 'd9a083ccf1379127bf25699d67e4791b',
  106. 'info_dict': {
  107. 'id': '52aeeaa3-b3d1-30d8-9ef8-5d0cf05efb7c',
  108. 'ext': 'mp4',
  109. 'title': 'Connect the Dots: Dark Side of Virgo',
  110. 'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
  111. 'duration': 201,
  112. }
  113. }, {
  114. 'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
  115. 'only_matching': True,
  116. }
  117. ]
  118. def _real_extract(self, url):
  119. mobj = re.match(self._VALID_URL, url)
  120. display_id = mobj.group('display_id')
  121. url = mobj.group('url')
  122. host = mobj.group('host')
  123. webpage = self._download_webpage(url, display_id)
  124. # Look for iframed media first
  125. iframe_m = re.search(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
  126. if iframe_m:
  127. iframepage = self._download_webpage(
  128. host + iframe_m.group(1), display_id, 'Downloading iframe webpage')
  129. items_json = self._search_regex(
  130. r'mediaItems: (\[.+?\])$', iframepage, 'items', flags=re.MULTILINE, default=None)
  131. if items_json:
  132. items = json.loads(items_json)
  133. video_id = items[0]['id']
  134. return self._get_info(video_id, display_id, webpage)
  135. items_json = self._search_regex(
  136. r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
  137. default=None)
  138. if items_json is None:
  139. CONTENT_ID_REGEXES = [
  140. r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
  141. r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
  142. r'"first_videoid"\s*:\s*"([^"]+)"',
  143. ]
  144. video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
  145. else:
  146. items = json.loads(items_json)
  147. info = items['mediaItems']['query']['results']['mediaObj'][0]
  148. # The 'meta' field is not always in the video webpage, we request it
  149. # from another page
  150. video_id = info['id']
  151. return self._get_info(video_id, display_id, webpage)
  152. def _get_info(self, video_id, display_id, webpage):
  153. region = self._search_regex(
  154. r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
  155. webpage, 'region', fatal=False, default='US')
  156. query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
  157. ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="%s"'
  158. ' AND protocol="http"' % (video_id, region))
  159. data = compat_urllib_parse.urlencode({
  160. 'q': query,
  161. 'env': 'prod',
  162. 'format': 'json',
  163. })
  164. query_result = self._download_json(
  165. 'http://video.query.yahoo.com/v1/public/yql?' + data,
  166. display_id, 'Downloading video info')
  167. info = query_result['query']['results']['mediaObj'][0]
  168. meta = info.get('meta')
  169. if not meta:
  170. msg = info['status'].get('msg')
  171. if msg:
  172. raise ExtractorError(
  173. '%s returned error: %s' % (self.IE_NAME, msg), expected=True)
  174. raise ExtractorError('Unable to extract media object meta')
  175. formats = []
  176. for s in info['streams']:
  177. format_info = {
  178. 'width': int_or_none(s.get('width')),
  179. 'height': int_or_none(s.get('height')),
  180. 'tbr': int_or_none(s.get('bitrate')),
  181. }
  182. host = s['host']
  183. path = s['path']
  184. if host.startswith('rtmp'):
  185. format_info.update({
  186. 'url': host,
  187. 'play_path': path,
  188. 'ext': 'flv',
  189. })
  190. else:
  191. format_url = compat_urlparse.urljoin(host, path)
  192. format_info['url'] = format_url
  193. formats.append(format_info)
  194. self._sort_formats(formats)
  195. return {
  196. 'id': video_id,
  197. 'display_id': display_id,
  198. 'title': meta['title'],
  199. 'formats': formats,
  200. 'description': clean_html(meta['description']),
  201. 'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
  202. 'duration': int_or_none(meta.get('duration')),
  203. }
  204. class YahooSearchIE(SearchInfoExtractor):
  205. IE_DESC = 'Yahoo screen search'
  206. _MAX_RESULTS = 1000
  207. IE_NAME = 'screen.yahoo:search'
  208. _SEARCH_KEY = 'yvsearch'
  209. def _get_n_results(self, query, n):
  210. """Get a specified number of results for a query"""
  211. entries = []
  212. for pagenum in itertools.count(0):
  213. result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
  214. info = self._download_json(result_url, query,
  215. note='Downloading results page ' + str(pagenum + 1))
  216. m = info['m']
  217. results = info['results']
  218. for (i, r) in enumerate(results):
  219. if (pagenum * 30) + i >= n:
  220. break
  221. mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
  222. e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
  223. entries.append(e)
  224. if (pagenum * 30 + i >= n) or (m['last'] >= (m['total'] - 1)):
  225. break
  226. return {
  227. '_type': 'playlist',
  228. 'id': query,
  229. 'entries': entries,
  230. }