You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

158 lines
5.6 KiB

  1. import itertools
  2. import json
  3. import re
  4. from .common import InfoExtractor, SearchInfoExtractor
  5. from ..utils import (
  6. compat_urllib_parse,
  7. compat_urlparse,
  8. clean_html,
  9. int_or_none,
  10. )
  11. class YahooIE(InfoExtractor):
  12. IE_DESC = u'Yahoo screen'
  13. _VALID_URL = r'http://screen\.yahoo\.com/.*?-(?P<id>\d*?)\.html'
  14. _TESTS = [
  15. {
  16. u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
  17. u'file': u'214727115.mp4',
  18. u'md5': u'4962b075c08be8690a922ee026d05e69',
  19. u'info_dict': {
  20. u'title': u'Julian Smith & Travis Legg Watch Julian Smith',
  21. u'description': u'Julian and Travis watch Julian Smith',
  22. },
  23. },
  24. {
  25. u'url': u'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
  26. u'file': u'103000935.mp4',
  27. u'md5': u'd6e6fc6e1313c608f316ddad7b82b306',
  28. u'info_dict': {
  29. u'title': u'Codefellas - The Cougar Lies with Spanish Moss',
  30. u'description': u'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?',
  31. },
  32. },
  33. ]
  34. def _real_extract(self, url):
  35. mobj = re.match(self._VALID_URL, url)
  36. video_id = mobj.group('id')
  37. webpage = self._download_webpage(url, video_id)
  38. items_json = self._search_regex(r'mediaItems: ({.*?})$',
  39. webpage, u'items', flags=re.MULTILINE)
  40. items = json.loads(items_json)
  41. info = items['mediaItems']['query']['results']['mediaObj'][0]
  42. # The 'meta' field is not always in the video webpage, we request it
  43. # from another page
  44. long_id = info['id']
  45. return self._get_info(long_id, video_id)
  46. def _get_info(self, long_id, video_id):
  47. query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
  48. ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="US"'
  49. ' AND protocol="http"' % long_id)
  50. data = compat_urllib_parse.urlencode({
  51. 'q': query,
  52. 'env': 'prod',
  53. 'format': 'json',
  54. })
  55. query_result_json = self._download_webpage(
  56. 'http://video.query.yahoo.com/v1/public/yql?' + data,
  57. video_id, u'Downloading video info')
  58. query_result = json.loads(query_result_json)
  59. info = query_result['query']['results']['mediaObj'][0]
  60. meta = info['meta']
  61. formats = []
  62. for s in info['streams']:
  63. format_info = {
  64. 'width': int_or_none(s.get('width')),
  65. 'height': int_or_none(s.get('height')),
  66. 'tbr': int_or_none(s.get('bitrate')),
  67. }
  68. host = s['host']
  69. path = s['path']
  70. if host.startswith('rtmp'):
  71. format_info.update({
  72. 'url': host,
  73. 'play_path': path,
  74. 'ext': 'flv',
  75. })
  76. else:
  77. format_url = compat_urlparse.urljoin(host, path)
  78. format_info['url'] = format_url
  79. formats.append(format_info)
  80. self._sort_formats(formats)
  81. return {
  82. 'id': video_id,
  83. 'title': meta['title'],
  84. 'formats': formats,
  85. 'description': clean_html(meta['description']),
  86. 'thumbnail': meta['thumbnail'],
  87. }
  88. class YahooNewsIE(YahooIE):
  89. IE_NAME = 'yahoo:news'
  90. _VALID_URL = r'http://news\.yahoo\.com/video/.*?-(?P<id>\d*?)\.html'
  91. _TEST = {
  92. u'url': u'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
  93. u'md5': u'67010fdf3a08d290e060a4dd96baa07b',
  94. u'info_dict': {
  95. u'id': u'104538833',
  96. u'ext': u'mp4',
  97. u'title': u'China Moses Is Crazy About the Blues',
  98. u'description': u'md5:9900ab8cd5808175c7b3fe55b979bed0',
  99. },
  100. }
  101. # Overwrite YahooIE properties we don't want
  102. _TESTS = []
  103. def _real_extract(self, url):
  104. mobj = re.match(self._VALID_URL, url)
  105. video_id = mobj.group('id')
  106. webpage = self._download_webpage(url, video_id)
  107. long_id = self._search_regex(r'contentId: \'(.+?)\',', webpage, u'long id')
  108. return self._get_info(long_id, video_id)
  109. class YahooSearchIE(SearchInfoExtractor):
  110. IE_DESC = u'Yahoo screen search'
  111. _MAX_RESULTS = 1000
  112. IE_NAME = u'screen.yahoo:search'
  113. _SEARCH_KEY = 'yvsearch'
  114. def _get_n_results(self, query, n):
  115. """Get a specified number of results for a query"""
  116. res = {
  117. '_type': 'playlist',
  118. 'id': query,
  119. 'entries': []
  120. }
  121. for pagenum in itertools.count(0):
  122. result_url = u'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
  123. webpage = self._download_webpage(result_url, query,
  124. note='Downloading results page '+str(pagenum+1))
  125. info = json.loads(webpage)
  126. m = info[u'm']
  127. results = info[u'results']
  128. for (i, r) in enumerate(results):
  129. if (pagenum * 30) +i >= n:
  130. break
  131. mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
  132. e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
  133. res['entries'].append(e)
  134. if (pagenum * 30 +i >= n) or (m[u'last'] >= (m[u'total'] -1)):
  135. break
  136. return res