You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

138 lines
4.9 KiB

  1. import itertools
  2. import json
  3. import re
  4. from .common import InfoExtractor, SearchInfoExtractor
  5. from ..utils import (
  6. compat_urllib_parse,
  7. compat_urlparse,
  8. determine_ext,
  9. clean_html,
  10. )
  11. class YahooIE(InfoExtractor):
  12. IE_DESC = u'Yahoo screen'
  13. _VALID_URL = r'http://screen\.yahoo\.com/.*?-(?P<id>\d*?)\.html'
  14. _TESTS = [
  15. {
  16. u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
  17. u'file': u'214727115.flv',
  18. u'info_dict': {
  19. u'title': u'Julian Smith & Travis Legg Watch Julian Smith',
  20. u'description': u'Julian and Travis watch Julian Smith',
  21. },
  22. u'params': {
  23. # Requires rtmpdump
  24. u'skip_download': True,
  25. },
  26. },
  27. {
  28. u'url': u'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
  29. u'file': u'103000935.flv',
  30. u'info_dict': {
  31. u'title': u'Codefellas - The Cougar Lies with Spanish Moss',
  32. u'description': u'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?',
  33. },
  34. u'params': {
  35. # Requires rtmpdump
  36. u'skip_download': True,
  37. },
  38. },
  39. ]
  40. def _real_extract(self, url):
  41. mobj = re.match(self._VALID_URL, url)
  42. video_id = mobj.group('id')
  43. webpage = self._download_webpage(url, video_id)
  44. items_json = self._search_regex(r'YVIDEO_INIT_ITEMS = ({.*?});$',
  45. webpage, u'items', flags=re.MULTILINE)
  46. items = json.loads(items_json)
  47. info = items['mediaItems']['query']['results']['mediaObj'][0]
  48. # The 'meta' field is not always in the video webpage, we request it
  49. # from another page
  50. long_id = info['id']
  51. query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
  52. ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2"' % long_id)
  53. data = compat_urllib_parse.urlencode({
  54. 'q': query,
  55. 'env': 'prod',
  56. 'format': 'json',
  57. })
  58. query_result_json = self._download_webpage(
  59. 'http://video.query.yahoo.com/v1/public/yql?' + data,
  60. video_id, u'Downloading video info')
  61. query_result = json.loads(query_result_json)
  62. info = query_result['query']['results']['mediaObj'][0]
  63. meta = info['meta']
  64. formats = []
  65. for s in info['streams']:
  66. format_info = {
  67. 'width': s.get('width'),
  68. 'height': s.get('height'),
  69. 'bitrate': s.get('bitrate'),
  70. }
  71. host = s['host']
  72. path = s['path']
  73. if host.startswith('rtmp'):
  74. format_info.update({
  75. 'url': host,
  76. 'play_path': path,
  77. 'ext': 'flv',
  78. })
  79. else:
  80. format_url = compat_urlparse.urljoin(host, path)
  81. format_info['url'] = format_url
  82. format_info['ext'] = determine_ext(format_url)
  83. formats.append(format_info)
  84. formats = sorted(formats, key=lambda f:(f['height'], f['width']))
  85. info = {
  86. 'id': video_id,
  87. 'title': meta['title'],
  88. 'formats': formats,
  89. 'description': clean_html(meta['description']),
  90. 'thumbnail': meta['thumbnail'],
  91. }
  92. # TODO: Remove when #980 has been merged
  93. info.update(formats[-1])
  94. return info
  95. class YahooSearchIE(SearchInfoExtractor):
  96. IE_DESC = u'Yahoo screen search'
  97. _MAX_RESULTS = 1000
  98. IE_NAME = u'screen.yahoo:search'
  99. _SEARCH_KEY = 'yvsearch'
  100. def _get_n_results(self, query, n):
  101. """Get a specified number of results for a query"""
  102. res = {
  103. '_type': 'playlist',
  104. 'id': query,
  105. 'entries': []
  106. }
  107. for pagenum in itertools.count(0):
  108. result_url = u'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
  109. webpage = self._download_webpage(result_url, query,
  110. note='Downloading results page '+str(pagenum+1))
  111. info = json.loads(webpage)
  112. m = info[u'm']
  113. results = info[u'results']
  114. for (i, r) in enumerate(results):
  115. if (pagenum * 30) +i >= n:
  116. break
  117. mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
  118. e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
  119. res['entries'].append(e)
  120. if (pagenum * 30 +i >= n) or (m[u'last'] >= (m[u'total'] -1 )):
  121. break
  122. return res