You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

242 lines
8.7 KiB

  1. from __future__ import unicode_literals
  2. import re
  3. from .common import InfoExtractor
  4. from .once import OnceIE
  5. from ..compat import compat_str
  6. from ..utils import (
  7. determine_ext,
  8. int_or_none,
  9. unified_timestamp,
  10. )
  11. class ESPNIE(OnceIE):
  12. _VALID_URL = r'''(?x)
  13. https?://
  14. (?:
  15. (?:
  16. (?:
  17. (?:(?:\w+\.)+)?espn\.go|
  18. (?:www\.)?espn
  19. )\.com/
  20. (?:
  21. (?:
  22. video/(?:clip|iframe/twitter)|
  23. watch/player
  24. )
  25. (?:
  26. .*?\?.*?\bid=|
  27. /_/id/
  28. )|
  29. [^/]+/video/
  30. )
  31. )|
  32. (?:www\.)espnfc\.(?:com|us)/(?:video/)?[^/]+/\d+/video/
  33. )
  34. (?P<id>\d+)
  35. '''
  36. _TESTS = [{
  37. 'url': 'http://espn.go.com/video/clip?id=10365079',
  38. 'info_dict': {
  39. 'id': '10365079',
  40. 'ext': 'mp4',
  41. 'title': '30 for 30 Shorts: Judging Jewell',
  42. 'description': 'md5:39370c2e016cb4ecf498ffe75bef7f0f',
  43. 'timestamp': 1390936111,
  44. 'upload_date': '20140128',
  45. },
  46. 'params': {
  47. 'skip_download': True,
  48. },
  49. }, {
  50. 'url': 'https://broadband.espn.go.com/video/clip?id=18910086',
  51. 'info_dict': {
  52. 'id': '18910086',
  53. 'ext': 'mp4',
  54. 'title': 'Kyrie spins around defender for two',
  55. 'description': 'md5:2b0f5bae9616d26fba8808350f0d2b9b',
  56. 'timestamp': 1489539155,
  57. 'upload_date': '20170315',
  58. },
  59. 'params': {
  60. 'skip_download': True,
  61. },
  62. 'expected_warnings': ['Unable to download f4m manifest'],
  63. }, {
  64. 'url': 'http://nonredline.sports.espn.go.com/video/clip?id=19744672',
  65. 'only_matching': True,
  66. }, {
  67. 'url': 'https://cdn.espn.go.com/video/clip/_/id/19771774',
  68. 'only_matching': True,
  69. }, {
  70. 'url': 'http://www.espn.com/watch/player?id=19141491',
  71. 'only_matching': True,
  72. }, {
  73. 'url': 'http://www.espn.com/watch/player?bucketId=257&id=19505875',
  74. 'only_matching': True,
  75. }, {
  76. 'url': 'http://www.espn.com/watch/player/_/id/19141491',
  77. 'only_matching': True,
  78. }, {
  79. 'url': 'http://www.espn.com/video/clip?id=10365079',
  80. 'only_matching': True,
  81. }, {
  82. 'url': 'http://www.espn.com/video/clip/_/id/17989860',
  83. 'only_matching': True,
  84. }, {
  85. 'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
  86. 'only_matching': True,
  87. }, {
  88. 'url': 'http://www.espnfc.us/video/espn-fc-tv/86/video/3319154/nashville-unveiled-as-the-newest-club-in-mls',
  89. 'only_matching': True,
  90. }, {
  91. 'url': 'http://www.espnfc.com/english-premier-league/23/video/3324163/premier-league-in-90-seconds-golden-tweets',
  92. 'only_matching': True,
  93. }, {
  94. 'url': 'http://www.espn.com/espnw/video/26066627/arkansas-gibson-completes-hr-cycle-four-innings',
  95. 'only_matching': True,
  96. }]
  97. def _real_extract(self, url):
  98. video_id = self._match_id(url)
  99. clip = self._download_json(
  100. 'http://api-app.espn.com/v1/video/clips/%s' % video_id,
  101. video_id)['videos'][0]
  102. title = clip['headline']
  103. format_urls = set()
  104. formats = []
  105. def traverse_source(source, base_source_id=None):
  106. for source_id, source in source.items():
  107. if source_id == 'alert':
  108. continue
  109. elif isinstance(source, compat_str):
  110. extract_source(source, base_source_id)
  111. elif isinstance(source, dict):
  112. traverse_source(
  113. source,
  114. '%s-%s' % (base_source_id, source_id)
  115. if base_source_id else source_id)
  116. def extract_source(source_url, source_id=None):
  117. if source_url in format_urls:
  118. return
  119. format_urls.add(source_url)
  120. ext = determine_ext(source_url)
  121. if OnceIE.suitable(source_url):
  122. formats.extend(self._extract_once_formats(source_url))
  123. elif ext == 'smil':
  124. formats.extend(self._extract_smil_formats(
  125. source_url, video_id, fatal=False))
  126. elif ext == 'f4m':
  127. formats.extend(self._extract_f4m_formats(
  128. source_url, video_id, f4m_id=source_id, fatal=False))
  129. elif ext == 'm3u8':
  130. formats.extend(self._extract_m3u8_formats(
  131. source_url, video_id, 'mp4', entry_protocol='m3u8_native',
  132. m3u8_id=source_id, fatal=False))
  133. else:
  134. f = {
  135. 'url': source_url,
  136. 'format_id': source_id,
  137. }
  138. mobj = re.search(r'(\d+)p(\d+)_(\d+)k\.', source_url)
  139. if mobj:
  140. f.update({
  141. 'height': int(mobj.group(1)),
  142. 'fps': int(mobj.group(2)),
  143. 'tbr': int(mobj.group(3)),
  144. })
  145. if source_id == 'mezzanine':
  146. f['preference'] = 1
  147. formats.append(f)
  148. links = clip.get('links', {})
  149. traverse_source(links.get('source', {}))
  150. traverse_source(links.get('mobile', {}))
  151. self._sort_formats(formats)
  152. description = clip.get('caption') or clip.get('description')
  153. thumbnail = clip.get('thumbnail')
  154. duration = int_or_none(clip.get('duration'))
  155. timestamp = unified_timestamp(clip.get('originalPublishDate'))
  156. return {
  157. 'id': video_id,
  158. 'title': title,
  159. 'description': description,
  160. 'thumbnail': thumbnail,
  161. 'timestamp': timestamp,
  162. 'duration': duration,
  163. 'formats': formats,
  164. }
  165. class ESPNArticleIE(InfoExtractor):
  166. _VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/(?:[^/]+/)*(?P<id>[^/]+)'
  167. _TESTS = [{
  168. 'url': 'http://espn.go.com/nba/recap?gameId=400793786',
  169. 'only_matching': True,
  170. }, {
  171. 'url': 'http://espn.go.com/blog/golden-state-warriors/post/_/id/593/how-warriors-rapidly-regained-a-winning-edge',
  172. 'only_matching': True,
  173. }, {
  174. 'url': 'http://espn.go.com/sports/endurance/story/_/id/12893522/dzhokhar-tsarnaev-sentenced-role-boston-marathon-bombings',
  175. 'only_matching': True,
  176. }, {
  177. 'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
  178. 'only_matching': True,
  179. }]
  180. @classmethod
  181. def suitable(cls, url):
  182. return False if ESPNIE.suitable(url) else super(ESPNArticleIE, cls).suitable(url)
  183. def _real_extract(self, url):
  184. video_id = self._match_id(url)
  185. webpage = self._download_webpage(url, video_id)
  186. video_id = self._search_regex(
  187. r'class=(["\']).*?video-play-button.*?\1[^>]+data-id=["\'](?P<id>\d+)',
  188. webpage, 'video id', group='id')
  189. return self.url_result(
  190. 'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())
  191. class FiveThirtyEightIE(InfoExtractor):
  192. _VALID_URL = r'https?://(?:www\.)?fivethirtyeight\.com/features/(?P<id>[^/?#]+)'
  193. _TEST = {
  194. 'url': 'http://fivethirtyeight.com/features/how-the-6-8-raiders-can-still-make-the-playoffs/',
  195. 'info_dict': {
  196. 'id': '21846851',
  197. 'ext': 'mp4',
  198. 'title': 'FiveThirtyEight: The Raiders can still make the playoffs',
  199. 'description': 'Neil Paine breaks down the simplest scenario that will put the Raiders into the playoffs at 8-8.',
  200. 'timestamp': 1513960621,
  201. 'upload_date': '20171222',
  202. },
  203. 'params': {
  204. 'skip_download': True,
  205. },
  206. 'expected_warnings': ['Unable to download f4m manifest'],
  207. }
  208. def _real_extract(self, url):
  209. video_id = self._match_id(url)
  210. webpage = self._download_webpage(url, video_id)
  211. video_id = self._search_regex(
  212. r'data-video-id=["\'](?P<id>\d+)',
  213. webpage, 'video id', group='id')
  214. return self.url_result(
  215. 'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())