You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

328 lines
12 KiB

  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import itertools
  5. from .common import InfoExtractor
  6. from ..utils import (
  7. compat_str,
  8. compat_urlparse,
  9. compat_urllib_parse,
  10. ExtractorError,
  11. int_or_none,
  12. unified_strdate,
  13. )
  14. class SoundcloudIE(InfoExtractor):
  15. """Information extractor for soundcloud.com
  16. To access the media, the uid of the song and a stream token
  17. must be extracted from the page source and the script must make
  18. a request to media.soundcloud.com/crossdomain.xml. Then
  19. the media can be grabbed by requesting from an url composed
  20. of the stream token and uid
  21. """
  22. _VALID_URL = r'''(?x)^(?:https?://)?
  23. (?:(?:(?:www\.|m\.)?soundcloud\.com/
  24. (?P<uploader>[\w\d-]+)/
  25. (?!sets/)(?P<title>[\w\d-]+)/?
  26. (?P<token>[^?]+?)?(?:[?].*)?$)
  27. |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+))
  28. |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
  29. )
  30. '''
  31. IE_NAME = 'soundcloud'
  32. _TESTS = [
  33. {
  34. 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
  35. 'file': '62986583.mp3',
  36. 'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
  37. 'info_dict': {
  38. "upload_date": "20121011",
  39. "description": "No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
  40. "uploader": "E.T. ExTerrestrial Music",
  41. "title": "Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1",
  42. "duration": 143,
  43. }
  44. },
  45. # not streamable song
  46. {
  47. 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
  48. 'info_dict': {
  49. 'id': '47127627',
  50. 'ext': 'mp3',
  51. 'title': 'Goldrushed',
  52. 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
  53. 'uploader': 'The Royal Concept',
  54. 'upload_date': '20120521',
  55. 'duration': 227,
  56. },
  57. 'params': {
  58. # rtmp
  59. 'skip_download': True,
  60. },
  61. },
  62. # private link
  63. {
  64. 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
  65. 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
  66. 'info_dict': {
  67. 'id': '123998367',
  68. 'ext': 'mp3',
  69. 'title': 'Youtube - Dl Test Video \'\' Ä↭',
  70. 'uploader': 'jaimeMF',
  71. 'description': 'test chars: \"\'/\\ä↭',
  72. 'upload_date': '20131209',
  73. 'duration': 9,
  74. },
  75. },
  76. # downloadable song
  77. {
  78. 'url': 'https://soundcloud.com/oddsamples/bus-brakes',
  79. 'md5': '7624f2351f8a3b2e7cd51522496e7631',
  80. 'info_dict': {
  81. 'id': '128590877',
  82. 'ext': 'mp3',
  83. 'title': 'Bus Brakes',
  84. 'description': 'md5:0170be75dd395c96025d210d261c784e',
  85. 'uploader': 'oddsamples',
  86. 'upload_date': '20140109',
  87. 'duration': 17,
  88. },
  89. },
  90. ]
  91. _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28'
  92. _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
  93. def report_resolve(self, video_id):
  94. """Report information extraction."""
  95. self.to_screen('%s: Resolving id' % video_id)
  96. @classmethod
  97. def _resolv_url(cls, url):
  98. return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
  99. def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
  100. track_id = compat_str(info['id'])
  101. name = full_title or track_id
  102. if quiet:
  103. self.report_extraction(name)
  104. thumbnail = info['artwork_url']
  105. if thumbnail is not None:
  106. thumbnail = thumbnail.replace('-large', '-t500x500')
  107. ext = 'mp3'
  108. result = {
  109. 'id': track_id,
  110. 'uploader': info['user']['username'],
  111. 'upload_date': unified_strdate(info['created_at']),
  112. 'title': info['title'],
  113. 'description': info['description'],
  114. 'thumbnail': thumbnail,
  115. 'duration': int_or_none(info.get('duration'), 1000),
  116. }
  117. formats = []
  118. if info.get('downloadable', False):
  119. # We can build a direct link to the song
  120. format_url = (
  121. 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(
  122. track_id, self._CLIENT_ID))
  123. formats.append({
  124. 'format_id': 'download',
  125. 'ext': info.get('original_format', 'mp3'),
  126. 'url': format_url,
  127. 'vcodec': 'none',
  128. 'preference': 10,
  129. })
  130. # We have to retrieve the url
  131. streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?'
  132. 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
  133. format_dict = self._download_json(
  134. streams_url,
  135. track_id, 'Downloading track url')
  136. for key, stream_url in format_dict.items():
  137. if key.startswith('http'):
  138. formats.append({
  139. 'format_id': key,
  140. 'ext': ext,
  141. 'url': stream_url,
  142. 'vcodec': 'none',
  143. })
  144. elif key.startswith('rtmp'):
  145. # The url doesn't have an rtmp app, we have to extract the playpath
  146. url, path = stream_url.split('mp3:', 1)
  147. formats.append({
  148. 'format_id': key,
  149. 'url': url,
  150. 'play_path': 'mp3:' + path,
  151. 'ext': ext,
  152. 'vcodec': 'none',
  153. })
  154. if not formats:
  155. # We fallback to the stream_url in the original info, this
  156. # cannot be always used, sometimes it can give an HTTP 404 error
  157. formats.append({
  158. 'format_id': 'fallback',
  159. 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
  160. 'ext': ext,
  161. 'vcodec': 'none',
  162. })
  163. for f in formats:
  164. if f['format_id'].startswith('http'):
  165. f['protocol'] = 'http'
  166. if f['format_id'].startswith('rtmp'):
  167. f['protocol'] = 'rtmp'
  168. self._sort_formats(formats)
  169. result['formats'] = formats
  170. return result
  171. def _real_extract(self, url):
  172. mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
  173. if mobj is None:
  174. raise ExtractorError('Invalid URL: %s' % url)
  175. track_id = mobj.group('track_id')
  176. token = None
  177. if track_id is not None:
  178. info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
  179. full_title = track_id
  180. elif mobj.group('player'):
  181. query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  182. return self.url_result(query['url'][0])
  183. else:
  184. # extract uploader (which is in the url)
  185. uploader = mobj.group('uploader')
  186. # extract simple title (uploader + slug of song title)
  187. slug_title = mobj.group('title')
  188. token = mobj.group('token')
  189. full_title = resolve_title = '%s/%s' % (uploader, slug_title)
  190. if token:
  191. resolve_title += '/%s' % token
  192. self.report_resolve(full_title)
  193. url = 'http://soundcloud.com/%s' % resolve_title
  194. info_json_url = self._resolv_url(url)
  195. info = self._download_json(info_json_url, full_title, 'Downloading info JSON')
  196. return self._extract_info_dict(info, full_title, secret_token=token)
  197. class SoundcloudSetIE(SoundcloudIE):
  198. _VALID_URL = r'https?://(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)'
  199. IE_NAME = 'soundcloud:set'
  200. # it's in tests/test_playlists.py
  201. _TESTS = []
  202. def _real_extract(self, url):
  203. mobj = re.match(self._VALID_URL, url)
  204. if mobj is None:
  205. raise ExtractorError('Invalid URL: %s' % url)
  206. # extract uploader (which is in the url)
  207. uploader = mobj.group(1)
  208. # extract simple title (uploader + slug of song title)
  209. slug_title = mobj.group(2)
  210. full_title = '%s/sets/%s' % (uploader, slug_title)
  211. self.report_resolve(full_title)
  212. url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
  213. resolv_url = self._resolv_url(url)
  214. info = self._download_json(resolv_url, full_title)
  215. if 'errors' in info:
  216. for err in info['errors']:
  217. self._downloader.report_error('unable to download video webpage: %s' % compat_str(err['error_message']))
  218. return
  219. self.report_extraction(full_title)
  220. return {'_type': 'playlist',
  221. 'entries': [self._extract_info_dict(track) for track in info['tracks']],
  222. 'id': info['id'],
  223. 'title': info['title'],
  224. }
  225. class SoundcloudUserIE(SoundcloudIE):
  226. _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
  227. IE_NAME = 'soundcloud:user'
  228. # it's in tests/test_playlists.py
  229. _TESTS = []
  230. def _real_extract(self, url):
  231. mobj = re.match(self._VALID_URL, url)
  232. uploader = mobj.group('user')
  233. resource = mobj.group('rsrc')
  234. if resource is None:
  235. resource = 'tracks'
  236. elif resource == 'likes':
  237. resource = 'favorites'
  238. url = 'http://soundcloud.com/%s/' % uploader
  239. resolv_url = self._resolv_url(url)
  240. user = self._download_json(
  241. resolv_url, uploader, 'Downloading user info')
  242. base_url = 'http://api.soundcloud.com/users/%s/%s.json?' % (uploader, resource)
  243. entries = []
  244. for i in itertools.count():
  245. data = compat_urllib_parse.urlencode({
  246. 'offset': i * 50,
  247. 'limit': 50,
  248. 'client_id': self._CLIENT_ID,
  249. })
  250. new_entries = self._download_json(
  251. base_url + data, uploader, 'Downloading track page %s' % (i + 1))
  252. if len(new_entries) == 0:
  253. self.to_screen('%s: End page received' % uploader)
  254. break
  255. entries.extend(self._extract_info_dict(e, quiet=True) for e in new_entries)
  256. return {
  257. '_type': 'playlist',
  258. 'id': compat_str(user['id']),
  259. 'title': user['username'],
  260. 'entries': entries,
  261. }
  262. class SoundcloudPlaylistIE(SoundcloudIE):
  263. _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)'
  264. IE_NAME = 'soundcloud:playlist'
  265. # it's in tests/test_playlists.py
  266. _TESTS = []
  267. def _real_extract(self, url):
  268. mobj = re.match(self._VALID_URL, url)
  269. playlist_id = mobj.group('id')
  270. base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id)
  271. data = compat_urllib_parse.urlencode({
  272. 'client_id': self._CLIENT_ID,
  273. })
  274. data = self._download_json(
  275. base_url + data, playlist_id, 'Downloading playlist')
  276. entries = [
  277. self._extract_info_dict(t, quiet=True) for t in data['tracks']]
  278. return {
  279. '_type': 'playlist',
  280. 'id': playlist_id,
  281. 'title': data.get('title'),
  282. 'description': data.get('description'),
  283. 'entries': entries,
  284. }