You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

322 lines
12 KiB

  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import json
  4. import re
  5. import itertools
  6. from .common import InfoExtractor
  7. from ..utils import (
  8. compat_str,
  9. compat_urlparse,
  10. compat_urllib_parse,
  11. ExtractorError,
  12. int_or_none,
  13. unified_strdate,
  14. )
  15. class SoundcloudIE(InfoExtractor):
  16. """Information extractor for soundcloud.com
  17. To access the media, the uid of the song and a stream token
  18. must be extracted from the page source and the script must make
  19. a request to media.soundcloud.com/crossdomain.xml. Then
  20. the media can be grabbed by requesting from an url composed
  21. of the stream token and uid
  22. """
  23. _VALID_URL = r'''(?x)^(?:https?://)?
  24. (?:(?:(?:www\.|m\.)?soundcloud\.com/
  25. (?P<uploader>[\w\d-]+)/
  26. (?!sets/)(?P<title>[\w\d-]+)/?
  27. (?P<token>[^?]+?)?(?:[?].*)?$)
  28. |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+))
  29. |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
  30. )
  31. '''
  32. IE_NAME = 'soundcloud'
  33. _TESTS = [
  34. {
  35. 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
  36. 'file': '62986583.mp3',
  37. 'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
  38. 'info_dict': {
  39. "upload_date": "20121011",
  40. "description": "No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
  41. "uploader": "E.T. ExTerrestrial Music",
  42. "title": "Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1",
  43. "duration": 143,
  44. }
  45. },
  46. # not streamable song
  47. {
  48. 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
  49. 'info_dict': {
  50. 'id': '47127627',
  51. 'ext': 'mp3',
  52. 'title': 'Goldrushed',
  53. 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
  54. 'uploader': 'The Royal Concept',
  55. 'upload_date': '20120521',
  56. 'duration': 227,
  57. },
  58. 'params': {
  59. # rtmp
  60. 'skip_download': True,
  61. },
  62. },
  63. # private link
  64. {
  65. 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
  66. 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
  67. 'info_dict': {
  68. 'id': '123998367',
  69. 'ext': 'mp3',
  70. 'title': 'Youtube - Dl Test Video \'\' Ä↭',
  71. 'uploader': 'jaimeMF',
  72. 'description': 'test chars: \"\'/\\ä↭',
  73. 'upload_date': '20131209',
  74. 'duration': 9,
  75. },
  76. },
  77. # downloadable song
  78. {
  79. 'url': 'https://soundcloud.com/simgretina/just-your-problem-baby-1',
  80. 'md5': '56a8b69568acaa967b4c49f9d1d52d19',
  81. 'info_dict': {
  82. 'id': '105614606',
  83. 'ext': 'wav',
  84. 'title': 'Just Your Problem Baby (Acapella)',
  85. 'description': 'Vocals',
  86. 'uploader': 'Sim Gretina',
  87. 'upload_date': '20130815',
  88. #'duration': 42,
  89. },
  90. },
  91. ]
  92. _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28'
  93. _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
  94. def report_resolve(self, video_id):
  95. """Report information extraction."""
  96. self.to_screen('%s: Resolving id' % video_id)
  97. @classmethod
  98. def _resolv_url(cls, url):
  99. return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
  100. def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
  101. track_id = compat_str(info['id'])
  102. name = full_title or track_id
  103. if quiet:
  104. self.report_extraction(name)
  105. thumbnail = info['artwork_url']
  106. if thumbnail is not None:
  107. thumbnail = thumbnail.replace('-large', '-t500x500')
  108. ext = 'mp3'
  109. result = {
  110. 'id': track_id,
  111. 'uploader': info['user']['username'],
  112. 'upload_date': unified_strdate(info['created_at']),
  113. 'title': info['title'],
  114. 'description': info['description'],
  115. 'thumbnail': thumbnail,
  116. 'duration': int_or_none(info.get('duration'), 1000),
  117. }
  118. formats = []
  119. if info.get('downloadable', False):
  120. # We can build a direct link to the song
  121. format_url = (
  122. 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(
  123. track_id, self._CLIENT_ID))
  124. formats.append({
  125. 'format_id': 'download',
  126. 'ext': info.get('original_format', 'mp3'),
  127. 'url': format_url,
  128. 'vcodec': 'none',
  129. 'preference': 10,
  130. })
  131. # We have to retrieve the url
  132. streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?'
  133. 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
  134. format_dict = self._download_json(
  135. streams_url,
  136. track_id, 'Downloading track url')
  137. for key, stream_url in format_dict.items():
  138. if key.startswith('http'):
  139. formats.append({
  140. 'format_id': key,
  141. 'ext': ext,
  142. 'url': stream_url,
  143. 'vcodec': 'none',
  144. })
  145. elif key.startswith('rtmp'):
  146. # The url doesn't have an rtmp app, we have to extract the playpath
  147. url, path = stream_url.split('mp3:', 1)
  148. formats.append({
  149. 'format_id': key,
  150. 'url': url,
  151. 'play_path': 'mp3:' + path,
  152. 'ext': ext,
  153. 'vcodec': 'none',
  154. })
  155. if not formats:
  156. # We fallback to the stream_url in the original info, this
  157. # cannot be always used, sometimes it can give an HTTP 404 error
  158. formats.append({
  159. 'format_id': 'fallback',
  160. 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
  161. 'ext': ext,
  162. 'vcodec': 'none',
  163. })
  164. for f in formats:
  165. if f['format_id'].startswith('http'):
  166. f['protocol'] = 'http'
  167. if f['format_id'].startswith('rtmp'):
  168. f['protocol'] = 'rtmp'
  169. self._sort_formats(formats)
  170. result['formats'] = formats
  171. return result
  172. def _real_extract(self, url):
  173. mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
  174. if mobj is None:
  175. raise ExtractorError('Invalid URL: %s' % url)
  176. track_id = mobj.group('track_id')
  177. token = None
  178. if track_id is not None:
  179. info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
  180. full_title = track_id
  181. elif mobj.group('player'):
  182. query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  183. return self.url_result(query['url'][0])
  184. else:
  185. # extract uploader (which is in the url)
  186. uploader = mobj.group('uploader')
  187. # extract simple title (uploader + slug of song title)
  188. slug_title = mobj.group('title')
  189. token = mobj.group('token')
  190. full_title = resolve_title = '%s/%s' % (uploader, slug_title)
  191. if token:
  192. resolve_title += '/%s' % token
  193. self.report_resolve(full_title)
  194. url = 'http://soundcloud.com/%s' % resolve_title
  195. info_json_url = self._resolv_url(url)
  196. info = self._download_json(info_json_url, full_title, 'Downloading info JSON')
  197. return self._extract_info_dict(info, full_title, secret_token=token)
  198. class SoundcloudSetIE(SoundcloudIE):
  199. _VALID_URL = r'https?://(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)'
  200. IE_NAME = 'soundcloud:set'
  201. # it's in tests/test_playlists.py
  202. _TESTS = []
  203. def _real_extract(self, url):
  204. mobj = re.match(self._VALID_URL, url)
  205. if mobj is None:
  206. raise ExtractorError('Invalid URL: %s' % url)
  207. # extract uploader (which is in the url)
  208. uploader = mobj.group(1)
  209. # extract simple title (uploader + slug of song title)
  210. slug_title = mobj.group(2)
  211. full_title = '%s/sets/%s' % (uploader, slug_title)
  212. self.report_resolve(full_title)
  213. url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
  214. resolv_url = self._resolv_url(url)
  215. info = self._download_json(resolv_url, full_title)
  216. if 'errors' in info:
  217. for err in info['errors']:
  218. self._downloader.report_error('unable to download video webpage: %s' % compat_str(err['error_message']))
  219. return
  220. self.report_extraction(full_title)
  221. return {'_type': 'playlist',
  222. 'entries': [self._extract_info_dict(track) for track in info['tracks']],
  223. 'id': info['id'],
  224. 'title': info['title'],
  225. }
  226. class SoundcloudUserIE(SoundcloudIE):
  227. _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)(/?(tracks/)?)?(\?.*)?$'
  228. IE_NAME = 'soundcloud:user'
  229. # it's in tests/test_playlists.py
  230. _TESTS = []
  231. def _real_extract(self, url):
  232. mobj = re.match(self._VALID_URL, url)
  233. uploader = mobj.group('user')
  234. url = 'http://soundcloud.com/%s/' % uploader
  235. resolv_url = self._resolv_url(url)
  236. user = self._download_json(
  237. resolv_url, uploader, 'Downloading user info')
  238. base_url = 'http://api.soundcloud.com/users/%s/tracks.json?' % uploader
  239. entries = []
  240. for i in itertools.count():
  241. data = compat_urllib_parse.urlencode({
  242. 'offset': i * 50,
  243. 'client_id': self._CLIENT_ID,
  244. })
  245. new_entries = self._download_json(
  246. base_url + data, uploader, 'Downloading track page %s' % (i + 1))
  247. entries.extend(self._extract_info_dict(e, quiet=True) for e in new_entries)
  248. if len(new_entries) < 50:
  249. break
  250. return {
  251. '_type': 'playlist',
  252. 'id': compat_str(user['id']),
  253. 'title': user['username'],
  254. 'entries': entries,
  255. }
  256. class SoundcloudPlaylistIE(SoundcloudIE):
  257. _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)'
  258. IE_NAME = 'soundcloud:playlist'
  259. # it's in tests/test_playlists.py
  260. _TESTS = []
  261. def _real_extract(self, url):
  262. mobj = re.match(self._VALID_URL, url)
  263. playlist_id = mobj.group('id')
  264. base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id)
  265. data = compat_urllib_parse.urlencode({
  266. 'client_id': self._CLIENT_ID,
  267. })
  268. data = self._download_json(
  269. base_url + data, playlist_id, 'Downloading playlist')
  270. entries = [
  271. self._extract_info_dict(t, quiet=True) for t in data['tracks']]
  272. return {
  273. '_type': 'playlist',
  274. 'id': playlist_id,
  275. 'title': data.get('title'),
  276. 'description': data.get('description'),
  277. 'entries': entries,
  278. }