You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

382 lines
14 KiB

10 years ago
10 years ago
  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import itertools
  5. from .common import InfoExtractor
  6. from ..utils import (
  7. compat_str,
  8. compat_urlparse,
  9. compat_urllib_parse,
  10. ExtractorError,
  11. int_or_none,
  12. unified_strdate,
  13. )
  14. class SoundcloudIE(InfoExtractor):
  15. """Information extractor for soundcloud.com
  16. To access the media, the uid of the song and a stream token
  17. must be extracted from the page source and the script must make
  18. a request to media.soundcloud.com/crossdomain.xml. Then
  19. the media can be grabbed by requesting from an url composed
  20. of the stream token and uid
  21. """
  22. _VALID_URL = r'''(?x)^(?:https?://)?
  23. (?:(?:(?:www\.|m\.)?soundcloud\.com/
  24. (?P<uploader>[\w\d-]+)/
  25. (?!sets/|likes/?(?:$|[?#]))
  26. (?P<title>[\w\d-]+)/?
  27. (?P<token>[^?]+?)?(?:[?].*)?$)
  28. |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
  29. (?:/?\?secret_token=(?P<secret_token>[^&]+?))?$)
  30. |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
  31. )
  32. '''
  33. IE_NAME = 'soundcloud'
  34. _TESTS = [
  35. {
  36. 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
  37. 'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
  38. 'info_dict': {
  39. 'id': '62986583',
  40. 'ext': 'mp3',
  41. 'upload_date': '20121011',
  42. 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
  43. 'uploader': 'E.T. ExTerrestrial Music',
  44. 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
  45. 'duration': 143,
  46. }
  47. },
  48. # not streamable song
  49. {
  50. 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
  51. 'info_dict': {
  52. 'id': '47127627',
  53. 'ext': 'mp3',
  54. 'title': 'Goldrushed',
  55. 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
  56. 'uploader': 'The Royal Concept',
  57. 'upload_date': '20120521',
  58. 'duration': 227,
  59. },
  60. 'params': {
  61. # rtmp
  62. 'skip_download': True,
  63. },
  64. },
  65. # private link
  66. {
  67. 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
  68. 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
  69. 'info_dict': {
  70. 'id': '123998367',
  71. 'ext': 'mp3',
  72. 'title': 'Youtube - Dl Test Video \'\' Ä↭',
  73. 'uploader': 'jaimeMF',
  74. 'description': 'test chars: \"\'/\\ä↭',
  75. 'upload_date': '20131209',
  76. 'duration': 9,
  77. },
  78. },
  79. # private link (alt format)
  80. {
  81. 'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
  82. 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
  83. 'info_dict': {
  84. 'id': '123998367',
  85. 'ext': 'mp3',
  86. 'title': 'Youtube - Dl Test Video \'\' Ä↭',
  87. 'uploader': 'jaimeMF',
  88. 'description': 'test chars: \"\'/\\ä↭',
  89. 'upload_date': '20131209',
  90. 'duration': 9,
  91. },
  92. },
  93. # downloadable song
  94. {
  95. 'url': 'https://soundcloud.com/oddsamples/bus-brakes',
  96. 'md5': '7624f2351f8a3b2e7cd51522496e7631',
  97. 'info_dict': {
  98. 'id': '128590877',
  99. 'ext': 'mp3',
  100. 'title': 'Bus Brakes',
  101. 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
  102. 'uploader': 'oddsamples',
  103. 'upload_date': '20140109',
  104. 'duration': 17,
  105. },
  106. },
  107. ]
  108. _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28'
  109. _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
  110. def report_resolve(self, video_id):
  111. """Report information extraction."""
  112. self.to_screen('%s: Resolving id' % video_id)
  113. @classmethod
  114. def _resolv_url(cls, url):
  115. return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
  116. def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
  117. track_id = compat_str(info['id'])
  118. name = full_title or track_id
  119. if quiet:
  120. self.report_extraction(name)
  121. thumbnail = info['artwork_url']
  122. if thumbnail is not None:
  123. thumbnail = thumbnail.replace('-large', '-t500x500')
  124. ext = 'mp3'
  125. result = {
  126. 'id': track_id,
  127. 'uploader': info['user']['username'],
  128. 'upload_date': unified_strdate(info['created_at']),
  129. 'title': info['title'],
  130. 'description': info['description'],
  131. 'thumbnail': thumbnail,
  132. 'duration': int_or_none(info.get('duration'), 1000),
  133. 'webpage_url': info.get('permalink_url'),
  134. }
  135. formats = []
  136. if info.get('downloadable', False):
  137. # We can build a direct link to the song
  138. format_url = (
  139. 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(
  140. track_id, self._CLIENT_ID))
  141. formats.append({
  142. 'format_id': 'download',
  143. 'ext': info.get('original_format', 'mp3'),
  144. 'url': format_url,
  145. 'vcodec': 'none',
  146. 'preference': 10,
  147. })
  148. # We have to retrieve the url
  149. streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?'
  150. 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
  151. format_dict = self._download_json(
  152. streams_url,
  153. track_id, 'Downloading track url')
  154. for key, stream_url in format_dict.items():
  155. if key.startswith('http'):
  156. formats.append({
  157. 'format_id': key,
  158. 'ext': ext,
  159. 'url': stream_url,
  160. 'vcodec': 'none',
  161. })
  162. elif key.startswith('rtmp'):
  163. # The url doesn't have an rtmp app, we have to extract the playpath
  164. url, path = stream_url.split('mp3:', 1)
  165. formats.append({
  166. 'format_id': key,
  167. 'url': url,
  168. 'play_path': 'mp3:' + path,
  169. 'ext': ext,
  170. 'vcodec': 'none',
  171. })
  172. if not formats:
  173. # We fallback to the stream_url in the original info, this
  174. # cannot be always used, sometimes it can give an HTTP 404 error
  175. formats.append({
  176. 'format_id': 'fallback',
  177. 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
  178. 'ext': ext,
  179. 'vcodec': 'none',
  180. })
  181. for f in formats:
  182. if f['format_id'].startswith('http'):
  183. f['protocol'] = 'http'
  184. if f['format_id'].startswith('rtmp'):
  185. f['protocol'] = 'rtmp'
  186. self._sort_formats(formats)
  187. result['formats'] = formats
  188. return result
  189. def _real_extract(self, url):
  190. mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
  191. if mobj is None:
  192. raise ExtractorError('Invalid URL: %s' % url)
  193. track_id = mobj.group('track_id')
  194. token = None
  195. if track_id is not None:
  196. info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
  197. full_title = track_id
  198. token = mobj.group('secret_token')
  199. if token:
  200. info_json_url += "&secret_token=" + token
  201. elif mobj.group('player'):
  202. query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  203. return self.url_result(query['url'][0])
  204. else:
  205. # extract uploader (which is in the url)
  206. uploader = mobj.group('uploader')
  207. # extract simple title (uploader + slug of song title)
  208. slug_title = mobj.group('title')
  209. token = mobj.group('token')
  210. full_title = resolve_title = '%s/%s' % (uploader, slug_title)
  211. if token:
  212. resolve_title += '/%s' % token
  213. self.report_resolve(full_title)
  214. url = 'http://soundcloud.com/%s' % resolve_title
  215. info_json_url = self._resolv_url(url)
  216. info = self._download_json(info_json_url, full_title, 'Downloading info JSON')
  217. return self._extract_info_dict(info, full_title, secret_token=token)
  218. class SoundcloudSetIE(SoundcloudIE):
  219. _VALID_URL = r'https?://(?:www\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
  220. IE_NAME = 'soundcloud:set'
  221. _TESTS = [{
  222. 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
  223. 'info_dict': {
  224. 'title': 'The Royal Concept EP',
  225. },
  226. 'playlist_mincount': 6,
  227. }]
  228. def _real_extract(self, url):
  229. mobj = re.match(self._VALID_URL, url)
  230. # extract uploader (which is in the url)
  231. uploader = mobj.group('uploader')
  232. # extract simple title (uploader + slug of song title)
  233. slug_title = mobj.group('slug_title')
  234. full_title = '%s/sets/%s' % (uploader, slug_title)
  235. url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
  236. token = mobj.group('token')
  237. if token:
  238. full_title += '/' + token
  239. url += '/' + token
  240. self.report_resolve(full_title)
  241. resolv_url = self._resolv_url(url)
  242. info = self._download_json(resolv_url, full_title)
  243. if 'errors' in info:
  244. for err in info['errors']:
  245. self._downloader.report_error('unable to download video webpage: %s' % compat_str(err['error_message']))
  246. return
  247. return {
  248. '_type': 'playlist',
  249. 'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
  250. 'id': info['id'],
  251. 'title': info['title'],
  252. }
  253. class SoundcloudUserIE(SoundcloudIE):
  254. _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
  255. IE_NAME = 'soundcloud:user'
  256. _TESTS = [{
  257. 'url': 'https://soundcloud.com/the-concept-band',
  258. 'info_dict': {
  259. 'id': '9615865',
  260. 'title': 'The Royal Concept',
  261. },
  262. 'playlist_mincount': 12
  263. }, {
  264. 'url': 'https://soundcloud.com/the-concept-band/likes',
  265. 'info_dict': {
  266. 'id': '9615865',
  267. 'title': 'The Royal Concept',
  268. },
  269. 'playlist_mincount': 1,
  270. }]
  271. def _real_extract(self, url):
  272. mobj = re.match(self._VALID_URL, url)
  273. uploader = mobj.group('user')
  274. resource = mobj.group('rsrc')
  275. if resource is None:
  276. resource = 'tracks'
  277. elif resource == 'likes':
  278. resource = 'favorites'
  279. url = 'http://soundcloud.com/%s/' % uploader
  280. resolv_url = self._resolv_url(url)
  281. user = self._download_json(
  282. resolv_url, uploader, 'Downloading user info')
  283. base_url = 'http://api.soundcloud.com/users/%s/%s.json?' % (uploader, resource)
  284. entries = []
  285. for i in itertools.count():
  286. data = compat_urllib_parse.urlencode({
  287. 'offset': i * 50,
  288. 'limit': 50,
  289. 'client_id': self._CLIENT_ID,
  290. })
  291. new_entries = self._download_json(
  292. base_url + data, uploader, 'Downloading track page %s' % (i + 1))
  293. if len(new_entries) == 0:
  294. self.to_screen('%s: End page received' % uploader)
  295. break
  296. entries.extend(self._extract_info_dict(e, quiet=True) for e in new_entries)
  297. return {
  298. '_type': 'playlist',
  299. 'id': compat_str(user['id']),
  300. 'title': user['username'],
  301. 'entries': entries,
  302. }
  303. class SoundcloudPlaylistIE(SoundcloudIE):
  304. _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
  305. IE_NAME = 'soundcloud:playlist'
  306. _TESTS = [{
  307. 'url': 'http://api.soundcloud.com/playlists/4110309',
  308. 'info_dict': {
  309. 'id': '4110309',
  310. 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
  311. 'description': 're:.*?TILT Brass - Bowery Poetry Club',
  312. },
  313. 'playlist_count': 6,
  314. }]
  315. def _real_extract(self, url):
  316. mobj = re.match(self._VALID_URL, url)
  317. playlist_id = mobj.group('id')
  318. base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id)
  319. data_dict = {
  320. 'client_id': self._CLIENT_ID,
  321. }
  322. token = mobj.group('token')
  323. if token:
  324. data_dict['secret_token'] = token
  325. data = compat_urllib_parse.urlencode(data_dict)
  326. data = self._download_json(
  327. base_url + data, playlist_id, 'Downloading playlist')
  328. entries = [
  329. self._extract_info_dict(t, quiet=True, secret_token=token)
  330. for t in data['tracks']]
  331. return {
  332. '_type': 'playlist',
  333. 'id': playlist_id,
  334. 'title': data.get('title'),
  335. 'description': data.get('description'),
  336. 'entries': entries,
  337. }