You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

532 lines
19 KiB

10 years ago
10 years ago
  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import itertools
  5. from .common import (
  6. InfoExtractor,
  7. SearchInfoExtractor
  8. )
  9. from ..compat import (
  10. compat_str,
  11. compat_urlparse,
  12. compat_urllib_parse,
  13. )
  14. from ..utils import (
  15. encode_dict,
  16. ExtractorError,
  17. int_or_none,
  18. unified_strdate,
  19. )
  20. class SoundcloudIE(InfoExtractor):
  21. """Information extractor for soundcloud.com
  22. To access the media, the uid of the song and a stream token
  23. must be extracted from the page source and the script must make
  24. a request to media.soundcloud.com/crossdomain.xml. Then
  25. the media can be grabbed by requesting from an url composed
  26. of the stream token and uid
  27. """
  28. _VALID_URL = r'''(?x)^(?:https?://)?
  29. (?:(?:(?:www\.|m\.)?soundcloud\.com/
  30. (?P<uploader>[\w\d-]+)/
  31. (?!(?:tracks|sets(?:/[^/?#]+)?|reposts|likes|spotlight)/?(?:$|[?#]))
  32. (?P<title>[\w\d-]+)/?
  33. (?P<token>[^?]+?)?(?:[?].*)?$)
  34. |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
  35. (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
  36. |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
  37. )
  38. '''
  39. IE_NAME = 'soundcloud'
  40. _TESTS = [
  41. {
  42. 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
  43. 'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
  44. 'info_dict': {
  45. 'id': '62986583',
  46. 'ext': 'mp3',
  47. 'upload_date': '20121011',
  48. 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
  49. 'uploader': 'E.T. ExTerrestrial Music',
  50. 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
  51. 'duration': 143,
  52. }
  53. },
  54. # not streamable song
  55. {
  56. 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
  57. 'info_dict': {
  58. 'id': '47127627',
  59. 'ext': 'mp3',
  60. 'title': 'Goldrushed',
  61. 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
  62. 'uploader': 'The Royal Concept',
  63. 'upload_date': '20120521',
  64. 'duration': 227,
  65. },
  66. 'params': {
  67. # rtmp
  68. 'skip_download': True,
  69. },
  70. },
  71. # private link
  72. {
  73. 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
  74. 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
  75. 'info_dict': {
  76. 'id': '123998367',
  77. 'ext': 'mp3',
  78. 'title': 'Youtube - Dl Test Video \'\' Ä↭',
  79. 'uploader': 'jaimeMF',
  80. 'description': 'test chars: \"\'/\\ä↭',
  81. 'upload_date': '20131209',
  82. 'duration': 9,
  83. },
  84. },
  85. # private link (alt format)
  86. {
  87. 'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
  88. 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
  89. 'info_dict': {
  90. 'id': '123998367',
  91. 'ext': 'mp3',
  92. 'title': 'Youtube - Dl Test Video \'\' Ä↭',
  93. 'uploader': 'jaimeMF',
  94. 'description': 'test chars: \"\'/\\ä↭',
  95. 'upload_date': '20131209',
  96. 'duration': 9,
  97. },
  98. },
  99. # downloadable song
  100. {
  101. 'url': 'https://soundcloud.com/oddsamples/bus-brakes',
  102. 'md5': '7624f2351f8a3b2e7cd51522496e7631',
  103. 'info_dict': {
  104. 'id': '128590877',
  105. 'ext': 'mp3',
  106. 'title': 'Bus Brakes',
  107. 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
  108. 'uploader': 'oddsamples',
  109. 'upload_date': '20140109',
  110. 'duration': 17,
  111. },
  112. },
  113. ]
  114. _CLIENT_ID = '02gUJC0hH2ct1EGOcYXQIzRFU91c72Ea'
  115. _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
  116. def report_resolve(self, video_id):
  117. """Report information extraction."""
  118. self.to_screen('%s: Resolving id' % video_id)
  119. @classmethod
  120. def _resolv_url(cls, url):
  121. return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
  122. def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
  123. track_id = compat_str(info['id'])
  124. name = full_title or track_id
  125. if quiet:
  126. self.report_extraction(name)
  127. thumbnail = info['artwork_url']
  128. if thumbnail is not None:
  129. thumbnail = thumbnail.replace('-large', '-t500x500')
  130. ext = 'mp3'
  131. result = {
  132. 'id': track_id,
  133. 'uploader': info['user']['username'],
  134. 'upload_date': unified_strdate(info['created_at']),
  135. 'title': info['title'],
  136. 'description': info['description'],
  137. 'thumbnail': thumbnail,
  138. 'duration': int_or_none(info.get('duration'), 1000),
  139. 'webpage_url': info.get('permalink_url'),
  140. }
  141. formats = []
  142. if info.get('downloadable', False):
  143. # We can build a direct link to the song
  144. format_url = (
  145. 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(
  146. track_id, self._CLIENT_ID))
  147. formats.append({
  148. 'format_id': 'download',
  149. 'ext': info.get('original_format', 'mp3'),
  150. 'url': format_url,
  151. 'vcodec': 'none',
  152. 'preference': 10,
  153. })
  154. # We have to retrieve the url
  155. streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?'
  156. 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
  157. format_dict = self._download_json(
  158. streams_url,
  159. track_id, 'Downloading track url')
  160. for key, stream_url in format_dict.items():
  161. if key.startswith('http'):
  162. formats.append({
  163. 'format_id': key,
  164. 'ext': ext,
  165. 'url': stream_url,
  166. 'vcodec': 'none',
  167. })
  168. elif key.startswith('rtmp'):
  169. # The url doesn't have an rtmp app, we have to extract the playpath
  170. url, path = stream_url.split('mp3:', 1)
  171. formats.append({
  172. 'format_id': key,
  173. 'url': url,
  174. 'play_path': 'mp3:' + path,
  175. 'ext': 'flv',
  176. 'vcodec': 'none',
  177. })
  178. if not formats:
  179. # We fallback to the stream_url in the original info, this
  180. # cannot be always used, sometimes it can give an HTTP 404 error
  181. formats.append({
  182. 'format_id': 'fallback',
  183. 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
  184. 'ext': ext,
  185. 'vcodec': 'none',
  186. })
  187. for f in formats:
  188. if f['format_id'].startswith('http'):
  189. f['protocol'] = 'http'
  190. if f['format_id'].startswith('rtmp'):
  191. f['protocol'] = 'rtmp'
  192. self._check_formats(formats, track_id)
  193. self._sort_formats(formats)
  194. result['formats'] = formats
  195. return result
  196. def _real_extract(self, url):
  197. mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
  198. if mobj is None:
  199. raise ExtractorError('Invalid URL: %s' % url)
  200. track_id = mobj.group('track_id')
  201. token = None
  202. if track_id is not None:
  203. info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
  204. full_title = track_id
  205. token = mobj.group('secret_token')
  206. if token:
  207. info_json_url += "&secret_token=" + token
  208. elif mobj.group('player'):
  209. query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  210. real_url = query['url'][0]
  211. # If the token is in the query of the original url we have to
  212. # manually add it
  213. if 'secret_token' in query:
  214. real_url += '?secret_token=' + query['secret_token'][0]
  215. return self.url_result(real_url)
  216. else:
  217. # extract uploader (which is in the url)
  218. uploader = mobj.group('uploader')
  219. # extract simple title (uploader + slug of song title)
  220. slug_title = mobj.group('title')
  221. token = mobj.group('token')
  222. full_title = resolve_title = '%s/%s' % (uploader, slug_title)
  223. if token:
  224. resolve_title += '/%s' % token
  225. self.report_resolve(full_title)
  226. url = 'http://soundcloud.com/%s' % resolve_title
  227. info_json_url = self._resolv_url(url)
  228. info = self._download_json(info_json_url, full_title, 'Downloading info JSON')
  229. return self._extract_info_dict(info, full_title, secret_token=token)
  230. class SoundcloudSetIE(SoundcloudIE):
  231. _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
  232. IE_NAME = 'soundcloud:set'
  233. _TESTS = [{
  234. 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
  235. 'info_dict': {
  236. 'id': '2284613',
  237. 'title': 'The Royal Concept EP',
  238. },
  239. 'playlist_mincount': 6,
  240. }]
  241. def _real_extract(self, url):
  242. mobj = re.match(self._VALID_URL, url)
  243. # extract uploader (which is in the url)
  244. uploader = mobj.group('uploader')
  245. # extract simple title (uploader + slug of song title)
  246. slug_title = mobj.group('slug_title')
  247. full_title = '%s/sets/%s' % (uploader, slug_title)
  248. url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
  249. token = mobj.group('token')
  250. if token:
  251. full_title += '/' + token
  252. url += '/' + token
  253. self.report_resolve(full_title)
  254. resolv_url = self._resolv_url(url)
  255. info = self._download_json(resolv_url, full_title)
  256. if 'errors' in info:
  257. msgs = (compat_str(err['error_message']) for err in info['errors'])
  258. raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
  259. entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in info['tracks']]
  260. return {
  261. '_type': 'playlist',
  262. 'entries': entries,
  263. 'id': '%s' % info['id'],
  264. 'title': info['title'],
  265. }
  266. class SoundcloudUserIE(SoundcloudIE):
  267. _VALID_URL = r'''(?x)
  268. https?://
  269. (?:(?:www|m)\.)?soundcloud\.com/
  270. (?P<user>[^/]+)
  271. (?:/
  272. (?P<rsrc>tracks|sets|reposts|likes|spotlight)
  273. )?
  274. /?(?:[?#].*)?$
  275. '''
  276. IE_NAME = 'soundcloud:user'
  277. _TESTS = [{
  278. 'url': 'https://soundcloud.com/the-akashic-chronicler',
  279. 'info_dict': {
  280. 'id': '114582580',
  281. 'title': 'The Akashic Chronicler (All)',
  282. },
  283. 'playlist_mincount': 111,
  284. }, {
  285. 'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
  286. 'info_dict': {
  287. 'id': '114582580',
  288. 'title': 'The Akashic Chronicler (Tracks)',
  289. },
  290. 'playlist_mincount': 50,
  291. }, {
  292. 'url': 'https://soundcloud.com/the-akashic-chronicler/sets',
  293. 'info_dict': {
  294. 'id': '114582580',
  295. 'title': 'The Akashic Chronicler (Playlists)',
  296. },
  297. 'playlist_mincount': 3,
  298. }, {
  299. 'url': 'https://soundcloud.com/the-akashic-chronicler/reposts',
  300. 'info_dict': {
  301. 'id': '114582580',
  302. 'title': 'The Akashic Chronicler (Reposts)',
  303. },
  304. 'playlist_mincount': 7,
  305. }, {
  306. 'url': 'https://soundcloud.com/the-akashic-chronicler/likes',
  307. 'info_dict': {
  308. 'id': '114582580',
  309. 'title': 'The Akashic Chronicler (Likes)',
  310. },
  311. 'playlist_mincount': 321,
  312. }, {
  313. 'url': 'https://soundcloud.com/grynpyret/spotlight',
  314. 'info_dict': {
  315. 'id': '7098329',
  316. 'title': 'Grynpyret (Spotlight)',
  317. },
  318. 'playlist_mincount': 1,
  319. }]
  320. _API_BASE = 'https://api.soundcloud.com'
  321. _API_V2_BASE = 'https://api-v2.soundcloud.com'
  322. _BASE_URL_MAP = {
  323. 'all': '%s/profile/soundcloud:users:%%s' % _API_V2_BASE,
  324. 'tracks': '%s/users/%%s/tracks' % _API_BASE,
  325. 'sets': '%s/users/%%s/playlists' % _API_V2_BASE,
  326. 'reposts': '%s/profile/soundcloud:users:%%s/reposts' % _API_V2_BASE,
  327. 'likes': '%s/users/%%s/likes' % _API_V2_BASE,
  328. 'spotlight': '%s/users/%%s/spotlight' % _API_V2_BASE,
  329. }
  330. _TITLE_MAP = {
  331. 'all': 'All',
  332. 'tracks': 'Tracks',
  333. 'sets': 'Playlists',
  334. 'reposts': 'Reposts',
  335. 'likes': 'Likes',
  336. 'spotlight': 'Spotlight',
  337. }
  338. def _real_extract(self, url):
  339. mobj = re.match(self._VALID_URL, url)
  340. uploader = mobj.group('user')
  341. url = 'http://soundcloud.com/%s/' % uploader
  342. resolv_url = self._resolv_url(url)
  343. user = self._download_json(
  344. resolv_url, uploader, 'Downloading user info')
  345. resource = mobj.group('rsrc') or 'all'
  346. base_url = self._BASE_URL_MAP[resource] % user['id']
  347. COMMON_QUERY = {
  348. 'limit': 50,
  349. 'client_id': self._CLIENT_ID,
  350. 'linked_partitioning': '1',
  351. }
  352. query = COMMON_QUERY.copy()
  353. query['offset'] = 0
  354. next_href = base_url + '?' + compat_urllib_parse.urlencode(query)
  355. entries = []
  356. for i in itertools.count():
  357. response = self._download_json(
  358. next_href, uploader, 'Downloading track page %s' % (i + 1))
  359. collection = response['collection']
  360. if not collection:
  361. break
  362. def resolve_permalink_url(candidates):
  363. for cand in candidates:
  364. if isinstance(cand, dict):
  365. permalink_url = cand.get('permalink_url')
  366. if permalink_url and permalink_url.startswith('http'):
  367. return permalink_url
  368. for e in collection:
  369. permalink_url = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
  370. if permalink_url:
  371. entries.append(self.url_result(permalink_url))
  372. next_href = response.get('next_href')
  373. if not next_href:
  374. break
  375. parsed_next_href = compat_urlparse.urlparse(response['next_href'])
  376. qs = compat_urlparse.parse_qs(parsed_next_href.query)
  377. qs.update(COMMON_QUERY)
  378. next_href = compat_urlparse.urlunparse(
  379. parsed_next_href._replace(query=compat_urllib_parse.urlencode(qs, True)))
  380. return {
  381. '_type': 'playlist',
  382. 'id': compat_str(user['id']),
  383. 'title': '%s (%s)' % (user['username'], self._TITLE_MAP[resource]),
  384. 'entries': entries,
  385. }
  386. class SoundcloudPlaylistIE(SoundcloudIE):
  387. _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
  388. IE_NAME = 'soundcloud:playlist'
  389. _TESTS = [{
  390. 'url': 'http://api.soundcloud.com/playlists/4110309',
  391. 'info_dict': {
  392. 'id': '4110309',
  393. 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
  394. 'description': 're:.*?TILT Brass - Bowery Poetry Club',
  395. },
  396. 'playlist_count': 6,
  397. }]
  398. def _real_extract(self, url):
  399. mobj = re.match(self._VALID_URL, url)
  400. playlist_id = mobj.group('id')
  401. base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id)
  402. data_dict = {
  403. 'client_id': self._CLIENT_ID,
  404. }
  405. token = mobj.group('token')
  406. if token:
  407. data_dict['secret_token'] = token
  408. data = compat_urllib_parse.urlencode(data_dict)
  409. data = self._download_json(
  410. base_url + data, playlist_id, 'Downloading playlist')
  411. entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in data['tracks']]
  412. return {
  413. '_type': 'playlist',
  414. 'id': playlist_id,
  415. 'title': data.get('title'),
  416. 'description': data.get('description'),
  417. 'entries': entries,
  418. }
  419. class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
  420. IE_NAME = 'soundcloud:search'
  421. IE_DESC = 'Soundcloud search'
  422. _MAX_RESULTS = float('inf')
  423. _TESTS = [{
  424. 'url': 'scsearch15:post-avant jazzcore',
  425. 'info_dict': {
  426. 'title': 'post-avant jazzcore',
  427. },
  428. 'playlist_count': 15,
  429. }]
  430. _SEARCH_KEY = 'scsearch'
  431. _MAX_RESULTS_PER_PAGE = 200
  432. _DEFAULT_RESULTS_PER_PAGE = 50
  433. _API_V2_BASE = 'https://api-v2.soundcloud.com'
  434. def _get_collection(self, endpoint, collection_id, **query):
  435. limit = min(
  436. query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
  437. self._MAX_RESULTS_PER_PAGE)
  438. query['limit'] = limit
  439. query['client_id'] = self._CLIENT_ID
  440. query['linked_partitioning'] = '1'
  441. query['offset'] = 0
  442. data = compat_urllib_parse.urlencode(encode_dict(query))
  443. next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
  444. collected_results = 0
  445. for i in itertools.count(1):
  446. response = self._download_json(
  447. next_url, collection_id, 'Downloading page {0}'.format(i),
  448. 'Unable to download API page')
  449. collection = response.get('collection', [])
  450. if not collection:
  451. break
  452. collection = list(filter(bool, collection))
  453. collected_results += len(collection)
  454. for item in collection:
  455. yield self.url_result(item['uri'], SoundcloudIE.ie_key())
  456. if not collection or collected_results >= limit:
  457. break
  458. next_url = response.get('next_href')
  459. if not next_url:
  460. break
  461. def _get_n_results(self, query, n):
  462. tracks = self._get_collection('/search/tracks', query, limit=n, q=query)
  463. return self.playlist_result(tracks, playlist_title=query)