You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

614 lines
27 KiB

11 years ago
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. from __future__ import absolute_import
  4. import io
  5. import os
  6. import re
  7. import shutil
  8. import socket
  9. import sys
  10. import time
  11. import traceback
  12. from .utils import *
  13. from .extractor import get_info_extractor, gen_extractors
  14. from .FileDownloader import FileDownloader
  15. class YoutubeDL(object):
  16. """YoutubeDL class.
  17. YoutubeDL objects are the ones responsible of downloading the
  18. actual video file and writing it to disk if the user has requested
  19. it, among some other tasks. In most cases there should be one per
  20. program. As, given a video URL, the downloader doesn't know how to
  21. extract all the needed information, task that InfoExtractors do, it
  22. has to pass the URL to one of them.
  23. For this, YoutubeDL objects have a method that allows
  24. InfoExtractors to be registered in a given order. When it is passed
  25. a URL, the YoutubeDL object handles it to the first InfoExtractor it
  26. finds that reports being able to handle it. The InfoExtractor extracts
  27. all the information about the video or videos the URL refers to, and
  28. YoutubeDL process the extracted information, possibly using a File
  29. Downloader to download the video.
  30. YoutubeDL objects accept a lot of parameters. In order not to saturate
  31. the object constructor with arguments, it receives a dictionary of
  32. options instead. These options are available through the params
  33. attribute for the InfoExtractors to use. The YoutubeDL also
  34. registers itself as the downloader in charge for the InfoExtractors
  35. that are added to it, so this is a "mutual registration".
  36. Available options:
  37. username: Username for authentication purposes.
  38. password: Password for authentication purposes.
  39. videopassword: Password for acces a video.
  40. usenetrc: Use netrc for authentication instead.
  41. verbose: Print additional info to stdout.
  42. quiet: Do not print messages to stdout.
  43. forceurl: Force printing final URL.
  44. forcetitle: Force printing title.
  45. forceid: Force printing ID.
  46. forcethumbnail: Force printing thumbnail URL.
  47. forcedescription: Force printing description.
  48. forcefilename: Force printing final filename.
  49. simulate: Do not download the video files.
  50. format: Video format code.
  51. format_limit: Highest quality format to try.
  52. outtmpl: Template for output names.
  53. restrictfilenames: Do not allow "&" and spaces in file names
  54. ignoreerrors: Do not stop on download errors.
  55. nooverwrites: Prevent overwriting files.
  56. playliststart: Playlist item to start at.
  57. playlistend: Playlist item to end at.
  58. matchtitle: Download only matching titles.
  59. rejecttitle: Reject downloads for matching titles.
  60. logtostderr: Log messages to stderr instead of stdout.
  61. writedescription: Write the video description to a .description file
  62. writeinfojson: Write the video description to a .info.json file
  63. writethumbnail: Write the thumbnail image to a file
  64. writesubtitles: Write the video subtitles to a file
  65. writeautomaticsub: Write the automatic subtitles to a file
  66. allsubtitles: Downloads all the subtitles of the video
  67. (requires writesubtitles or writeautomaticsub)
  68. listsubtitles: Lists all available subtitles for the video
  69. subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
  70. subtitleslangs: List of languages of the subtitles to download
  71. keepvideo: Keep the video file after post-processing
  72. daterange: A DateRange object, download only if the upload_date is in the range.
  73. skip_download: Skip the actual download of the video file
  74. cachedir: Location of the cache files in the filesystem.
  75. "NONE" to disable filesystem cache.
  76. The following parameters are not used by YoutubeDL itself, they are used by
  77. the FileDownloader:
  78. nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
  79. noresizebuffer, retries, continuedl, noprogress, consoletitle
  80. """
  81. params = None
  82. _ies = []
  83. _pps = []
  84. _download_retcode = None
  85. _num_downloads = None
  86. _screen_file = None
  87. def __init__(self, params):
  88. """Create a FileDownloader object with the given options."""
  89. self._ies = []
  90. self._ies_instances = {}
  91. self._pps = []
  92. self._progress_hooks = []
  93. self._download_retcode = 0
  94. self._num_downloads = 0
  95. self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  96. if (sys.version_info >= (3,) and sys.platform != 'win32' and
  97. sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
  98. and not params['restrictfilenames']):
  99. # On Python 3, the Unicode filesystem API will throw errors (#1474)
  100. self.report_warning(
  101. u'Assuming --restrict-filenames isnce file system encoding '
  102. u'cannot encode all charactes. '
  103. u'Set the LC_ALL environment variable to fix this.')
  104. params['restrictfilenames'] = True
  105. self.params = params
  106. self.fd = FileDownloader(self, self.params)
  107. if '%(stitle)s' in self.params['outtmpl']:
  108. self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
  109. def add_info_extractor(self, ie):
  110. """Add an InfoExtractor object to the end of the list."""
  111. self._ies.append(ie)
  112. self._ies_instances[ie.ie_key()] = ie
  113. ie.set_downloader(self)
  114. def get_info_extractor(self, ie_key):
  115. """
  116. Get an instance of an IE with name ie_key, it will try to get one from
  117. the _ies list, if there's no instance it will create a new one and add
  118. it to the extractor list.
  119. """
  120. ie = self._ies_instances.get(ie_key)
  121. if ie is None:
  122. ie = get_info_extractor(ie_key)()
  123. self.add_info_extractor(ie)
  124. return ie
  125. def add_default_info_extractors(self):
  126. """
  127. Add the InfoExtractors returned by gen_extractors to the end of the list
  128. """
  129. for ie in gen_extractors():
  130. self.add_info_extractor(ie)
  131. def add_post_processor(self, pp):
  132. """Add a PostProcessor object to the end of the chain."""
  133. self._pps.append(pp)
  134. pp.set_downloader(self)
  135. def to_screen(self, message, skip_eol=False):
  136. """Print message to stdout if not in quiet mode."""
  137. if not self.params.get('quiet', False):
  138. terminator = [u'\n', u''][skip_eol]
  139. output = message + terminator
  140. write_string(output, self._screen_file)
  141. def to_stderr(self, message):
  142. """Print message to stderr."""
  143. assert type(message) == type(u'')
  144. output = message + u'\n'
  145. if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
  146. output = output.encode(preferredencoding())
  147. sys.stderr.write(output)
  148. def fixed_template(self):
  149. """Checks if the output template is fixed."""
  150. return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
  151. def trouble(self, message=None, tb=None):
  152. """Determine action to take when a download problem appears.
  153. Depending on if the downloader has been configured to ignore
  154. download errors or not, this method may throw an exception or
  155. not when errors are found, after printing the message.
  156. tb, if given, is additional traceback information.
  157. """
  158. if message is not None:
  159. self.to_stderr(message)
  160. if self.params.get('verbose'):
  161. if tb is None:
  162. if sys.exc_info()[0]: # if .trouble has been called from an except block
  163. tb = u''
  164. if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
  165. tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
  166. tb += compat_str(traceback.format_exc())
  167. else:
  168. tb_data = traceback.format_list(traceback.extract_stack())
  169. tb = u''.join(tb_data)
  170. self.to_stderr(tb)
  171. if not self.params.get('ignoreerrors', False):
  172. if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
  173. exc_info = sys.exc_info()[1].exc_info
  174. else:
  175. exc_info = sys.exc_info()
  176. raise DownloadError(message, exc_info)
  177. self._download_retcode = 1
  178. def report_warning(self, message):
  179. '''
  180. Print the message to stderr, it will be prefixed with 'WARNING:'
  181. If stderr is a tty file the 'WARNING:' will be colored
  182. '''
  183. if sys.stderr.isatty() and os.name != 'nt':
  184. _msg_header=u'\033[0;33mWARNING:\033[0m'
  185. else:
  186. _msg_header=u'WARNING:'
  187. warning_message=u'%s %s' % (_msg_header,message)
  188. self.to_stderr(warning_message)
  189. def report_error(self, message, tb=None):
  190. '''
  191. Do the same as trouble, but prefixes the message with 'ERROR:', colored
  192. in red if stderr is a tty file.
  193. '''
  194. if sys.stderr.isatty() and os.name != 'nt':
  195. _msg_header = u'\033[0;31mERROR:\033[0m'
  196. else:
  197. _msg_header = u'ERROR:'
  198. error_message = u'%s %s' % (_msg_header, message)
  199. self.trouble(error_message, tb)
  200. def slow_down(self, start_time, byte_counter):
  201. """Sleep if the download speed is over the rate limit."""
  202. rate_limit = self.params.get('ratelimit', None)
  203. if rate_limit is None or byte_counter == 0:
  204. return
  205. now = time.time()
  206. elapsed = now - start_time
  207. if elapsed <= 0.0:
  208. return
  209. speed = float(byte_counter) / elapsed
  210. if speed > rate_limit:
  211. time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
  212. def report_writedescription(self, descfn):
  213. """ Report that the description file is being written """
  214. self.to_screen(u'[info] Writing video description to: ' + descfn)
  215. def report_writesubtitles(self, sub_filename):
  216. """ Report that the subtitles file is being written """
  217. self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
  218. def report_writeinfojson(self, infofn):
  219. """ Report that the metadata file has been written """
  220. self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
  221. def report_file_already_downloaded(self, file_name):
  222. """Report file has already been fully downloaded."""
  223. try:
  224. self.to_screen(u'[download] %s has already been downloaded' % file_name)
  225. except (UnicodeEncodeError) as err:
  226. self.to_screen(u'[download] The file has already been downloaded')
  227. def increment_downloads(self):
  228. """Increment the ordinal that assigns a number to each file."""
  229. self._num_downloads += 1
  230. def prepare_filename(self, info_dict):
  231. """Generate the output filename."""
  232. try:
  233. template_dict = dict(info_dict)
  234. template_dict['epoch'] = int(time.time())
  235. autonumber_size = self.params.get('autonumber_size')
  236. if autonumber_size is None:
  237. autonumber_size = 5
  238. autonumber_templ = u'%0' + str(autonumber_size) + u'd'
  239. template_dict['autonumber'] = autonumber_templ % self._num_downloads
  240. if template_dict['playlist_index'] is not None:
  241. template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
  242. sanitize = lambda k,v: sanitize_filename(
  243. u'NA' if v is None else compat_str(v),
  244. restricted=self.params.get('restrictfilenames'),
  245. is_id=(k==u'id'))
  246. template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
  247. filename = self.params['outtmpl'] % template_dict
  248. return filename
  249. except KeyError as err:
  250. self.report_error(u'Erroneous output template')
  251. return None
  252. except ValueError as err:
  253. self.report_error(u'Error in output template: ' + str(err) + u' (encoding: ' + repr(preferredencoding()) + ')')
  254. return None
  255. def _match_entry(self, info_dict):
  256. """ Returns None iff the file should be downloaded """
  257. title = info_dict['title']
  258. matchtitle = self.params.get('matchtitle', False)
  259. if matchtitle:
  260. if not re.search(matchtitle, title, re.IGNORECASE):
  261. return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
  262. rejecttitle = self.params.get('rejecttitle', False)
  263. if rejecttitle:
  264. if re.search(rejecttitle, title, re.IGNORECASE):
  265. return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
  266. date = info_dict.get('upload_date', None)
  267. if date is not None:
  268. dateRange = self.params.get('daterange', DateRange())
  269. if date not in dateRange:
  270. return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
  271. return None
  272. def extract_info(self, url, download=True, ie_key=None, extra_info={}):
  273. '''
  274. Returns a list with a dictionary for each video we find.
  275. If 'download', also downloads the videos.
  276. extra_info is a dict containing the extra values to add to each result
  277. '''
  278. if ie_key:
  279. ies = [self.get_info_extractor(ie_key)]
  280. else:
  281. ies = self._ies
  282. for ie in ies:
  283. if not ie.suitable(url):
  284. continue
  285. if not ie.working():
  286. self.report_warning(u'The program functionality for this site has been marked as broken, '
  287. u'and will probably not work.')
  288. try:
  289. ie_result = ie.extract(url)
  290. if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
  291. break
  292. if isinstance(ie_result, list):
  293. # Backwards compatibility: old IE result format
  294. for result in ie_result:
  295. result.update(extra_info)
  296. ie_result = {
  297. '_type': 'compat_list',
  298. 'entries': ie_result,
  299. }
  300. else:
  301. ie_result.update(extra_info)
  302. if 'extractor' not in ie_result:
  303. ie_result['extractor'] = ie.IE_NAME
  304. return self.process_ie_result(ie_result, download=download)
  305. except ExtractorError as de: # An error we somewhat expected
  306. self.report_error(compat_str(de), de.format_traceback())
  307. break
  308. except Exception as e:
  309. if self.params.get('ignoreerrors', False):
  310. self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
  311. break
  312. else:
  313. raise
  314. else:
  315. self.report_error(u'no suitable InfoExtractor: %s' % url)
  316. def process_ie_result(self, ie_result, download=True, extra_info={}):
  317. """
  318. Take the result of the ie(may be modified) and resolve all unresolved
  319. references (URLs, playlist items).
  320. It will also download the videos if 'download'.
  321. Returns the resolved ie_result.
  322. """
  323. result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
  324. if result_type == 'video':
  325. ie_result.update(extra_info)
  326. if 'playlist' not in ie_result:
  327. # It isn't part of a playlist
  328. ie_result['playlist'] = None
  329. ie_result['playlist_index'] = None
  330. if download:
  331. self.process_info(ie_result)
  332. return ie_result
  333. elif result_type == 'url':
  334. # We have to add extra_info to the results because it may be
  335. # contained in a playlist
  336. return self.extract_info(ie_result['url'],
  337. download,
  338. ie_key=ie_result.get('ie_key'),
  339. extra_info=extra_info)
  340. elif result_type == 'playlist':
  341. # We process each entry in the playlist
  342. playlist = ie_result.get('title', None) or ie_result.get('id', None)
  343. self.to_screen(u'[download] Downloading playlist: %s' % playlist)
  344. playlist_results = []
  345. n_all_entries = len(ie_result['entries'])
  346. playliststart = self.params.get('playliststart', 1) - 1
  347. playlistend = self.params.get('playlistend', -1)
  348. if playlistend == -1:
  349. entries = ie_result['entries'][playliststart:]
  350. else:
  351. entries = ie_result['entries'][playliststart:playlistend]
  352. n_entries = len(entries)
  353. self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
  354. (ie_result['extractor'], playlist, n_all_entries, n_entries))
  355. for i,entry in enumerate(entries,1):
  356. self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
  357. extra = {
  358. 'playlist': playlist,
  359. 'playlist_index': i + playliststart,
  360. }
  361. if not 'extractor' in entry:
  362. # We set the extractor, if it's an url it will be set then to
  363. # the new extractor, but if it's already a video we must make
  364. # sure it's present: see issue #877
  365. entry['extractor'] = ie_result['extractor']
  366. entry_result = self.process_ie_result(entry,
  367. download=download,
  368. extra_info=extra)
  369. playlist_results.append(entry_result)
  370. ie_result['entries'] = playlist_results
  371. return ie_result
  372. elif result_type == 'compat_list':
  373. def _fixup(r):
  374. r.setdefault('extractor', ie_result['extractor'])
  375. return r
  376. ie_result['entries'] = [
  377. self.process_ie_result(_fixup(r), download=download)
  378. for r in ie_result['entries']
  379. ]
  380. return ie_result
  381. else:
  382. raise Exception('Invalid result type: %s' % result_type)
  383. def process_info(self, info_dict):
  384. """Process a single resolved IE result."""
  385. assert info_dict.get('_type', 'video') == 'video'
  386. #We increment the download the download count here to match the previous behaviour.
  387. self.increment_downloads()
  388. info_dict['fulltitle'] = info_dict['title']
  389. if len(info_dict['title']) > 200:
  390. info_dict['title'] = info_dict['title'][:197] + u'...'
  391. # Keep for backwards compatibility
  392. info_dict['stitle'] = info_dict['title']
  393. if not 'format' in info_dict:
  394. info_dict['format'] = info_dict['ext']
  395. reason = self._match_entry(info_dict)
  396. if reason is not None:
  397. self.to_screen(u'[download] ' + reason)
  398. return
  399. max_downloads = self.params.get('max_downloads')
  400. if max_downloads is not None:
  401. if self._num_downloads > int(max_downloads):
  402. raise MaxDownloadsReached()
  403. filename = self.prepare_filename(info_dict)
  404. # Forced printings
  405. if self.params.get('forcetitle', False):
  406. compat_print(info_dict['title'])
  407. if self.params.get('forceid', False):
  408. compat_print(info_dict['id'])
  409. if self.params.get('forceurl', False):
  410. # For RTMP URLs, also include the playpath
  411. compat_print(info_dict['url'] + info_dict.get('play_path', u''))
  412. if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
  413. compat_print(info_dict['thumbnail'])
  414. if self.params.get('forcedescription', False) and 'description' in info_dict:
  415. compat_print(info_dict['description'])
  416. if self.params.get('forcefilename', False) and filename is not None:
  417. compat_print(filename)
  418. if self.params.get('forceformat', False):
  419. compat_print(info_dict['format'])
  420. # Do nothing else if in simulate mode
  421. if self.params.get('simulate', False):
  422. return
  423. if filename is None:
  424. return
  425. try:
  426. dn = os.path.dirname(encodeFilename(filename))
  427. if dn != '' and not os.path.exists(dn):
  428. os.makedirs(dn)
  429. except (OSError, IOError) as err:
  430. self.report_error(u'unable to create directory ' + compat_str(err))
  431. return
  432. if self.params.get('writedescription', False):
  433. try:
  434. descfn = filename + u'.description'
  435. self.report_writedescription(descfn)
  436. with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
  437. descfile.write(info_dict['description'])
  438. except (KeyError, TypeError):
  439. self.report_warning(u'There\'s no description to write.')
  440. except (OSError, IOError):
  441. self.report_error(u'Cannot write description file ' + descfn)
  442. return
  443. subtitles_are_requested = any([self.params.get('writesubtitles', False),
  444. self.params.get('writeautomaticsub')])
  445. if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
  446. # subtitles download errors are already managed as troubles in relevant IE
  447. # that way it will silently go on when used with unsupporting IE
  448. subtitles = info_dict['subtitles']
  449. sub_format = self.params.get('subtitlesformat')
  450. for sub_lang in subtitles.keys():
  451. sub = subtitles[sub_lang]
  452. if sub is None:
  453. continue
  454. try:
  455. sub_filename = subtitles_filename(filename, sub_lang, sub_format)
  456. self.report_writesubtitles(sub_filename)
  457. with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
  458. subfile.write(sub)
  459. except (OSError, IOError):
  460. self.report_error(u'Cannot write subtitles file ' + descfn)
  461. return
  462. if self.params.get('writeinfojson', False):
  463. infofn = filename + u'.info.json'
  464. self.report_writeinfojson(infofn)
  465. try:
  466. json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
  467. write_json_file(json_info_dict, encodeFilename(infofn))
  468. except (OSError, IOError):
  469. self.report_error(u'Cannot write metadata to JSON file ' + infofn)
  470. return
  471. if self.params.get('writethumbnail', False):
  472. if info_dict.get('thumbnail') is not None:
  473. thumb_format = determine_ext(info_dict['thumbnail'], u'jpg')
  474. thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
  475. self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
  476. (info_dict['extractor'], info_dict['id']))
  477. uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
  478. with open(thumb_filename, 'wb') as thumbf:
  479. shutil.copyfileobj(uf, thumbf)
  480. self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
  481. (info_dict['extractor'], info_dict['id'], thumb_filename))
  482. if not self.params.get('skip_download', False):
  483. if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
  484. success = True
  485. else:
  486. try:
  487. success = self.fd._do_download(filename, info_dict)
  488. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  489. self.report_error(u'unable to download video data: %s' % str(err))
  490. return
  491. except (OSError, IOError) as err:
  492. raise UnavailableVideoError(err)
  493. except (ContentTooShortError, ) as err:
  494. self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
  495. return
  496. if success:
  497. try:
  498. self.post_process(filename, info_dict)
  499. except (PostProcessingError) as err:
  500. self.report_error(u'postprocessing: %s' % str(err))
  501. return
  502. def download(self, url_list):
  503. """Download a given list of URLs."""
  504. if len(url_list) > 1 and self.fixed_template():
  505. raise SameFileError(self.params['outtmpl'])
  506. for url in url_list:
  507. try:
  508. #It also downloads the videos
  509. videos = self.extract_info(url)
  510. except UnavailableVideoError:
  511. self.report_error(u'unable to download video')
  512. except MaxDownloadsReached:
  513. self.to_screen(u'[info] Maximum number of downloaded files reached.')
  514. raise
  515. return self._download_retcode
  516. def post_process(self, filename, ie_info):
  517. """Run all the postprocessors on the given file."""
  518. info = dict(ie_info)
  519. info['filepath'] = filename
  520. keep_video = None
  521. for pp in self._pps:
  522. try:
  523. keep_video_wish,new_info = pp.run(info)
  524. if keep_video_wish is not None:
  525. if keep_video_wish:
  526. keep_video = keep_video_wish
  527. elif keep_video is None:
  528. # No clear decision yet, let IE decide
  529. keep_video = keep_video_wish
  530. except PostProcessingError as e:
  531. self.report_error(e.msg)
  532. if keep_video is False and not self.params.get('keepvideo', False):
  533. try:
  534. self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
  535. os.remove(encodeFilename(filename))
  536. except (IOError, OSError):
  537. self.report_warning(u'Unable to remove downloaded video file')