You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

591 lines
26 KiB

  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. from __future__ import absolute_import
  4. import io
  5. import os
  6. import re
  7. import shutil
  8. import socket
  9. import sys
  10. import time
  11. import traceback
  12. from .utils import *
  13. from .extractor import get_info_extractor, gen_extractors
  14. from .FileDownloader import FileDownloader
  15. class YoutubeDL(object):
  16. """YoutubeDL class.
  17. YoutubeDL objects are the ones responsible of downloading the
  18. actual video file and writing it to disk if the user has requested
  19. it, among some other tasks. In most cases there should be one per
  20. program. As, given a video URL, the downloader doesn't know how to
  21. extract all the needed information, task that InfoExtractors do, it
  22. has to pass the URL to one of them.
  23. For this, YoutubeDL objects have a method that allows
  24. InfoExtractors to be registered in a given order. When it is passed
  25. a URL, the YoutubeDL object handles it to the first InfoExtractor it
  26. finds that reports being able to handle it. The InfoExtractor extracts
  27. all the information about the video or videos the URL refers to, and
  28. YoutubeDL process the extracted information, possibly using a File
  29. Downloader to download the video.
  30. YoutubeDL objects accept a lot of parameters. In order not to saturate
  31. the object constructor with arguments, it receives a dictionary of
  32. options instead. These options are available through the params
  33. attribute for the InfoExtractors to use. The YoutubeDL also
  34. registers itself as the downloader in charge for the InfoExtractors
  35. that are added to it, so this is a "mutual registration".
  36. Available options:
  37. username: Username for authentication purposes.
  38. password: Password for authentication purposes.
  39. videopassword: Password for acces a video.
  40. usenetrc: Use netrc for authentication instead.
  41. verbose: Print additional info to stdout.
  42. quiet: Do not print messages to stdout.
  43. forceurl: Force printing final URL.
  44. forcetitle: Force printing title.
  45. forceid: Force printing ID.
  46. forcethumbnail: Force printing thumbnail URL.
  47. forcedescription: Force printing description.
  48. forcefilename: Force printing final filename.
  49. simulate: Do not download the video files.
  50. format: Video format code.
  51. format_limit: Highest quality format to try.
  52. outtmpl: Template for output names.
  53. restrictfilenames: Do not allow "&" and spaces in file names
  54. ignoreerrors: Do not stop on download errors.
  55. nooverwrites: Prevent overwriting files.
  56. playliststart: Playlist item to start at.
  57. playlistend: Playlist item to end at.
  58. matchtitle: Download only matching titles.
  59. rejecttitle: Reject downloads for matching titles.
  60. logtostderr: Log messages to stderr instead of stdout.
  61. writedescription: Write the video description to a .description file
  62. writeinfojson: Write the video description to a .info.json file
  63. writethumbnail: Write the thumbnail image to a file
  64. writesubtitles: Write the video subtitles to a file
  65. writeautomaticsub: Write the automatic subtitles to a file
  66. allsubtitles: Downloads all the subtitles of the video
  67. listsubtitles: Lists all available subtitles for the video
  68. subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
  69. subtitleslangs: List of languages of the subtitles to download
  70. keepvideo: Keep the video file after post-processing
  71. daterange: A DateRange object, download only if the upload_date is in the range.
  72. skip_download: Skip the actual download of the video file
  73. The following parameters are not used by YoutubeDL itself, they are used by
  74. the FileDownloader:
  75. nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
  76. noresizebuffer, retries, continuedl, noprogress, consoletitle
  77. """
  78. params = None
  79. _ies = []
  80. _pps = []
  81. _download_retcode = None
  82. _num_downloads = None
  83. _screen_file = None
  84. def __init__(self, params):
  85. """Create a FileDownloader object with the given options."""
  86. self._ies = []
  87. self._pps = []
  88. self._progress_hooks = []
  89. self._download_retcode = 0
  90. self._num_downloads = 0
  91. self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  92. self.params = params
  93. self.fd = FileDownloader(self, self.params)
  94. if '%(stitle)s' in self.params['outtmpl']:
  95. self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
  96. def add_info_extractor(self, ie):
  97. """Add an InfoExtractor object to the end of the list."""
  98. self._ies.append(ie)
  99. ie.set_downloader(self)
  100. def add_default_info_extractors(self):
  101. """
  102. Add the InfoExtractors returned by gen_extractors to the end of the list
  103. """
  104. for ie in gen_extractors():
  105. self.add_info_extractor(ie)
  106. def add_post_processor(self, pp):
  107. """Add a PostProcessor object to the end of the chain."""
  108. self._pps.append(pp)
  109. pp.set_downloader(self)
  110. def to_screen(self, message, skip_eol=False):
  111. """Print message to stdout if not in quiet mode."""
  112. assert type(message) == type(u'')
  113. if not self.params.get('quiet', False):
  114. terminator = [u'\n', u''][skip_eol]
  115. output = message + terminator
  116. if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
  117. output = output.encode(preferredencoding(), 'ignore')
  118. self._screen_file.write(output)
  119. self._screen_file.flush()
  120. def to_stderr(self, message):
  121. """Print message to stderr."""
  122. assert type(message) == type(u'')
  123. output = message + u'\n'
  124. if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
  125. output = output.encode(preferredencoding())
  126. sys.stderr.write(output)
  127. def fixed_template(self):
  128. """Checks if the output template is fixed."""
  129. return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
  130. def trouble(self, message=None, tb=None):
  131. """Determine action to take when a download problem appears.
  132. Depending on if the downloader has been configured to ignore
  133. download errors or not, this method may throw an exception or
  134. not when errors are found, after printing the message.
  135. tb, if given, is additional traceback information.
  136. """
  137. if message is not None:
  138. self.to_stderr(message)
  139. if self.params.get('verbose'):
  140. if tb is None:
  141. if sys.exc_info()[0]: # if .trouble has been called from an except block
  142. tb = u''
  143. if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
  144. tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
  145. tb += compat_str(traceback.format_exc())
  146. else:
  147. tb_data = traceback.format_list(traceback.extract_stack())
  148. tb = u''.join(tb_data)
  149. self.to_stderr(tb)
  150. if not self.params.get('ignoreerrors', False):
  151. if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
  152. exc_info = sys.exc_info()[1].exc_info
  153. else:
  154. exc_info = sys.exc_info()
  155. raise DownloadError(message, exc_info)
  156. self._download_retcode = 1
  157. def report_warning(self, message):
  158. '''
  159. Print the message to stderr, it will be prefixed with 'WARNING:'
  160. If stderr is a tty file the 'WARNING:' will be colored
  161. '''
  162. if sys.stderr.isatty() and os.name != 'nt':
  163. _msg_header=u'\033[0;33mWARNING:\033[0m'
  164. else:
  165. _msg_header=u'WARNING:'
  166. warning_message=u'%s %s' % (_msg_header,message)
  167. self.to_stderr(warning_message)
  168. def report_error(self, message, tb=None):
  169. '''
  170. Do the same as trouble, but prefixes the message with 'ERROR:', colored
  171. in red if stderr is a tty file.
  172. '''
  173. if sys.stderr.isatty() and os.name != 'nt':
  174. _msg_header = u'\033[0;31mERROR:\033[0m'
  175. else:
  176. _msg_header = u'ERROR:'
  177. error_message = u'%s %s' % (_msg_header, message)
  178. self.trouble(error_message, tb)
  179. def slow_down(self, start_time, byte_counter):
  180. """Sleep if the download speed is over the rate limit."""
  181. rate_limit = self.params.get('ratelimit', None)
  182. if rate_limit is None or byte_counter == 0:
  183. return
  184. now = time.time()
  185. elapsed = now - start_time
  186. if elapsed <= 0.0:
  187. return
  188. speed = float(byte_counter) / elapsed
  189. if speed > rate_limit:
  190. time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
  191. def report_writedescription(self, descfn):
  192. """ Report that the description file is being written """
  193. self.to_screen(u'[info] Writing video description to: ' + descfn)
  194. def report_writesubtitles(self, sub_filename):
  195. """ Report that the subtitles file is being written """
  196. self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
  197. def report_writeinfojson(self, infofn):
  198. """ Report that the metadata file has been written """
  199. self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
  200. def report_file_already_downloaded(self, file_name):
  201. """Report file has already been fully downloaded."""
  202. try:
  203. self.to_screen(u'[download] %s has already been downloaded' % file_name)
  204. except (UnicodeEncodeError) as err:
  205. self.to_screen(u'[download] The file has already been downloaded')
  206. def increment_downloads(self):
  207. """Increment the ordinal that assigns a number to each file."""
  208. self._num_downloads += 1
  209. def prepare_filename(self, info_dict):
  210. """Generate the output filename."""
  211. try:
  212. template_dict = dict(info_dict)
  213. template_dict['epoch'] = int(time.time())
  214. autonumber_size = self.params.get('autonumber_size')
  215. if autonumber_size is None:
  216. autonumber_size = 5
  217. autonumber_templ = u'%0' + str(autonumber_size) + u'd'
  218. template_dict['autonumber'] = autonumber_templ % self._num_downloads
  219. if template_dict['playlist_index'] is not None:
  220. template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
  221. sanitize = lambda k,v: sanitize_filename(
  222. u'NA' if v is None else compat_str(v),
  223. restricted=self.params.get('restrictfilenames'),
  224. is_id=(k==u'id'))
  225. template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
  226. filename = self.params['outtmpl'] % template_dict
  227. return filename
  228. except KeyError as err:
  229. self.report_error(u'Erroneous output template')
  230. return None
  231. except ValueError as err:
  232. self.report_error(u'Error in output template: ' + str(err) + u' (encoding: ' + repr(preferredencoding()) + ')')
  233. return None
  234. def _match_entry(self, info_dict):
  235. """ Returns None iff the file should be downloaded """
  236. title = info_dict['title']
  237. matchtitle = self.params.get('matchtitle', False)
  238. if matchtitle:
  239. if not re.search(matchtitle, title, re.IGNORECASE):
  240. return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
  241. rejecttitle = self.params.get('rejecttitle', False)
  242. if rejecttitle:
  243. if re.search(rejecttitle, title, re.IGNORECASE):
  244. return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
  245. date = info_dict.get('upload_date', None)
  246. if date is not None:
  247. dateRange = self.params.get('daterange', DateRange())
  248. if date not in dateRange:
  249. return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
  250. return None
  251. def extract_info(self, url, download=True, ie_key=None, extra_info={}):
  252. '''
  253. Returns a list with a dictionary for each video we find.
  254. If 'download', also downloads the videos.
  255. extra_info is a dict containing the extra values to add to each result
  256. '''
  257. if ie_key:
  258. ie = get_info_extractor(ie_key)()
  259. ie.set_downloader(self)
  260. ies = [ie]
  261. else:
  262. ies = self._ies
  263. for ie in ies:
  264. if not ie.suitable(url):
  265. continue
  266. if not ie.working():
  267. self.report_warning(u'The program functionality for this site has been marked as broken, '
  268. u'and will probably not work.')
  269. try:
  270. ie_result = ie.extract(url)
  271. if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
  272. break
  273. if isinstance(ie_result, list):
  274. # Backwards compatibility: old IE result format
  275. for result in ie_result:
  276. result.update(extra_info)
  277. ie_result = {
  278. '_type': 'compat_list',
  279. 'entries': ie_result,
  280. }
  281. else:
  282. ie_result.update(extra_info)
  283. if 'extractor' not in ie_result:
  284. ie_result['extractor'] = ie.IE_NAME
  285. return self.process_ie_result(ie_result, download=download)
  286. except ExtractorError as de: # An error we somewhat expected
  287. self.report_error(compat_str(de), de.format_traceback())
  288. break
  289. except Exception as e:
  290. if self.params.get('ignoreerrors', False):
  291. self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
  292. break
  293. else:
  294. raise
  295. else:
  296. self.report_error(u'no suitable InfoExtractor: %s' % url)
  297. def process_ie_result(self, ie_result, download=True, extra_info={}):
  298. """
  299. Take the result of the ie(may be modified) and resolve all unresolved
  300. references (URLs, playlist items).
  301. It will also download the videos if 'download'.
  302. Returns the resolved ie_result.
  303. """
  304. result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
  305. if result_type == 'video':
  306. ie_result.update(extra_info)
  307. if 'playlist' not in ie_result:
  308. # It isn't part of a playlist
  309. ie_result['playlist'] = None
  310. ie_result['playlist_index'] = None
  311. if download:
  312. self.process_info(ie_result)
  313. return ie_result
  314. elif result_type == 'url':
  315. # We have to add extra_info to the results because it may be
  316. # contained in a playlist
  317. return self.extract_info(ie_result['url'],
  318. download,
  319. ie_key=ie_result.get('ie_key'),
  320. extra_info=extra_info)
  321. elif result_type == 'playlist':
  322. # We process each entry in the playlist
  323. playlist = ie_result.get('title', None) or ie_result.get('id', None)
  324. self.to_screen(u'[download] Downloading playlist: %s' % playlist)
  325. playlist_results = []
  326. n_all_entries = len(ie_result['entries'])
  327. playliststart = self.params.get('playliststart', 1) - 1
  328. playlistend = self.params.get('playlistend', -1)
  329. if playlistend == -1:
  330. entries = ie_result['entries'][playliststart:]
  331. else:
  332. entries = ie_result['entries'][playliststart:playlistend]
  333. n_entries = len(entries)
  334. self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
  335. (ie_result['extractor'], playlist, n_all_entries, n_entries))
  336. for i,entry in enumerate(entries,1):
  337. self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
  338. extra = {
  339. 'playlist': playlist,
  340. 'playlist_index': i + playliststart,
  341. }
  342. if not 'extractor' in entry:
  343. # We set the extractor, if it's an url it will be set then to
  344. # the new extractor, but if it's already a video we must make
  345. # sure it's present: see issue #877
  346. entry['extractor'] = ie_result['extractor']
  347. entry_result = self.process_ie_result(entry,
  348. download=download,
  349. extra_info=extra)
  350. playlist_results.append(entry_result)
  351. ie_result['entries'] = playlist_results
  352. return ie_result
  353. elif result_type == 'compat_list':
  354. def _fixup(r):
  355. r.setdefault('extractor', ie_result['extractor'])
  356. return r
  357. ie_result['entries'] = [
  358. self.process_ie_result(_fixup(r), download=download)
  359. for r in ie_result['entries']
  360. ]
  361. return ie_result
  362. else:
  363. raise Exception('Invalid result type: %s' % result_type)
  364. def process_info(self, info_dict):
  365. """Process a single resolved IE result."""
  366. assert info_dict.get('_type', 'video') == 'video'
  367. #We increment the download the download count here to match the previous behaviour.
  368. self.increment_downloads()
  369. info_dict['fulltitle'] = info_dict['title']
  370. if len(info_dict['title']) > 200:
  371. info_dict['title'] = info_dict['title'][:197] + u'...'
  372. # Keep for backwards compatibility
  373. info_dict['stitle'] = info_dict['title']
  374. if not 'format' in info_dict:
  375. info_dict['format'] = info_dict['ext']
  376. reason = self._match_entry(info_dict)
  377. if reason is not None:
  378. self.to_screen(u'[download] ' + reason)
  379. return
  380. max_downloads = self.params.get('max_downloads')
  381. if max_downloads is not None:
  382. if self._num_downloads > int(max_downloads):
  383. raise MaxDownloadsReached()
  384. filename = self.prepare_filename(info_dict)
  385. # Forced printings
  386. if self.params.get('forcetitle', False):
  387. compat_print(info_dict['title'])
  388. if self.params.get('forceid', False):
  389. compat_print(info_dict['id'])
  390. if self.params.get('forceurl', False):
  391. # For RTMP URLs, also include the playpath
  392. compat_print(info_dict['url'] + info_dict.get('play_path', u''))
  393. if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
  394. compat_print(info_dict['thumbnail'])
  395. if self.params.get('forcedescription', False) and 'description' in info_dict:
  396. compat_print(info_dict['description'])
  397. if self.params.get('forcefilename', False) and filename is not None:
  398. compat_print(filename)
  399. if self.params.get('forceformat', False):
  400. compat_print(info_dict['format'])
  401. # Do nothing else if in simulate mode
  402. if self.params.get('simulate', False):
  403. return
  404. if filename is None:
  405. return
  406. try:
  407. dn = os.path.dirname(encodeFilename(filename))
  408. if dn != '' and not os.path.exists(dn):
  409. os.makedirs(dn)
  410. except (OSError, IOError) as err:
  411. self.report_error(u'unable to create directory ' + compat_str(err))
  412. return
  413. if self.params.get('writedescription', False):
  414. try:
  415. descfn = filename + u'.description'
  416. self.report_writedescription(descfn)
  417. with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
  418. descfile.write(info_dict['description'])
  419. except (OSError, IOError):
  420. self.report_error(u'Cannot write description file ' + descfn)
  421. return
  422. subtitles_are_requested = any([self.params.get('writesubtitles', False),
  423. self.params.get('writeautomaticsub'),
  424. self.params.get('allsubtitles', False)])
  425. if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
  426. # subtitles download errors are already managed as troubles in relevant IE
  427. # that way it will silently go on when used with unsupporting IE
  428. subtitles = info_dict['subtitles']
  429. sub_format = self.params.get('subtitlesformat')
  430. for sub_lang in subtitles.keys():
  431. sub = subtitles[sub_lang]
  432. if sub is None:
  433. continue
  434. try:
  435. sub_filename = subtitles_filename(filename, sub_lang, sub_format)
  436. self.report_writesubtitles(sub_filename)
  437. with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
  438. subfile.write(sub)
  439. except (OSError, IOError):
  440. self.report_error(u'Cannot write subtitles file ' + descfn)
  441. return
  442. if self.params.get('writeinfojson', False):
  443. infofn = filename + u'.info.json'
  444. self.report_writeinfojson(infofn)
  445. try:
  446. json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
  447. write_json_file(json_info_dict, encodeFilename(infofn))
  448. except (OSError, IOError):
  449. self.report_error(u'Cannot write metadata to JSON file ' + infofn)
  450. return
  451. if self.params.get('writethumbnail', False):
  452. if info_dict.get('thumbnail') is not None:
  453. thumb_format = determine_ext(info_dict['thumbnail'], u'jpg')
  454. thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
  455. self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
  456. (info_dict['extractor'], info_dict['id']))
  457. uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
  458. with open(thumb_filename, 'wb') as thumbf:
  459. shutil.copyfileobj(uf, thumbf)
  460. self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
  461. (info_dict['extractor'], info_dict['id'], thumb_filename))
  462. if not self.params.get('skip_download', False):
  463. if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
  464. success = True
  465. else:
  466. try:
  467. success = self.fd._do_download(filename, info_dict)
  468. except (OSError, IOError) as err:
  469. raise UnavailableVideoError(err)
  470. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  471. self.report_error(u'unable to download video data: %s' % str(err))
  472. return
  473. except (ContentTooShortError, ) as err:
  474. self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
  475. return
  476. if success:
  477. try:
  478. self.post_process(filename, info_dict)
  479. except (PostProcessingError) as err:
  480. self.report_error(u'postprocessing: %s' % str(err))
  481. return
  482. def download(self, url_list):
  483. """Download a given list of URLs."""
  484. if len(url_list) > 1 and self.fixed_template():
  485. raise SameFileError(self.params['outtmpl'])
  486. for url in url_list:
  487. try:
  488. #It also downloads the videos
  489. videos = self.extract_info(url)
  490. except UnavailableVideoError:
  491. self.report_error(u'unable to download video')
  492. except MaxDownloadsReached:
  493. self.to_screen(u'[info] Maximum number of downloaded files reached.')
  494. raise
  495. return self._download_retcode
  496. def post_process(self, filename, ie_info):
  497. """Run all the postprocessors on the given file."""
  498. info = dict(ie_info)
  499. info['filepath'] = filename
  500. keep_video = None
  501. for pp in self._pps:
  502. try:
  503. keep_video_wish,new_info = pp.run(info)
  504. if keep_video_wish is not None:
  505. if keep_video_wish:
  506. keep_video = keep_video_wish
  507. elif keep_video is None:
  508. # No clear decision yet, let IE decide
  509. keep_video = keep_video_wish
  510. except PostProcessingError as e:
  511. self.report_error(e.msg)
  512. if keep_video is False and not self.params.get('keepvideo', False):
  513. try:
  514. self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
  515. os.remove(encodeFilename(filename))
  516. except (IOError, OSError):
  517. self.report_warning(u'Unable to remove downloaded video file')