Philipp Hagemeister
11 years ago
10 changed files with 816 additions and 727 deletions
Split View
Diff Options
-
+1 -1setup.py
-
+1 -1test/test_download.py
-
+8 -720youtube_dl/FileDownloader.py
-
+10 -5youtube_dl/YoutubeDL.py
-
+23 -0youtube_dl/downloader/__init__.py
-
+321 -0youtube_dl/downloader/common.py
-
+44 -0youtube_dl/downloader/hls.py
-
+191 -0youtube_dl/downloader/http.py
-
+39 -0youtube_dl/downloader/mplayer.py
-
+178 -0youtube_dl/downloader/rtmp.py
+ 1
- 1
setup.py
View File
+ 1
- 1
test/test_download.py
View File
+ 8
- 720
youtube_dl/FileDownloader.py
View File
| @ -1,724 +1,12 @@ | |||
| import os | |||
| import re | |||
| import subprocess | |||
| import sys | |||
| import time | |||
| from .utils import ( | |||
| compat_urllib_error, | |||
| compat_urllib_request, | |||
| ContentTooShortError, | |||
| determine_ext, | |||
| encodeFilename, | |||
| format_bytes, | |||
| sanitize_open, | |||
| timeconvert, | |||
| ) | |||
| class FileDownloader(object): | |||
| """File Downloader class. | |||
| File downloader objects are the ones responsible of downloading the | |||
| actual video file and writing it to disk. | |||
| File downloaders accept a lot of parameters. In order not to saturate | |||
| the object constructor with arguments, it receives a dictionary of | |||
| options instead. | |||
| Available options: | |||
| verbose: Print additional info to stdout. | |||
| quiet: Do not print messages to stdout. | |||
| ratelimit: Download speed limit, in bytes/sec. | |||
| retries: Number of times to retry for HTTP error 5xx | |||
| buffersize: Size of download buffer in bytes. | |||
| noresizebuffer: Do not automatically resize the download buffer. | |||
| continuedl: Try to continue downloads if possible. | |||
| noprogress: Do not print the progress bar. | |||
| logtostderr: Log messages to stderr instead of stdout. | |||
| consoletitle: Display progress in console window's titlebar. | |||
| nopart: Do not use temporary .part files. | |||
| updatetime: Use the Last-modified header to set output file timestamps. | |||
| test: Download only first bytes to test the downloader. | |||
| min_filesize: Skip files smaller than this size | |||
| max_filesize: Skip files larger than this size | |||
| """ | |||
| params = None | |||
| def __init__(self, ydl, params): | |||
| """Create a FileDownloader object with the given options.""" | |||
| self.ydl = ydl | |||
| self._progress_hooks = [] | |||
| self.params = params | |||
| @staticmethod | |||
| def format_seconds(seconds): | |||
| (mins, secs) = divmod(seconds, 60) | |||
| (hours, mins) = divmod(mins, 60) | |||
| if hours > 99: | |||
| return '--:--:--' | |||
| if hours == 0: | |||
| return '%02d:%02d' % (mins, secs) | |||
| else: | |||
| return '%02d:%02d:%02d' % (hours, mins, secs) | |||
| @staticmethod | |||
| def calc_percent(byte_counter, data_len): | |||
| if data_len is None: | |||
| return None | |||
| return float(byte_counter) / float(data_len) * 100.0 | |||
| @staticmethod | |||
| def format_percent(percent): | |||
| if percent is None: | |||
| return '---.-%' | |||
| return '%6s' % ('%3.1f%%' % percent) | |||
| @staticmethod | |||
| def calc_eta(start, now, total, current): | |||
| if total is None: | |||
| return None | |||
| dif = now - start | |||
| if current == 0 or dif < 0.001: # One millisecond | |||
| return None | |||
| rate = float(current) / dif | |||
| return int((float(total) - float(current)) / rate) | |||
| @staticmethod | |||
| def format_eta(eta): | |||
| if eta is None: | |||
| return '--:--' | |||
| return FileDownloader.format_seconds(eta) | |||
| @staticmethod | |||
| def calc_speed(start, now, bytes): | |||
| dif = now - start | |||
| if bytes == 0 or dif < 0.001: # One millisecond | |||
| return None | |||
| return float(bytes) / dif | |||
| @staticmethod | |||
| def format_speed(speed): | |||
| if speed is None: | |||
| return '%10s' % '---b/s' | |||
| return '%10s' % ('%s/s' % format_bytes(speed)) | |||
| @staticmethod | |||
| def best_block_size(elapsed_time, bytes): | |||
| new_min = max(bytes / 2.0, 1.0) | |||
| new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB | |||
| if elapsed_time < 0.001: | |||
| return int(new_max) | |||
| rate = bytes / elapsed_time | |||
| if rate > new_max: | |||
| return int(new_max) | |||
| if rate < new_min: | |||
| return int(new_min) | |||
| return int(rate) | |||
| @staticmethod | |||
| def parse_bytes(bytestr): | |||
| """Parse a string indicating a byte quantity into an integer.""" | |||
| matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) | |||
| if matchobj is None: | |||
| return None | |||
| number = float(matchobj.group(1)) | |||
| multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) | |||
| return int(round(number * multiplier)) | |||
| def to_screen(self, *args, **kargs): | |||
| self.ydl.to_screen(*args, **kargs) | |||
| def to_stderr(self, message): | |||
| self.ydl.to_screen(message) | |||
| def to_console_title(self, message): | |||
| self.ydl.to_console_title(message) | |||
| def trouble(self, *args, **kargs): | |||
| self.ydl.trouble(*args, **kargs) | |||
| def report_warning(self, *args, **kargs): | |||
| self.ydl.report_warning(*args, **kargs) | |||
| def report_error(self, *args, **kargs): | |||
| self.ydl.report_error(*args, **kargs) | |||
| def slow_down(self, start_time, byte_counter): | |||
| """Sleep if the download speed is over the rate limit.""" | |||
| rate_limit = self.params.get('ratelimit', None) | |||
| if rate_limit is None or byte_counter == 0: | |||
| return | |||
| now = time.time() | |||
| elapsed = now - start_time | |||
| if elapsed <= 0.0: | |||
| return | |||
| speed = float(byte_counter) / elapsed | |||
| if speed > rate_limit: | |||
| time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) | |||
| def temp_name(self, filename): | |||
| """Returns a temporary filename for the given filename.""" | |||
| if self.params.get('nopart', False) or filename == u'-' or \ | |||
| (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): | |||
| return filename | |||
| return filename + u'.part' | |||
| def undo_temp_name(self, filename): | |||
| if filename.endswith(u'.part'): | |||
| return filename[:-len(u'.part')] | |||
| return filename | |||
| def try_rename(self, old_filename, new_filename): | |||
| try: | |||
| if old_filename == new_filename: | |||
| return | |||
| os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) | |||
| except (IOError, OSError): | |||
| self.report_error(u'unable to rename file') | |||
| def try_utime(self, filename, last_modified_hdr): | |||
| """Try to set the last-modified time of the given file.""" | |||
| if last_modified_hdr is None: | |||
| return | |||
| if not os.path.isfile(encodeFilename(filename)): | |||
| return | |||
| timestr = last_modified_hdr | |||
| if timestr is None: | |||
| return | |||
| filetime = timeconvert(timestr) | |||
| if filetime is None: | |||
| return filetime | |||
| # Ignore obviously invalid dates | |||
| if filetime == 0: | |||
| return | |||
| try: | |||
| os.utime(filename, (time.time(), filetime)) | |||
| except: | |||
| pass | |||
| return filetime | |||
| def report_destination(self, filename): | |||
| """Report destination filename.""" | |||
| self.to_screen(u'[download] Destination: ' + filename) | |||
| def _report_progress_status(self, msg, is_last_line=False): | |||
| fullmsg = u'[download] ' + msg | |||
| if self.params.get('progress_with_newline', False): | |||
| self.to_screen(fullmsg) | |||
| else: | |||
| if os.name == 'nt': | |||
| prev_len = getattr(self, '_report_progress_prev_line_length', | |||
| 0) | |||
| if prev_len > len(fullmsg): | |||
| fullmsg += u' ' * (prev_len - len(fullmsg)) | |||
| self._report_progress_prev_line_length = len(fullmsg) | |||
| clear_line = u'\r' | |||
| else: | |||
| clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r') | |||
| self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) | |||
| self.to_console_title(u'youtube-dl ' + msg) | |||
| def report_progress(self, percent, data_len_str, speed, eta): | |||
| """Report download progress.""" | |||
| if self.params.get('noprogress', False): | |||
| return | |||
| if eta is not None: | |||
| eta_str = self.format_eta(eta) | |||
| else: | |||
| eta_str = 'Unknown ETA' | |||
| if percent is not None: | |||
| percent_str = self.format_percent(percent) | |||
| else: | |||
| percent_str = 'Unknown %' | |||
| speed_str = self.format_speed(speed) | |||
| msg = (u'%s of %s at %s ETA %s' % | |||
| (percent_str, data_len_str, speed_str, eta_str)) | |||
| self._report_progress_status(msg) | |||
| def report_progress_live_stream(self, downloaded_data_len, speed, elapsed): | |||
| if self.params.get('noprogress', False): | |||
| return | |||
| downloaded_str = format_bytes(downloaded_data_len) | |||
| speed_str = self.format_speed(speed) | |||
| elapsed_str = FileDownloader.format_seconds(elapsed) | |||
| msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str) | |||
| self._report_progress_status(msg) | |||
| def report_finish(self, data_len_str, tot_time): | |||
| """Report download finished.""" | |||
| if self.params.get('noprogress', False): | |||
| self.to_screen(u'[download] Download completed') | |||
| else: | |||
| self._report_progress_status( | |||
| (u'100%% of %s in %s' % | |||
| (data_len_str, self.format_seconds(tot_time))), | |||
| is_last_line=True) | |||
| def report_resuming_byte(self, resume_len): | |||
| """Report attempt to resume at given byte.""" | |||
| self.to_screen(u'[download] Resuming download at byte %s' % resume_len) | |||
| def report_retry(self, count, retries): | |||
| """Report retry in case of HTTP error 5xx""" | |||
| self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) | |||
| def report_file_already_downloaded(self, file_name): | |||
| """Report file has already been fully downloaded.""" | |||
| try: | |||
| self.to_screen(u'[download] %s has already been downloaded' % file_name) | |||
| except UnicodeEncodeError: | |||
| self.to_screen(u'[download] The file has already been downloaded') | |||
| def report_unable_to_resume(self): | |||
| """Report it was impossible to resume download.""" | |||
| self.to_screen(u'[download] Unable to resume') | |||
| def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live, conn): | |||
| def run_rtmpdump(args): | |||
| start = time.time() | |||
| resume_percent = None | |||
| resume_downloaded_data_len = None | |||
| proc = subprocess.Popen(args, stderr=subprocess.PIPE) | |||
| cursor_in_new_line = True | |||
| proc_stderr_closed = False | |||
| while not proc_stderr_closed: | |||
| # read line from stderr | |||
| line = u'' | |||
| while True: | |||
| char = proc.stderr.read(1) | |||
| if not char: | |||
| proc_stderr_closed = True | |||
| break | |||
| if char in [b'\r', b'\n']: | |||
| break | |||
| line += char.decode('ascii', 'replace') | |||
| if not line: | |||
| # proc_stderr_closed is True | |||
| continue | |||
| mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line) | |||
| if mobj: | |||
| downloaded_data_len = int(float(mobj.group(1))*1024) | |||
| percent = float(mobj.group(2)) | |||
| if not resume_percent: | |||
| resume_percent = percent | |||
| resume_downloaded_data_len = downloaded_data_len | |||
| eta = self.calc_eta(start, time.time(), 100-resume_percent, percent-resume_percent) | |||
| speed = self.calc_speed(start, time.time(), downloaded_data_len-resume_downloaded_data_len) | |||
| data_len = None | |||
| if percent > 0: | |||
| data_len = int(downloaded_data_len * 100 / percent) | |||
| data_len_str = u'~' + format_bytes(data_len) | |||
| self.report_progress(percent, data_len_str, speed, eta) | |||
| cursor_in_new_line = False | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': downloaded_data_len, | |||
| 'total_bytes': data_len, | |||
| 'tmpfilename': tmpfilename, | |||
| 'filename': filename, | |||
| 'status': 'downloading', | |||
| 'eta': eta, | |||
| 'speed': speed, | |||
| }) | |||
| else: | |||
| # no percent for live streams | |||
| mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line) | |||
| if mobj: | |||
| downloaded_data_len = int(float(mobj.group(1))*1024) | |||
| time_now = time.time() | |||
| speed = self.calc_speed(start, time_now, downloaded_data_len) | |||
| self.report_progress_live_stream(downloaded_data_len, speed, time_now - start) | |||
| cursor_in_new_line = False | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': downloaded_data_len, | |||
| 'tmpfilename': tmpfilename, | |||
| 'filename': filename, | |||
| 'status': 'downloading', | |||
| 'speed': speed, | |||
| }) | |||
| elif self.params.get('verbose', False): | |||
| if not cursor_in_new_line: | |||
| self.to_screen(u'') | |||
| cursor_in_new_line = True | |||
| self.to_screen(u'[rtmpdump] '+line) | |||
| proc.wait() | |||
| if not cursor_in_new_line: | |||
| self.to_screen(u'') | |||
| return proc.returncode | |||
| self.report_destination(filename) | |||
| tmpfilename = self.temp_name(filename) | |||
| test = self.params.get('test', False) | |||
| # Check for rtmpdump first | |||
| try: | |||
| subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) | |||
| except (OSError, IOError): | |||
| self.report_error(u'RTMP download detected but "rtmpdump" could not be run') | |||
| return False | |||
| # Download using rtmpdump. rtmpdump returns exit code 2 when | |||
| # the connection was interrumpted and resuming appears to be | |||
| # possible. This is part of rtmpdump's normal usage, AFAIK. | |||
| basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename] | |||
| if player_url is not None: | |||
| basic_args += ['--swfVfy', player_url] | |||
| if page_url is not None: | |||
| basic_args += ['--pageUrl', page_url] | |||
| if play_path is not None: | |||
| basic_args += ['--playpath', play_path] | |||
| if tc_url is not None: | |||
| basic_args += ['--tcUrl', url] | |||
| if test: | |||
| basic_args += ['--stop', '1'] | |||
| if live: | |||
| basic_args += ['--live'] | |||
| if conn: | |||
| basic_args += ['--conn', conn] | |||
| args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)] | |||
| if sys.platform == 'win32' and sys.version_info < (3, 0): | |||
| # Windows subprocess module does not actually support Unicode | |||
| # on Python 2.x | |||
| # See http://stackoverflow.com/a/9951851/35070 | |||
| subprocess_encoding = sys.getfilesystemencoding() | |||
| args = [a.encode(subprocess_encoding, 'ignore') for a in args] | |||
| else: | |||
| subprocess_encoding = None | |||
| if self.params.get('verbose', False): | |||
| if subprocess_encoding: | |||
| str_args = [ | |||
| a.decode(subprocess_encoding) if isinstance(a, bytes) else a | |||
| for a in args] | |||
| else: | |||
| str_args = args | |||
| try: | |||
| import pipes | |||
| shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) | |||
| except ImportError: | |||
| shell_quote = repr | |||
| self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(str_args)) | |||
| retval = run_rtmpdump(args) | |||
| while (retval == 2 or retval == 1) and not test: | |||
| prevsize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| self.to_screen(u'[rtmpdump] %s bytes' % prevsize) | |||
| time.sleep(5.0) # This seems to be needed | |||
| retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) | |||
| cursize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| if prevsize == cursize and retval == 1: | |||
| break | |||
| # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those | |||
| if prevsize == cursize and retval == 2 and cursize > 1024: | |||
| self.to_screen(u'[rtmpdump] Could not download the whole video. This can happen for some advertisements.') | |||
| retval = 0 | |||
| break | |||
| if retval == 0 or (test and retval == 2): | |||
| fsize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| self.to_screen(u'[rtmpdump] %s bytes' % fsize) | |||
| self.try_rename(tmpfilename, filename) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': fsize, | |||
| 'total_bytes': fsize, | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| else: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'rtmpdump exited with code %d' % retval) | |||
| return False | |||
| def _download_with_mplayer(self, filename, url): | |||
| self.report_destination(filename) | |||
| tmpfilename = self.temp_name(filename) | |||
| args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url] | |||
| # Check for mplayer first | |||
| try: | |||
| subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) | |||
| except (OSError, IOError): | |||
| self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0] ) | |||
| return False | |||
| # Download using mplayer. | |||
| retval = subprocess.call(args) | |||
| if retval == 0: | |||
| fsize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) | |||
| self.try_rename(tmpfilename, filename) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': fsize, | |||
| 'total_bytes': fsize, | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| else: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'mplayer exited with code %d' % retval) | |||
| return False | |||
| def _download_m3u8_with_ffmpeg(self, filename, url): | |||
| self.report_destination(filename) | |||
| tmpfilename = self.temp_name(filename) | |||
| args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy', | |||
| '-bsf:a', 'aac_adtstoasc', tmpfilename] | |||
| for program in ['avconv', 'ffmpeg']: | |||
| try: | |||
| subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) | |||
| break | |||
| except (OSError, IOError): | |||
| pass | |||
| else: | |||
| self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found') | |||
| cmd = [program] + args | |||
| retval = subprocess.call(cmd) | |||
| if retval == 0: | |||
| fsize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) | |||
| self.try_rename(tmpfilename, filename) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': fsize, | |||
| 'total_bytes': fsize, | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| else: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'ffmpeg exited with code %d' % retval) | |||
| return False | |||
| # Legacy file for backwards compatibility, use youtube_dl.downloader instead! | |||
| from .downloader import FileDownloader as RealFileDownloader | |||
| from .downloader import get_suitable_downloader | |||
| # This class reproduces the old behaviour of FileDownloader | |||
| class FileDownloader(RealFileDownloader): | |||
| def _do_download(self, filename, info_dict): | |||
| url = info_dict['url'] | |||
| # Check file already present | |||
| if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): | |||
| self.report_file_already_downloaded(filename) | |||
| self._hook_progress({ | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| 'total_bytes': os.path.getsize(encodeFilename(filename)), | |||
| }) | |||
| return True | |||
| # Attempt to download using rtmpdump | |||
| if url.startswith('rtmp'): | |||
| return self._download_with_rtmpdump(filename, url, | |||
| info_dict.get('player_url', None), | |||
| info_dict.get('page_url', None), | |||
| info_dict.get('play_path', None), | |||
| info_dict.get('tc_url', None), | |||
| info_dict.get('rtmp_live', False), | |||
| info_dict.get('rtmp_conn', None)) | |||
| # Attempt to download using mplayer | |||
| if url.startswith('mms') or url.startswith('rtsp'): | |||
| return self._download_with_mplayer(filename, url) | |||
| # m3u8 manifest are downloaded with ffmpeg | |||
| if determine_ext(url) == u'm3u8': | |||
| return self._download_m3u8_with_ffmpeg(filename, url) | |||
| tmpfilename = self.temp_name(filename) | |||
| stream = None | |||
| # Do not include the Accept-Encoding header | |||
| headers = {'Youtubedl-no-compression': 'True'} | |||
| if 'user_agent' in info_dict: | |||
| headers['Youtubedl-user-agent'] = info_dict['user_agent'] | |||
| basic_request = compat_urllib_request.Request(url, None, headers) | |||
| request = compat_urllib_request.Request(url, None, headers) | |||
| if self.params.get('test', False): | |||
| request.add_header('Range','bytes=0-10240') | |||
| # Establish possible resume length | |||
| if os.path.isfile(encodeFilename(tmpfilename)): | |||
| resume_len = os.path.getsize(encodeFilename(tmpfilename)) | |||
| else: | |||
| resume_len = 0 | |||
| open_mode = 'wb' | |||
| if resume_len != 0: | |||
| if self.params.get('continuedl', False): | |||
| self.report_resuming_byte(resume_len) | |||
| request.add_header('Range','bytes=%d-' % resume_len) | |||
| open_mode = 'ab' | |||
| else: | |||
| resume_len = 0 | |||
| count = 0 | |||
| retries = self.params.get('retries', 0) | |||
| while count <= retries: | |||
| # Establish connection | |||
| try: | |||
| if count == 0 and 'urlhandle' in info_dict: | |||
| data = info_dict['urlhandle'] | |||
| data = compat_urllib_request.urlopen(request) | |||
| break | |||
| except (compat_urllib_error.HTTPError, ) as err: | |||
| if (err.code < 500 or err.code >= 600) and err.code != 416: | |||
| # Unexpected HTTP error | |||
| raise | |||
| elif err.code == 416: | |||
| # Unable to resume (requested range not satisfiable) | |||
| try: | |||
| # Open the connection again without the range header | |||
| data = compat_urllib_request.urlopen(basic_request) | |||
| content_length = data.info()['Content-Length'] | |||
| except (compat_urllib_error.HTTPError, ) as err: | |||
| if err.code < 500 or err.code >= 600: | |||
| raise | |||
| else: | |||
| # Examine the reported length | |||
| if (content_length is not None and | |||
| (resume_len - 100 < int(content_length) < resume_len + 100)): | |||
| # The file had already been fully downloaded. | |||
| # Explanation to the above condition: in issue #175 it was revealed that | |||
| # YouTube sometimes adds or removes a few bytes from the end of the file, | |||
| # changing the file size slightly and causing problems for some users. So | |||
| # I decided to implement a suggested change and consider the file | |||
| # completely downloaded if the file size differs less than 100 bytes from | |||
| # the one in the hard drive. | |||
| self.report_file_already_downloaded(filename) | |||
| self.try_rename(tmpfilename, filename) | |||
| self._hook_progress({ | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| else: | |||
| # The length does not match, we start the download over | |||
| self.report_unable_to_resume() | |||
| open_mode = 'wb' | |||
| break | |||
| # Retry | |||
| count += 1 | |||
| if count <= retries: | |||
| self.report_retry(count, retries) | |||
| if count > retries: | |||
| self.report_error(u'giving up after %s retries' % retries) | |||
| return False | |||
| data_len = data.info().get('Content-length', None) | |||
| if data_len is not None: | |||
| data_len = int(data_len) + resume_len | |||
| min_data_len = self.params.get("min_filesize", None) | |||
| max_data_len = self.params.get("max_filesize", None) | |||
| if min_data_len is not None and data_len < min_data_len: | |||
| self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len)) | |||
| return False | |||
| if max_data_len is not None and data_len > max_data_len: | |||
| self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len)) | |||
| return False | |||
| data_len_str = format_bytes(data_len) | |||
| byte_counter = 0 + resume_len | |||
| block_size = self.params.get('buffersize', 1024) | |||
| start = time.time() | |||
| while True: | |||
| # Download and write | |||
| before = time.time() | |||
| data_block = data.read(block_size) | |||
| after = time.time() | |||
| if len(data_block) == 0: | |||
| break | |||
| byte_counter += len(data_block) | |||
| # Open file just in time | |||
| if stream is None: | |||
| try: | |||
| (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) | |||
| assert stream is not None | |||
| filename = self.undo_temp_name(tmpfilename) | |||
| self.report_destination(filename) | |||
| except (OSError, IOError) as err: | |||
| self.report_error(u'unable to open for writing: %s' % str(err)) | |||
| return False | |||
| try: | |||
| stream.write(data_block) | |||
| except (IOError, OSError) as err: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'unable to write data: %s' % str(err)) | |||
| return False | |||
| if not self.params.get('noresizebuffer', False): | |||
| block_size = self.best_block_size(after - before, len(data_block)) | |||
| # Progress message | |||
| speed = self.calc_speed(start, time.time(), byte_counter - resume_len) | |||
| if data_len is None: | |||
| eta = percent = None | |||
| else: | |||
| percent = self.calc_percent(byte_counter, data_len) | |||
| eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len) | |||
| self.report_progress(percent, data_len_str, speed, eta) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': byte_counter, | |||
| 'total_bytes': data_len, | |||
| 'tmpfilename': tmpfilename, | |||
| 'filename': filename, | |||
| 'status': 'downloading', | |||
| 'eta': eta, | |||
| 'speed': speed, | |||
| }) | |||
| # Apply rate limit | |||
| self.slow_down(start, byte_counter - resume_len) | |||
| if stream is None: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'Did not get any data blocks') | |||
| return False | |||
| stream.close() | |||
| self.report_finish(data_len_str, (time.time() - start)) | |||
| if data_len is not None and byte_counter != data_len: | |||
| raise ContentTooShortError(byte_counter, int(data_len)) | |||
| self.try_rename(tmpfilename, filename) | |||
| # Update file modification time | |||
| if self.params.get('updatetime', True): | |||
| info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': byte_counter, | |||
| 'total_bytes': byte_counter, | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| def _hook_progress(self, status): | |||
| real_fd = get_suitable_downloader(info_dict)(self.ydl, self.params) | |||
| for ph in self._progress_hooks: | |||
| ph(status) | |||
| def add_progress_hook(self, ph): | |||
| """ ph gets called on download progress, with a dictionary with the entries | |||
| * filename: The final filename | |||
| * status: One of "downloading" and "finished" | |||
| It can also have some of the following entries: | |||
| * downloaded_bytes: Bytes on disks | |||
| * total_bytes: Total bytes, None if unknown | |||
| * tmpfilename: The filename we're currently writing to | |||
| * eta: The estimated time in seconds, None if unknown | |||
| * speed: The download speed in bytes/second, None if unknown | |||
| Hooks are guaranteed to be called at least once (with status "finished") | |||
| if the download is successful. | |||
| """ | |||
| self._progress_hooks.append(ph) | |||
| real_fd.add_progress_hook(ph) | |||
| return real_fd.download(filename, info_dict) | |||
+ 10
- 5
youtube_dl/YoutubeDL.py
View File
+ 23
- 0
youtube_dl/downloader/__init__.py
View File
| @ -0,0 +1,23 @@ | |||
| from .common import FileDownloader | |||
| from .hls import HlsFD | |||
| from .http import HttpFD | |||
| from .mplayer import MplayerFD | |||
| from .rtmp import RtmpFD | |||
| from ..utils import ( | |||
| determine_ext, | |||
| ) | |||
| def get_suitable_downloader(info_dict): | |||
| """Get the downloader class that can handle the info dict.""" | |||
| url = info_dict['url'] | |||
| if url.startswith('rtmp'): | |||
| return RtmpFD | |||
| if determine_ext(url) == u'm3u8': | |||
| return HlsFD | |||
| if url.startswith('mms') or url.startswith('rtsp'): | |||
| return MplayerFD | |||
| else: | |||
| return HttpFD | |||
+ 321
- 0
youtube_dl/downloader/common.py
View File
| @ -0,0 +1,321 @@ | |||
| import math | |||
| import os | |||
| import re | |||
| import subprocess | |||
| import sys | |||
| import time | |||
| from ..utils import ( | |||
| encodeFilename, | |||
| timeconvert, | |||
| format_bytes, | |||
| ) | |||
| class FileDownloader(object): | |||
| """File Downloader class. | |||
| File downloader objects are the ones responsible of downloading the | |||
| actual video file and writing it to disk. | |||
| File downloaders accept a lot of parameters. In order not to saturate | |||
| the object constructor with arguments, it receives a dictionary of | |||
| options instead. | |||
| Available options: | |||
| verbose: Print additional info to stdout. | |||
| quiet: Do not print messages to stdout. | |||
| ratelimit: Download speed limit, in bytes/sec. | |||
| retries: Number of times to retry for HTTP error 5xx | |||
| buffersize: Size of download buffer in bytes. | |||
| noresizebuffer: Do not automatically resize the download buffer. | |||
| continuedl: Try to continue downloads if possible. | |||
| noprogress: Do not print the progress bar. | |||
| logtostderr: Log messages to stderr instead of stdout. | |||
| consoletitle: Display progress in console window's titlebar. | |||
| nopart: Do not use temporary .part files. | |||
| updatetime: Use the Last-modified header to set output file timestamps. | |||
| test: Download only first bytes to test the downloader. | |||
| min_filesize: Skip files smaller than this size | |||
| max_filesize: Skip files larger than this size | |||
| Subclasses of this one must re-define the real_download method. | |||
| """ | |||
| params = None | |||
| def __init__(self, ydl, params): | |||
| """Create a FileDownloader object with the given options.""" | |||
| self.ydl = ydl | |||
| self._progress_hooks = [] | |||
| self.params = params | |||
| @staticmethod | |||
| def format_seconds(seconds): | |||
| (mins, secs) = divmod(seconds, 60) | |||
| (hours, mins) = divmod(mins, 60) | |||
| if hours > 99: | |||
| return '--:--:--' | |||
| if hours == 0: | |||
| return '%02d:%02d' % (mins, secs) | |||
| else: | |||
| return '%02d:%02d:%02d' % (hours, mins, secs) | |||
| @staticmethod | |||
| def calc_percent(byte_counter, data_len): | |||
| if data_len is None: | |||
| return None | |||
| return float(byte_counter) / float(data_len) * 100.0 | |||
| @staticmethod | |||
| def format_percent(percent): | |||
| if percent is None: | |||
| return '---.-%' | |||
| return '%6s' % ('%3.1f%%' % percent) | |||
| @staticmethod | |||
| def calc_eta(start, now, total, current): | |||
| if total is None: | |||
| return None | |||
| dif = now - start | |||
| if current == 0 or dif < 0.001: # One millisecond | |||
| return None | |||
| rate = float(current) / dif | |||
| return int((float(total) - float(current)) / rate) | |||
| @staticmethod | |||
| def format_eta(eta): | |||
| if eta is None: | |||
| return '--:--' | |||
| return FileDownloader.format_seconds(eta) | |||
| @staticmethod | |||
| def calc_speed(start, now, bytes): | |||
| dif = now - start | |||
| if bytes == 0 or dif < 0.001: # One millisecond | |||
| return None | |||
| return float(bytes) / dif | |||
| @staticmethod | |||
| def format_speed(speed): | |||
| if speed is None: | |||
| return '%10s' % '---b/s' | |||
| return '%10s' % ('%s/s' % format_bytes(speed)) | |||
| @staticmethod | |||
| def best_block_size(elapsed_time, bytes): | |||
| new_min = max(bytes / 2.0, 1.0) | |||
| new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB | |||
| if elapsed_time < 0.001: | |||
| return int(new_max) | |||
| rate = bytes / elapsed_time | |||
| if rate > new_max: | |||
| return int(new_max) | |||
| if rate < new_min: | |||
| return int(new_min) | |||
| return int(rate) | |||
| @staticmethod | |||
| def parse_bytes(bytestr): | |||
| """Parse a string indicating a byte quantity into an integer.""" | |||
| matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) | |||
| if matchobj is None: | |||
| return None | |||
| number = float(matchobj.group(1)) | |||
| multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) | |||
| return int(round(number * multiplier)) | |||
| def to_screen(self, *args, **kargs): | |||
| self.ydl.to_screen(*args, **kargs) | |||
| def to_stderr(self, message): | |||
| self.ydl.to_screen(message) | |||
| def to_console_title(self, message): | |||
| self.ydl.to_console_title(message) | |||
| def trouble(self, *args, **kargs): | |||
| self.ydl.trouble(*args, **kargs) | |||
| def report_warning(self, *args, **kargs): | |||
| self.ydl.report_warning(*args, **kargs) | |||
| def report_error(self, *args, **kargs): | |||
| self.ydl.report_error(*args, **kargs) | |||
| def slow_down(self, start_time, byte_counter): | |||
| """Sleep if the download speed is over the rate limit.""" | |||
| rate_limit = self.params.get('ratelimit', None) | |||
| if rate_limit is None or byte_counter == 0: | |||
| return | |||
| now = time.time() | |||
| elapsed = now - start_time | |||
| if elapsed <= 0.0: | |||
| return | |||
| speed = float(byte_counter) / elapsed | |||
| if speed > rate_limit: | |||
| time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) | |||
| def temp_name(self, filename): | |||
| """Returns a temporary filename for the given filename.""" | |||
| if self.params.get('nopart', False) or filename == u'-' or \ | |||
| (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): | |||
| return filename | |||
| return filename + u'.part' | |||
| def undo_temp_name(self, filename): | |||
| if filename.endswith(u'.part'): | |||
| return filename[:-len(u'.part')] | |||
| return filename | |||
| def try_rename(self, old_filename, new_filename): | |||
| try: | |||
| if old_filename == new_filename: | |||
| return | |||
| os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) | |||
| except (IOError, OSError) as err: | |||
| self.report_error(u'unable to rename file') | |||
| def try_utime(self, filename, last_modified_hdr): | |||
| """Try to set the last-modified time of the given file.""" | |||
| if last_modified_hdr is None: | |||
| return | |||
| if not os.path.isfile(encodeFilename(filename)): | |||
| return | |||
| timestr = last_modified_hdr | |||
| if timestr is None: | |||
| return | |||
| filetime = timeconvert(timestr) | |||
| if filetime is None: | |||
| return filetime | |||
| # Ignore obviously invalid dates | |||
| if filetime == 0: | |||
| return | |||
| try: | |||
| os.utime(filename, (time.time(), filetime)) | |||
| except: | |||
| pass | |||
| return filetime | |||
| def report_destination(self, filename): | |||
| """Report destination filename.""" | |||
| self.to_screen(u'[download] Destination: ' + filename) | |||
| def _report_progress_status(self, msg, is_last_line=False): | |||
| fullmsg = u'[download] ' + msg | |||
| if self.params.get('progress_with_newline', False): | |||
| self.to_screen(fullmsg) | |||
| else: | |||
| if os.name == 'nt': | |||
| prev_len = getattr(self, '_report_progress_prev_line_length', | |||
| 0) | |||
| if prev_len > len(fullmsg): | |||
| fullmsg += u' ' * (prev_len - len(fullmsg)) | |||
| self._report_progress_prev_line_length = len(fullmsg) | |||
| clear_line = u'\r' | |||
| else: | |||
| clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r') | |||
| self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) | |||
| self.to_console_title(u'youtube-dl ' + msg) | |||
| def report_progress(self, percent, data_len_str, speed, eta): | |||
| """Report download progress.""" | |||
| if self.params.get('noprogress', False): | |||
| return | |||
| if eta is not None: | |||
| eta_str = self.format_eta(eta) | |||
| else: | |||
| eta_str = 'Unknown ETA' | |||
| if percent is not None: | |||
| percent_str = self.format_percent(percent) | |||
| else: | |||
| percent_str = 'Unknown %' | |||
| speed_str = self.format_speed(speed) | |||
| msg = (u'%s of %s at %s ETA %s' % | |||
| (percent_str, data_len_str, speed_str, eta_str)) | |||
| self._report_progress_status(msg) | |||
| def report_progress_live_stream(self, downloaded_data_len, speed, elapsed): | |||
| if self.params.get('noprogress', False): | |||
| return | |||
| downloaded_str = format_bytes(downloaded_data_len) | |||
| speed_str = self.format_speed(speed) | |||
| elapsed_str = FileDownloader.format_seconds(elapsed) | |||
| msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str) | |||
| self._report_progress_status(msg) | |||
| def report_finish(self, data_len_str, tot_time): | |||
| """Report download finished.""" | |||
| if self.params.get('noprogress', False): | |||
| self.to_screen(u'[download] Download completed') | |||
| else: | |||
| self._report_progress_status( | |||
| (u'100%% of %s in %s' % | |||
| (data_len_str, self.format_seconds(tot_time))), | |||
| is_last_line=True) | |||
| def report_resuming_byte(self, resume_len): | |||
| """Report attempt to resume at given byte.""" | |||
| self.to_screen(u'[download] Resuming download at byte %s' % resume_len) | |||
| def report_retry(self, count, retries): | |||
| """Report retry in case of HTTP error 5xx""" | |||
| self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) | |||
| def report_file_already_downloaded(self, file_name): | |||
| """Report file has already been fully downloaded.""" | |||
| try: | |||
| self.to_screen(u'[download] %s has already been downloaded' % file_name) | |||
| except UnicodeEncodeError: | |||
| self.to_screen(u'[download] The file has already been downloaded') | |||
| def report_unable_to_resume(self): | |||
| """Report it was impossible to resume download.""" | |||
| self.to_screen(u'[download] Unable to resume') | |||
| def download(self, filename, info_dict): | |||
| """Download to a filename using the info from info_dict | |||
| Return True on success and False otherwise | |||
| """ | |||
| url = info_dict['url'] | |||
| # Check file already present | |||
| if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): | |||
| self.report_file_already_downloaded(filename) | |||
| self._hook_progress({ | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| 'total_bytes': os.path.getsize(encodeFilename(filename)), | |||
| }) | |||
| return True | |||
| else: | |||
| return self.real_download(filename, info_dict) | |||
| def real_download(self, filename, info_dict): | |||
| """Real download process. Redefine in subclasses.""" | |||
| raise NotImplementedError(u'This method must be implemented by sublcasses') | |||
| def _hook_progress(self, status): | |||
| for ph in self._progress_hooks: | |||
| ph(status) | |||
| def add_progress_hook(self, ph): | |||
| """ ph gets called on download progress, with a dictionary with the entries | |||
| * filename: The final filename | |||
| * status: One of "downloading" and "finished" | |||
| It can also have some of the following entries: | |||
| * downloaded_bytes: Bytes on disks | |||
| * total_bytes: Total bytes, None if unknown | |||
| * tmpfilename: The filename we're currently writing to | |||
| * eta: The estimated time in seconds, None if unknown | |||
| * speed: The download speed in bytes/second, None if unknown | |||
| Hooks are guaranteed to be called at least once (with status "finished") | |||
| if the download is successful. | |||
| """ | |||
| self._progress_hooks.append(ph) | |||
+ 44
- 0
youtube_dl/downloader/hls.py
View File
| @ -0,0 +1,44 @@ | |||
| import os | |||
| import subprocess | |||
| from .common import FileDownloader | |||
| from ..utils import ( | |||
| encodeFilename, | |||
| ) | |||
| class HlsFD(FileDownloader): | |||
| def real_download(self, filename, info_dict): | |||
| url = info_dict['url'] | |||
| self.report_destination(filename) | |||
| tmpfilename = self.temp_name(filename) | |||
| args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy', | |||
| '-bsf:a', 'aac_adtstoasc', tmpfilename] | |||
| for program in ['avconv', 'ffmpeg']: | |||
| try: | |||
| subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) | |||
| break | |||
| except (OSError, IOError): | |||
| pass | |||
| else: | |||
| self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found') | |||
| cmd = [program] + args | |||
| retval = subprocess.call(cmd) | |||
| if retval == 0: | |||
| fsize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) | |||
| self.try_rename(tmpfilename, filename) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': fsize, | |||
| 'total_bytes': fsize, | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| else: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'ffmpeg exited with code %d' % retval) | |||
| return False | |||
+ 191
- 0
youtube_dl/downloader/http.py
View File
| @ -0,0 +1,191 @@ | |||
| import os | |||
| import re | |||
| import subprocess | |||
| import sys | |||
| import time | |||
| from .common import FileDownloader | |||
| from ..utils import ( | |||
| compat_urllib_request, | |||
| compat_urllib_error, | |||
| ContentTooShortError, | |||
| encodeFilename, | |||
| sanitize_open, | |||
| format_bytes, | |||
| ) | |||
| class HttpFD(FileDownloader): | |||
| def real_download(self, filename, info_dict): | |||
| url = info_dict['url'] | |||
| tmpfilename = self.temp_name(filename) | |||
| stream = None | |||
| # Do not include the Accept-Encoding header | |||
| headers = {'Youtubedl-no-compression': 'True'} | |||
| if 'user_agent' in info_dict: | |||
| headers['Youtubedl-user-agent'] = info_dict['user_agent'] | |||
| basic_request = compat_urllib_request.Request(url, None, headers) | |||
| request = compat_urllib_request.Request(url, None, headers) | |||
| if self.params.get('test', False): | |||
| request.add_header('Range','bytes=0-10240') | |||
| # Establish possible resume length | |||
| if os.path.isfile(encodeFilename(tmpfilename)): | |||
| resume_len = os.path.getsize(encodeFilename(tmpfilename)) | |||
| else: | |||
| resume_len = 0 | |||
| open_mode = 'wb' | |||
| if resume_len != 0: | |||
| if self.params.get('continuedl', False): | |||
| self.report_resuming_byte(resume_len) | |||
| request.add_header('Range','bytes=%d-' % resume_len) | |||
| open_mode = 'ab' | |||
| else: | |||
| resume_len = 0 | |||
| count = 0 | |||
| retries = self.params.get('retries', 0) | |||
| while count <= retries: | |||
| # Establish connection | |||
| try: | |||
| if count == 0 and 'urlhandle' in info_dict: | |||
| data = info_dict['urlhandle'] | |||
| data = compat_urllib_request.urlopen(request) | |||
| break | |||
| except (compat_urllib_error.HTTPError, ) as err: | |||
| if (err.code < 500 or err.code >= 600) and err.code != 416: | |||
| # Unexpected HTTP error | |||
| raise | |||
| elif err.code == 416: | |||
| # Unable to resume (requested range not satisfiable) | |||
| try: | |||
| # Open the connection again without the range header | |||
| data = compat_urllib_request.urlopen(basic_request) | |||
| content_length = data.info()['Content-Length'] | |||
| except (compat_urllib_error.HTTPError, ) as err: | |||
| if err.code < 500 or err.code >= 600: | |||
| raise | |||
| else: | |||
| # Examine the reported length | |||
| if (content_length is not None and | |||
| (resume_len - 100 < int(content_length) < resume_len + 100)): | |||
| # The file had already been fully downloaded. | |||
| # Explanation to the above condition: in issue #175 it was revealed that | |||
| # YouTube sometimes adds or removes a few bytes from the end of the file, | |||
| # changing the file size slightly and causing problems for some users. So | |||
| # I decided to implement a suggested change and consider the file | |||
| # completely downloaded if the file size differs less than 100 bytes from | |||
| # the one in the hard drive. | |||
| self.report_file_already_downloaded(filename) | |||
| self.try_rename(tmpfilename, filename) | |||
| self._hook_progress({ | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| else: | |||
| # The length does not match, we start the download over | |||
| self.report_unable_to_resume() | |||
| open_mode = 'wb' | |||
| break | |||
| # Retry | |||
| count += 1 | |||
| if count <= retries: | |||
| self.report_retry(count, retries) | |||
| if count > retries: | |||
| self.report_error(u'giving up after %s retries' % retries) | |||
| return False | |||
| data_len = data.info().get('Content-length', None) | |||
| if data_len is not None: | |||
| data_len = int(data_len) + resume_len | |||
| min_data_len = self.params.get("min_filesize", None) | |||
| max_data_len = self.params.get("max_filesize", None) | |||
| if min_data_len is not None and data_len < min_data_len: | |||
| self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len)) | |||
| return False | |||
| if max_data_len is not None and data_len > max_data_len: | |||
| self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len)) | |||
| return False | |||
| data_len_str = format_bytes(data_len) | |||
| byte_counter = 0 + resume_len | |||
| block_size = self.params.get('buffersize', 1024) | |||
| start = time.time() | |||
| while True: | |||
| # Download and write | |||
| before = time.time() | |||
| data_block = data.read(block_size) | |||
| after = time.time() | |||
| if len(data_block) == 0: | |||
| break | |||
| byte_counter += len(data_block) | |||
| # Open file just in time | |||
| if stream is None: | |||
| try: | |||
| (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) | |||
| assert stream is not None | |||
| filename = self.undo_temp_name(tmpfilename) | |||
| self.report_destination(filename) | |||
| except (OSError, IOError) as err: | |||
| self.report_error(u'unable to open for writing: %s' % str(err)) | |||
| return False | |||
| try: | |||
| stream.write(data_block) | |||
| except (IOError, OSError): | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'unable to write data: %s' % str(err)) | |||
| return False | |||
| if not self.params.get('noresizebuffer', False): | |||
| block_size = self.best_block_size(after - before, len(data_block)) | |||
| # Progress message | |||
| speed = self.calc_speed(start, time.time(), byte_counter - resume_len) | |||
| if data_len is None: | |||
| eta = percent = None | |||
| else: | |||
| percent = self.calc_percent(byte_counter, data_len) | |||
| eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len) | |||
| self.report_progress(percent, data_len_str, speed, eta) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': byte_counter, | |||
| 'total_bytes': data_len, | |||
| 'tmpfilename': tmpfilename, | |||
| 'filename': filename, | |||
| 'status': 'downloading', | |||
| 'eta': eta, | |||
| 'speed': speed, | |||
| }) | |||
| # Apply rate limit | |||
| self.slow_down(start, byte_counter - resume_len) | |||
| if stream is None: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'Did not get any data blocks') | |||
| return False | |||
| stream.close() | |||
| self.report_finish(data_len_str, (time.time() - start)) | |||
| if data_len is not None and byte_counter != data_len: | |||
| raise ContentTooShortError(byte_counter, int(data_len)) | |||
| self.try_rename(tmpfilename, filename) | |||
| # Update file modification time | |||
| if self.params.get('updatetime', True): | |||
| info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': byte_counter, | |||
| 'total_bytes': byte_counter, | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
+ 39
- 0
youtube_dl/downloader/mplayer.py
View File
| @ -0,0 +1,39 @@ | |||
| import os | |||
| import subprocess | |||
| from .common import FileDownloader | |||
| from ..utils import ( | |||
| encodeFilename, | |||
| ) | |||
| class MplayerFD(FileDownloader): | |||
| def real_download(self, filename, info_dict): | |||
| self.report_destination(filename) | |||
| tmpfilename = self.temp_name(filename) | |||
| args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url] | |||
| # Check for mplayer first | |||
| try: | |||
| subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) | |||
| except (OSError, IOError): | |||
| self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0] ) | |||
| return False | |||
| # Download using mplayer. | |||
| retval = subprocess.call(args) | |||
| if retval == 0: | |||
| fsize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) | |||
| self.try_rename(tmpfilename, filename) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': fsize, | |||
| 'total_bytes': fsize, | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| else: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'mplayer exited with code %d' % retval) | |||
| return False | |||
+ 178
- 0
youtube_dl/downloader/rtmp.py
View File
| @ -0,0 +1,178 @@ | |||
| import os | |||
| import re | |||
| import subprocess | |||
| import sys | |||
| import time | |||
| from .common import FileDownloader | |||
| from ..utils import ( | |||
| encodeFilename, | |||
| format_bytes, | |||
| ) | |||
| class RtmpFD(FileDownloader): | |||
| def real_download(self, filename, info_dict): | |||
| def run_rtmpdump(args): | |||
| start = time.time() | |||
| resume_percent = None | |||
| resume_downloaded_data_len = None | |||
| proc = subprocess.Popen(args, stderr=subprocess.PIPE) | |||
| cursor_in_new_line = True | |||
| proc_stderr_closed = False | |||
| while not proc_stderr_closed: | |||
| # read line from stderr | |||
| line = u'' | |||
| while True: | |||
| char = proc.stderr.read(1) | |||
| if not char: | |||
| proc_stderr_closed = True | |||
| break | |||
| if char in [b'\r', b'\n']: | |||
| break | |||
| line += char.decode('ascii', 'replace') | |||
| if not line: | |||
| # proc_stderr_closed is True | |||
| continue | |||
| mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line) | |||
| if mobj: | |||
| downloaded_data_len = int(float(mobj.group(1))*1024) | |||
| percent = float(mobj.group(2)) | |||
| if not resume_percent: | |||
| resume_percent = percent | |||
| resume_downloaded_data_len = downloaded_data_len | |||
| eta = self.calc_eta(start, time.time(), 100-resume_percent, percent-resume_percent) | |||
| speed = self.calc_speed(start, time.time(), downloaded_data_len-resume_downloaded_data_len) | |||
| data_len = None | |||
| if percent > 0: | |||
| data_len = int(downloaded_data_len * 100 / percent) | |||
| data_len_str = u'~' + format_bytes(data_len) | |||
| self.report_progress(percent, data_len_str, speed, eta) | |||
| cursor_in_new_line = False | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': downloaded_data_len, | |||
| 'total_bytes': data_len, | |||
| 'tmpfilename': tmpfilename, | |||
| 'filename': filename, | |||
| 'status': 'downloading', | |||
| 'eta': eta, | |||
| 'speed': speed, | |||
| }) | |||
| else: | |||
| # no percent for live streams | |||
| mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line) | |||
| if mobj: | |||
| downloaded_data_len = int(float(mobj.group(1))*1024) | |||
| time_now = time.time() | |||
| speed = self.calc_speed(start, time_now, downloaded_data_len) | |||
| self.report_progress_live_stream(downloaded_data_len, speed, time_now - start) | |||
| cursor_in_new_line = False | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': downloaded_data_len, | |||
| 'tmpfilename': tmpfilename, | |||
| 'filename': filename, | |||
| 'status': 'downloading', | |||
| 'speed': speed, | |||
| }) | |||
| elif self.params.get('verbose', False): | |||
| if not cursor_in_new_line: | |||
| self.to_screen(u'') | |||
| cursor_in_new_line = True | |||
| self.to_screen(u'[rtmpdump] '+line) | |||
| proc.wait() | |||
| if not cursor_in_new_line: | |||
| self.to_screen(u'') | |||
| return proc.returncode | |||
| url = info_dict['url'] | |||
| player_url = info_dict.get('player_url', None) | |||
| page_url = info_dict.get('page_url', None) | |||
| play_path = info_dict.get('play_path', None) | |||
| tc_url = info_dict.get('tc_url', None) | |||
| live = info_dict.get('rtmp_live', False) | |||
| conn = info_dict.get('rtmp_conn', None) | |||
| self.report_destination(filename) | |||
| tmpfilename = self.temp_name(filename) | |||
| test = self.params.get('test', False) | |||
| # Check for rtmpdump first | |||
| try: | |||
| subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) | |||
| except (OSError, IOError): | |||
| self.report_error(u'RTMP download detected but "rtmpdump" could not be run') | |||
| return False | |||
| # Download using rtmpdump. rtmpdump returns exit code 2 when | |||
| # the connection was interrumpted and resuming appears to be | |||
| # possible. This is part of rtmpdump's normal usage, AFAIK. | |||
| basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename] | |||
| if player_url is not None: | |||
| basic_args += ['--swfVfy', player_url] | |||
| if page_url is not None: | |||
| basic_args += ['--pageUrl', page_url] | |||
| if play_path is not None: | |||
| basic_args += ['--playpath', play_path] | |||
| if tc_url is not None: | |||
| basic_args += ['--tcUrl', url] | |||
| if test: | |||
| basic_args += ['--stop', '1'] | |||
| if live: | |||
| basic_args += ['--live'] | |||
| if conn: | |||
| basic_args += ['--conn', conn] | |||
| args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)] | |||
| if sys.platform == 'win32' and sys.version_info < (3, 0): | |||
| # Windows subprocess module does not actually support Unicode | |||
| # on Python 2.x | |||
| # See http://stackoverflow.com/a/9951851/35070 | |||
| subprocess_encoding = sys.getfilesystemencoding() | |||
| args = [a.encode(subprocess_encoding, 'ignore') for a in args] | |||
| else: | |||
| subprocess_encoding = None | |||
| if self.params.get('verbose', False): | |||
| if subprocess_encoding: | |||
| str_args = [ | |||
| a.decode(subprocess_encoding) if isinstance(a, bytes) else a | |||
| for a in args] | |||
| else: | |||
| str_args = args | |||
| try: | |||
| import pipes | |||
| shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) | |||
| except ImportError: | |||
| shell_quote = repr | |||
| self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(str_args)) | |||
| retval = run_rtmpdump(args) | |||
| while (retval == 2 or retval == 1) and not test: | |||
| prevsize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| self.to_screen(u'[rtmpdump] %s bytes' % prevsize) | |||
| time.sleep(5.0) # This seems to be needed | |||
| retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) | |||
| cursize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| if prevsize == cursize and retval == 1: | |||
| break | |||
| # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those | |||
| if prevsize == cursize and retval == 2 and cursize > 1024: | |||
| self.to_screen(u'[rtmpdump] Could not download the whole video. This can happen for some advertisements.') | |||
| retval = 0 | |||
| break | |||
| if retval == 0 or (test and retval == 2): | |||
| fsize = os.path.getsize(encodeFilename(tmpfilename)) | |||
| self.to_screen(u'[rtmpdump] %s bytes' % fsize) | |||
| self.try_rename(tmpfilename, filename) | |||
| self._hook_progress({ | |||
| 'downloaded_bytes': fsize, | |||
| 'total_bytes': fsize, | |||
| 'filename': filename, | |||
| 'status': 'finished', | |||
| }) | |||
| return True | |||
| else: | |||
| self.to_stderr(u"\n") | |||
| self.report_error(u'rtmpdump exited with code %d' % retval) | |||
| return False | |||