You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

354 lines
15 KiB

10 years ago
Switch codebase to use sanitized_Request instead of compat_urllib_request.Request [downloader/dash] Use sanitized_Request [downloader/http] Use sanitized_Request [atresplayer] Use sanitized_Request [bambuser] Use sanitized_Request [bliptv] Use sanitized_Request [brightcove] Use sanitized_Request [cbs] Use sanitized_Request [ceskatelevize] Use sanitized_Request [collegerama] Use sanitized_Request [extractor/common] Use sanitized_Request [crunchyroll] Use sanitized_Request [dailymotion] Use sanitized_Request [dcn] Use sanitized_Request [dramafever] Use sanitized_Request [dumpert] Use sanitized_Request [eitb] Use sanitized_Request [escapist] Use sanitized_Request [everyonesmixtape] Use sanitized_Request [extremetube] Use sanitized_Request [facebook] Use sanitized_Request [fc2] Use sanitized_Request [flickr] Use sanitized_Request [4tube] Use sanitized_Request [gdcvault] Use sanitized_Request [extractor/generic] Use sanitized_Request [hearthisat] Use sanitized_Request [hotnewhiphop] Use sanitized_Request [hypem] Use sanitized_Request [iprima] Use sanitized_Request [ivi] Use sanitized_Request [keezmovies] Use sanitized_Request [letv] Use sanitized_Request [lynda] Use sanitized_Request [metacafe] Use sanitized_Request [minhateca] Use sanitized_Request [miomio] Use sanitized_Request [meovideo] Use sanitized_Request [mofosex] Use sanitized_Request [moniker] Use sanitized_Request [mooshare] Use sanitized_Request [movieclips] Use sanitized_Request [mtv] Use sanitized_Request [myvideo] Use sanitized_Request [neteasemusic] Use sanitized_Request [nfb] Use sanitized_Request [niconico] Use sanitized_Request [noco] Use sanitized_Request [nosvideo] Use sanitized_Request [novamov] Use sanitized_Request [nowness] Use sanitized_Request [nuvid] Use sanitized_Request [played] Use sanitized_Request [pluralsight] Use sanitized_Request [pornhub] Use sanitized_Request [pornotube] Use sanitized_Request [primesharetv] Use sanitized_Request [promptfile] Use sanitized_Request [qqmusic] Use sanitized_Request [rtve] Use sanitized_Request [safari] Use sanitized_Request [sandia] Use sanitized_Request [shared] Use sanitized_Request [sharesix] Use sanitized_Request [sina] Use sanitized_Request [smotri] Use sanitized_Request [sohu] Use sanitized_Request [spankwire] Use sanitized_Request [sportdeutschland] Use sanitized_Request [streamcloud] Use sanitized_Request [streamcz] Use sanitized_Request [tapely] Use sanitized_Request [tube8] Use sanitized_Request [tubitv] Use sanitized_Request [twitch] Use sanitized_Request [twitter] Use sanitized_Request [udemy] Use sanitized_Request [vbox7] Use sanitized_Request [veoh] Use sanitized_Request [vessel] Use sanitized_Request [vevo] Use sanitized_Request [viddler] Use sanitized_Request [videomega] Use sanitized_Request [viewvster] Use sanitized_Request [viki] Use sanitized_Request [vk] Use sanitized_Request [vodlocker] Use sanitized_Request [voicerepublic] Use sanitized_Request [wistia] Use sanitized_Request [xfileshare] Use sanitized_Request [xtube] Use sanitized_Request [xvideos] Use sanitized_Request [yandexmusic] Use sanitized_Request [youku] Use sanitized_Request [youporn] Use sanitized_Request [youtube] Use sanitized_Request [patreon] Use sanitized_Request [extractor/common] Remove unused import [nfb] PEP 8
9 years ago
10 years ago
  1. from __future__ import unicode_literals
  2. import errno
  3. import os
  4. import socket
  5. import time
  6. import random
  7. import re
  8. from .common import FileDownloader
  9. from ..compat import (
  10. compat_str,
  11. compat_urllib_error,
  12. )
  13. from ..utils import (
  14. ContentTooShortError,
  15. encodeFilename,
  16. int_or_none,
  17. sanitize_open,
  18. sanitized_Request,
  19. write_xattr,
  20. XAttrMetadataError,
  21. XAttrUnavailableError,
  22. )
  23. class HttpFD(FileDownloader):
  24. def real_download(self, filename, info_dict):
  25. url = info_dict['url']
  26. class DownloadContext(dict):
  27. __getattr__ = dict.get
  28. __setattr__ = dict.__setitem__
  29. __delattr__ = dict.__delitem__
  30. ctx = DownloadContext()
  31. ctx.filename = filename
  32. ctx.tmpfilename = self.temp_name(filename)
  33. ctx.stream = None
  34. # Do not include the Accept-Encoding header
  35. headers = {'Youtubedl-no-compression': 'True'}
  36. add_headers = info_dict.get('http_headers')
  37. if add_headers:
  38. headers.update(add_headers)
  39. is_test = self.params.get('test', False)
  40. chunk_size = self._TEST_FILE_SIZE if is_test else (
  41. info_dict.get('downloader_options', {}).get('http_chunk_size') or
  42. self.params.get('http_chunk_size') or 0)
  43. ctx.open_mode = 'wb'
  44. ctx.resume_len = 0
  45. ctx.data_len = None
  46. ctx.block_size = self.params.get('buffersize', 1024)
  47. ctx.start_time = time.time()
  48. ctx.chunk_size = None
  49. if self.params.get('continuedl', True):
  50. # Establish possible resume length
  51. if os.path.isfile(encodeFilename(ctx.tmpfilename)):
  52. ctx.resume_len = os.path.getsize(
  53. encodeFilename(ctx.tmpfilename))
  54. ctx.is_resume = ctx.resume_len > 0
  55. count = 0
  56. retries = self.params.get('retries', 0)
  57. class SucceedDownload(Exception):
  58. pass
  59. class RetryDownload(Exception):
  60. def __init__(self, source_error):
  61. self.source_error = source_error
  62. class NextFragment(Exception):
  63. pass
  64. def set_range(req, start, end):
  65. range_header = 'bytes=%d-' % start
  66. if end:
  67. range_header += compat_str(end)
  68. req.add_header('Range', range_header)
  69. def establish_connection():
  70. ctx.chunk_size = (random.randint(int(chunk_size * 0.95), chunk_size)
  71. if not is_test and chunk_size else chunk_size)
  72. if ctx.resume_len > 0:
  73. range_start = ctx.resume_len
  74. if ctx.is_resume:
  75. self.report_resuming_byte(ctx.resume_len)
  76. ctx.open_mode = 'ab'
  77. elif ctx.chunk_size > 0:
  78. range_start = 0
  79. else:
  80. range_start = None
  81. ctx.is_resume = False
  82. range_end = range_start + ctx.chunk_size - 1 if ctx.chunk_size else None
  83. if range_end and ctx.data_len is not None and range_end >= ctx.data_len:
  84. range_end = ctx.data_len - 1
  85. has_range = range_start is not None
  86. ctx.has_range = has_range
  87. request = sanitized_Request(url, None, headers)
  88. if has_range:
  89. set_range(request, range_start, range_end)
  90. # Establish connection
  91. try:
  92. ctx.data = self.ydl.urlopen(request)
  93. # When trying to resume, Content-Range HTTP header of response has to be checked
  94. # to match the value of requested Range HTTP header. This is due to a webservers
  95. # that don't support resuming and serve a whole file with no Content-Range
  96. # set in response despite of requested Range (see
  97. # https://github.com/rg3/youtube-dl/issues/6057#issuecomment-126129799)
  98. if has_range:
  99. content_range = ctx.data.headers.get('Content-Range')
  100. if content_range:
  101. content_range_m = re.search(r'bytes (\d+)-(\d+)?(?:/(\d+))?', content_range)
  102. # Content-Range is present and matches requested Range, resume is possible
  103. if content_range_m:
  104. if range_start == int(content_range_m.group(1)):
  105. content_range_end = int_or_none(content_range_m.group(2))
  106. content_len = int_or_none(content_range_m.group(3))
  107. accept_content_len = (
  108. # Non-chunked download
  109. not ctx.chunk_size or
  110. # Chunked download and requested piece or
  111. # its part is promised to be served
  112. content_range_end == range_end or
  113. content_len < range_end)
  114. if accept_content_len:
  115. ctx.data_len = content_len
  116. return
  117. # Content-Range is either not present or invalid. Assuming remote webserver is
  118. # trying to send the whole file, resume is not possible, so wiping the local file
  119. # and performing entire redownload
  120. self.report_unable_to_resume()
  121. ctx.resume_len = 0
  122. ctx.open_mode = 'wb'
  123. ctx.data_len = int_or_none(ctx.data.info().get('Content-length', None))
  124. return
  125. except (compat_urllib_error.HTTPError, ) as err:
  126. if err.code == 416:
  127. # Unable to resume (requested range not satisfiable)
  128. try:
  129. # Open the connection again without the range header
  130. ctx.data = self.ydl.urlopen(
  131. sanitized_Request(url, None, headers))
  132. content_length = ctx.data.info()['Content-Length']
  133. except (compat_urllib_error.HTTPError, ) as err:
  134. if err.code < 500 or err.code >= 600:
  135. raise
  136. else:
  137. # Examine the reported length
  138. if (content_length is not None and
  139. (ctx.resume_len - 100 < int(content_length) < ctx.resume_len + 100)):
  140. # The file had already been fully downloaded.
  141. # Explanation to the above condition: in issue #175 it was revealed that
  142. # YouTube sometimes adds or removes a few bytes from the end of the file,
  143. # changing the file size slightly and causing problems for some users. So
  144. # I decided to implement a suggested change and consider the file
  145. # completely downloaded if the file size differs less than 100 bytes from
  146. # the one in the hard drive.
  147. self.report_file_already_downloaded(ctx.filename)
  148. self.try_rename(ctx.tmpfilename, ctx.filename)
  149. self._hook_progress({
  150. 'filename': ctx.filename,
  151. 'status': 'finished',
  152. 'downloaded_bytes': ctx.resume_len,
  153. 'total_bytes': ctx.resume_len,
  154. })
  155. raise SucceedDownload()
  156. else:
  157. # The length does not match, we start the download over
  158. self.report_unable_to_resume()
  159. ctx.resume_len = 0
  160. ctx.open_mode = 'wb'
  161. return
  162. elif err.code < 500 or err.code >= 600:
  163. # Unexpected HTTP error
  164. raise
  165. raise RetryDownload(err)
  166. except socket.error as err:
  167. if err.errno != errno.ECONNRESET:
  168. # Connection reset is no problem, just retry
  169. raise
  170. raise RetryDownload(err)
  171. def download():
  172. data_len = ctx.data.info().get('Content-length', None)
  173. # Range HTTP header may be ignored/unsupported by a webserver
  174. # (e.g. extractor/scivee.py, extractor/bambuser.py).
  175. # However, for a test we still would like to download just a piece of a file.
  176. # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
  177. # block size when downloading a file.
  178. if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
  179. data_len = self._TEST_FILE_SIZE
  180. if data_len is not None:
  181. data_len = int(data_len) + ctx.resume_len
  182. min_data_len = self.params.get('min_filesize')
  183. max_data_len = self.params.get('max_filesize')
  184. if min_data_len is not None and data_len < min_data_len:
  185. self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
  186. return False
  187. if max_data_len is not None and data_len > max_data_len:
  188. self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
  189. return False
  190. byte_counter = 0 + ctx.resume_len
  191. block_size = ctx.block_size
  192. start = time.time()
  193. # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
  194. now = None # needed for slow_down() in the first loop run
  195. before = start # start measuring
  196. def retry(e):
  197. to_stdout = ctx.tmpfilename == '-'
  198. if not to_stdout:
  199. ctx.stream.close()
  200. ctx.stream = None
  201. ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
  202. raise RetryDownload(e)
  203. while True:
  204. try:
  205. # Download and write
  206. data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
  207. # socket.timeout is a subclass of socket.error but may not have
  208. # errno set
  209. except socket.timeout as e:
  210. retry(e)
  211. except socket.error as e:
  212. if e.errno not in (errno.ECONNRESET, errno.ETIMEDOUT):
  213. raise
  214. retry(e)
  215. byte_counter += len(data_block)
  216. # exit loop when download is finished
  217. if len(data_block) == 0:
  218. break
  219. # Open destination file just in time
  220. if ctx.stream is None:
  221. try:
  222. ctx.stream, ctx.tmpfilename = sanitize_open(
  223. ctx.tmpfilename, ctx.open_mode)
  224. assert ctx.stream is not None
  225. ctx.filename = self.undo_temp_name(ctx.tmpfilename)
  226. self.report_destination(ctx.filename)
  227. except (OSError, IOError) as err:
  228. self.report_error('unable to open for writing: %s' % str(err))
  229. return False
  230. if self.params.get('xattr_set_filesize', False) and data_len is not None:
  231. try:
  232. write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode('utf-8'))
  233. except (XAttrUnavailableError, XAttrMetadataError) as err:
  234. self.report_error('unable to set filesize xattr: %s' % str(err))
  235. try:
  236. ctx.stream.write(data_block)
  237. except (IOError, OSError) as err:
  238. self.to_stderr('\n')
  239. self.report_error('unable to write data: %s' % str(err))
  240. return False
  241. # Apply rate limit
  242. self.slow_down(start, now, byte_counter - ctx.resume_len)
  243. # end measuring of one loop run
  244. now = time.time()
  245. after = now
  246. # Adjust block size
  247. if not self.params.get('noresizebuffer', False):
  248. block_size = self.best_block_size(after - before, len(data_block))
  249. before = after
  250. # Progress message
  251. speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
  252. if ctx.data_len is None:
  253. eta = None
  254. else:
  255. eta = self.calc_eta(start, time.time(), ctx.data_len - ctx.resume_len, byte_counter - ctx.resume_len)
  256. self._hook_progress({
  257. 'status': 'downloading',
  258. 'downloaded_bytes': byte_counter,
  259. 'total_bytes': ctx.data_len,
  260. 'tmpfilename': ctx.tmpfilename,
  261. 'filename': ctx.filename,
  262. 'eta': eta,
  263. 'speed': speed,
  264. 'elapsed': now - ctx.start_time,
  265. })
  266. if is_test and byte_counter == data_len:
  267. break
  268. if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len:
  269. ctx.resume_len = byte_counter
  270. # ctx.block_size = block_size
  271. raise NextFragment()
  272. if ctx.stream is None:
  273. self.to_stderr('\n')
  274. self.report_error('Did not get any data blocks')
  275. return False
  276. if ctx.tmpfilename != '-':
  277. ctx.stream.close()
  278. if data_len is not None and byte_counter != data_len:
  279. err = ContentTooShortError(byte_counter, int(data_len))
  280. if count <= retries:
  281. retry(err)
  282. raise err
  283. self.try_rename(ctx.tmpfilename, ctx.filename)
  284. # Update file modification time
  285. if self.params.get('updatetime', True):
  286. info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
  287. self._hook_progress({
  288. 'downloaded_bytes': byte_counter,
  289. 'total_bytes': byte_counter,
  290. 'filename': ctx.filename,
  291. 'status': 'finished',
  292. 'elapsed': time.time() - ctx.start_time,
  293. })
  294. return True
  295. while count <= retries:
  296. try:
  297. establish_connection()
  298. return download()
  299. except RetryDownload as e:
  300. count += 1
  301. if count <= retries:
  302. self.report_retry(e.source_error, count, retries)
  303. continue
  304. except NextFragment:
  305. continue
  306. except SucceedDownload:
  307. return True
  308. self.report_error('giving up after %s retries' % retries)
  309. return False