You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

581 lines
19 KiB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
  1. from __future__ import unicode_literals
  2. import binascii
  3. import collections
  4. import email
  5. import getpass
  6. import io
  7. import optparse
  8. import os
  9. import re
  10. import shlex
  11. import shutil
  12. import socket
  13. import subprocess
  14. import sys
  15. import itertools
  16. import xml.etree.ElementTree
  17. try:
  18. import urllib.request as compat_urllib_request
  19. except ImportError: # Python 2
  20. import urllib2 as compat_urllib_request
  21. try:
  22. import urllib.error as compat_urllib_error
  23. except ImportError: # Python 2
  24. import urllib2 as compat_urllib_error
  25. try:
  26. import urllib.parse as compat_urllib_parse
  27. except ImportError: # Python 2
  28. import urllib as compat_urllib_parse
  29. try:
  30. from urllib.parse import urlparse as compat_urllib_parse_urlparse
  31. except ImportError: # Python 2
  32. from urlparse import urlparse as compat_urllib_parse_urlparse
  33. try:
  34. import urllib.parse as compat_urlparse
  35. except ImportError: # Python 2
  36. import urlparse as compat_urlparse
  37. try:
  38. import urllib.response as compat_urllib_response
  39. except ImportError: # Python 2
  40. import urllib as compat_urllib_response
  41. try:
  42. import http.cookiejar as compat_cookiejar
  43. except ImportError: # Python 2
  44. import cookielib as compat_cookiejar
  45. try:
  46. import http.cookies as compat_cookies
  47. except ImportError: # Python 2
  48. import Cookie as compat_cookies
  49. try:
  50. import html.entities as compat_html_entities
  51. except ImportError: # Python 2
  52. import htmlentitydefs as compat_html_entities
  53. try:
  54. import http.client as compat_http_client
  55. except ImportError: # Python 2
  56. import httplib as compat_http_client
  57. try:
  58. from urllib.error import HTTPError as compat_HTTPError
  59. except ImportError: # Python 2
  60. from urllib2 import HTTPError as compat_HTTPError
  61. try:
  62. from urllib.request import urlretrieve as compat_urlretrieve
  63. except ImportError: # Python 2
  64. from urllib import urlretrieve as compat_urlretrieve
  65. try:
  66. from subprocess import DEVNULL
  67. compat_subprocess_get_DEVNULL = lambda: DEVNULL
  68. except ImportError:
  69. compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
  70. try:
  71. import http.server as compat_http_server
  72. except ImportError:
  73. import BaseHTTPServer as compat_http_server
  74. try:
  75. compat_str = unicode # Python 2
  76. except NameError:
  77. compat_str = str
  78. try:
  79. from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
  80. from urllib.parse import unquote as compat_urllib_parse_unquote
  81. from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
  82. except ImportError: # Python 2
  83. _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
  84. else re.compile('([\x00-\x7f]+)'))
  85. # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
  86. # implementations from cpython 3.4.3's stdlib. Python 2's version
  87. # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
  88. def compat_urllib_parse_unquote_to_bytes(string):
  89. """unquote_to_bytes('abc%20def') -> b'abc def'."""
  90. # Note: strings are encoded as UTF-8. This is only an issue if it contains
  91. # unescaped non-ASCII characters, which URIs should not.
  92. if not string:
  93. # Is it a string-like object?
  94. string.split
  95. return b''
  96. if isinstance(string, compat_str):
  97. string = string.encode('utf-8')
  98. bits = string.split(b'%')
  99. if len(bits) == 1:
  100. return string
  101. res = [bits[0]]
  102. append = res.append
  103. for item in bits[1:]:
  104. try:
  105. append(compat_urllib_parse._hextochr[item[:2]])
  106. append(item[2:])
  107. except KeyError:
  108. append(b'%')
  109. append(item)
  110. return b''.join(res)
  111. def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
  112. """Replace %xx escapes by their single-character equivalent. The optional
  113. encoding and errors parameters specify how to decode percent-encoded
  114. sequences into Unicode characters, as accepted by the bytes.decode()
  115. method.
  116. By default, percent-encoded sequences are decoded with UTF-8, and invalid
  117. sequences are replaced by a placeholder character.
  118. unquote('abc%20def') -> 'abc def'.
  119. """
  120. if '%' not in string:
  121. string.split
  122. return string
  123. if encoding is None:
  124. encoding = 'utf-8'
  125. if errors is None:
  126. errors = 'replace'
  127. bits = _asciire.split(string)
  128. res = [bits[0]]
  129. append = res.append
  130. for i in range(1, len(bits), 2):
  131. append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors))
  132. append(bits[i + 1])
  133. return ''.join(res)
  134. def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'):
  135. """Like unquote(), but also replace plus signs by spaces, as required for
  136. unquoting HTML form values.
  137. unquote_plus('%7e/abc+def') -> '~/abc def'
  138. """
  139. string = string.replace('+', ' ')
  140. return compat_urllib_parse_unquote(string, encoding, errors)
  141. try:
  142. from urllib.request import DataHandler as compat_urllib_request_DataHandler
  143. except ImportError: # Python < 3.4
  144. # Ported from CPython 98774:1733b3bd46db, Lib/urllib/request.py
  145. class compat_urllib_request_DataHandler(compat_urllib_request.BaseHandler):
  146. def data_open(self, req):
  147. # data URLs as specified in RFC 2397.
  148. #
  149. # ignores POSTed data
  150. #
  151. # syntax:
  152. # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
  153. # mediatype := [ type "/" subtype ] *( ";" parameter )
  154. # data := *urlchar
  155. # parameter := attribute "=" value
  156. url = req.get_full_url()
  157. scheme, data = url.split(":", 1)
  158. mediatype, data = data.split(",", 1)
  159. # even base64 encoded data URLs might be quoted so unquote in any case:
  160. data = compat_urllib_parse_unquote_to_bytes(data)
  161. if mediatype.endswith(";base64"):
  162. data = binascii.a2b_base64(data)
  163. mediatype = mediatype[:-7]
  164. if not mediatype:
  165. mediatype = "text/plain;charset=US-ASCII"
  166. headers = email.message_from_string(
  167. "Content-type: %s\nContent-length: %d\n" % (mediatype, len(data)))
  168. return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
  169. try:
  170. compat_basestring = basestring # Python 2
  171. except NameError:
  172. compat_basestring = str
  173. try:
  174. compat_chr = unichr # Python 2
  175. except NameError:
  176. compat_chr = chr
  177. try:
  178. from xml.etree.ElementTree import ParseError as compat_xml_parse_error
  179. except ImportError: # Python 2.6
  180. from xml.parsers.expat import ExpatError as compat_xml_parse_error
  181. if sys.version_info[0] >= 3:
  182. compat_etree_fromstring = xml.etree.ElementTree.fromstring
  183. else:
  184. # python 2.x tries to encode unicode strings with ascii (see the
  185. # XMLParser._fixtext method)
  186. etree = xml.etree.ElementTree
  187. try:
  188. _etree_iter = etree.Element.iter
  189. except AttributeError: # Python <=2.6
  190. def _etree_iter(root):
  191. for el in root.findall('*'):
  192. yield el
  193. for sub in _etree_iter(el):
  194. yield sub
  195. # on 2.6 XML doesn't have a parser argument, function copied from CPython
  196. # 2.7 source
  197. def _XML(text, parser=None):
  198. if not parser:
  199. parser = etree.XMLParser(target=etree.TreeBuilder())
  200. parser.feed(text)
  201. return parser.close()
  202. def _element_factory(*args, **kwargs):
  203. el = etree.Element(*args, **kwargs)
  204. for k, v in el.items():
  205. if isinstance(v, bytes):
  206. el.set(k, v.decode('utf-8'))
  207. return el
  208. def compat_etree_fromstring(text):
  209. doc = _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
  210. for el in _etree_iter(doc):
  211. if el.text is not None and isinstance(el.text, bytes):
  212. el.text = el.text.decode('utf-8')
  213. return doc
  214. try:
  215. from urllib.parse import parse_qs as compat_parse_qs
  216. except ImportError: # Python 2
  217. # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
  218. # Python 2's version is apparently totally broken
  219. def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
  220. encoding='utf-8', errors='replace'):
  221. qs, _coerce_result = qs, compat_str
  222. pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
  223. r = []
  224. for name_value in pairs:
  225. if not name_value and not strict_parsing:
  226. continue
  227. nv = name_value.split('=', 1)
  228. if len(nv) != 2:
  229. if strict_parsing:
  230. raise ValueError("bad query field: %r" % (name_value,))
  231. # Handle case of a control-name with no equal sign
  232. if keep_blank_values:
  233. nv.append('')
  234. else:
  235. continue
  236. if len(nv[1]) or keep_blank_values:
  237. name = nv[0].replace('+', ' ')
  238. name = compat_urllib_parse_unquote(
  239. name, encoding=encoding, errors=errors)
  240. name = _coerce_result(name)
  241. value = nv[1].replace('+', ' ')
  242. value = compat_urllib_parse_unquote(
  243. value, encoding=encoding, errors=errors)
  244. value = _coerce_result(value)
  245. r.append((name, value))
  246. return r
  247. def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
  248. encoding='utf-8', errors='replace'):
  249. parsed_result = {}
  250. pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
  251. encoding=encoding, errors=errors)
  252. for name, value in pairs:
  253. if name in parsed_result:
  254. parsed_result[name].append(value)
  255. else:
  256. parsed_result[name] = [value]
  257. return parsed_result
  258. try:
  259. from shlex import quote as shlex_quote
  260. except ImportError: # Python < 3.3
  261. def shlex_quote(s):
  262. if re.match(r'^[-_\w./]+$', s):
  263. return s
  264. else:
  265. return "'" + s.replace("'", "'\"'\"'") + "'"
  266. if sys.version_info >= (2, 7, 3):
  267. compat_shlex_split = shlex.split
  268. else:
  269. # Working around shlex issue with unicode strings on some python 2
  270. # versions (see http://bugs.python.org/issue1548891)
  271. def compat_shlex_split(s, comments=False, posix=True):
  272. if isinstance(s, compat_str):
  273. s = s.encode('utf-8')
  274. return shlex.split(s, comments, posix)
  275. def compat_ord(c):
  276. if type(c) is int:
  277. return c
  278. else:
  279. return ord(c)
  280. if sys.version_info >= (3, 0):
  281. compat_getenv = os.getenv
  282. compat_expanduser = os.path.expanduser
  283. else:
  284. # Environment variables should be decoded with filesystem encoding.
  285. # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
  286. def compat_getenv(key, default=None):
  287. from .utils import get_filesystem_encoding
  288. env = os.getenv(key, default)
  289. if env:
  290. env = env.decode(get_filesystem_encoding())
  291. return env
  292. # HACK: The default implementations of os.path.expanduser from cpython do not decode
  293. # environment variables with filesystem encoding. We will work around this by
  294. # providing adjusted implementations.
  295. # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
  296. # for different platforms with correct environment variables decoding.
  297. if os.name == 'posix':
  298. def compat_expanduser(path):
  299. """Expand ~ and ~user constructions. If user or $HOME is unknown,
  300. do nothing."""
  301. if not path.startswith('~'):
  302. return path
  303. i = path.find('/', 1)
  304. if i < 0:
  305. i = len(path)
  306. if i == 1:
  307. if 'HOME' not in os.environ:
  308. import pwd
  309. userhome = pwd.getpwuid(os.getuid()).pw_dir
  310. else:
  311. userhome = compat_getenv('HOME')
  312. else:
  313. import pwd
  314. try:
  315. pwent = pwd.getpwnam(path[1:i])
  316. except KeyError:
  317. return path
  318. userhome = pwent.pw_dir
  319. userhome = userhome.rstrip('/')
  320. return (userhome + path[i:]) or '/'
  321. elif os.name == 'nt' or os.name == 'ce':
  322. def compat_expanduser(path):
  323. """Expand ~ and ~user constructs.
  324. If user or $HOME is unknown, do nothing."""
  325. if path[:1] != '~':
  326. return path
  327. i, n = 1, len(path)
  328. while i < n and path[i] not in '/\\':
  329. i = i + 1
  330. if 'HOME' in os.environ:
  331. userhome = compat_getenv('HOME')
  332. elif 'USERPROFILE' in os.environ:
  333. userhome = compat_getenv('USERPROFILE')
  334. elif 'HOMEPATH' not in os.environ:
  335. return path
  336. else:
  337. try:
  338. drive = compat_getenv('HOMEDRIVE')
  339. except KeyError:
  340. drive = ''
  341. userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
  342. if i != 1: # ~user
  343. userhome = os.path.join(os.path.dirname(userhome), path[1:i])
  344. return userhome + path[i:]
  345. else:
  346. compat_expanduser = os.path.expanduser
  347. if sys.version_info < (3, 0):
  348. def compat_print(s):
  349. from .utils import preferredencoding
  350. print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
  351. else:
  352. def compat_print(s):
  353. assert isinstance(s, compat_str)
  354. print(s)
  355. try:
  356. subprocess_check_output = subprocess.check_output
  357. except AttributeError:
  358. def subprocess_check_output(*args, **kwargs):
  359. assert 'input' not in kwargs
  360. p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
  361. output, _ = p.communicate()
  362. ret = p.poll()
  363. if ret:
  364. raise subprocess.CalledProcessError(ret, p.args, output=output)
  365. return output
  366. if sys.version_info < (3, 0) and sys.platform == 'win32':
  367. def compat_getpass(prompt, *args, **kwargs):
  368. if isinstance(prompt, compat_str):
  369. from .utils import preferredencoding
  370. prompt = prompt.encode(preferredencoding())
  371. return getpass.getpass(prompt, *args, **kwargs)
  372. else:
  373. compat_getpass = getpass.getpass
  374. # Old 2.6 and 2.7 releases require kwargs to be bytes
  375. try:
  376. def _testfunc(x):
  377. pass
  378. _testfunc(**{'x': 0})
  379. except TypeError:
  380. def compat_kwargs(kwargs):
  381. return dict((bytes(k), v) for k, v in kwargs.items())
  382. else:
  383. compat_kwargs = lambda kwargs: kwargs
  384. if sys.version_info < (2, 7):
  385. def compat_socket_create_connection(address, timeout, source_address=None):
  386. host, port = address
  387. err = None
  388. for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
  389. af, socktype, proto, canonname, sa = res
  390. sock = None
  391. try:
  392. sock = socket.socket(af, socktype, proto)
  393. sock.settimeout(timeout)
  394. if source_address:
  395. sock.bind(source_address)
  396. sock.connect(sa)
  397. return sock
  398. except socket.error as _:
  399. err = _
  400. if sock is not None:
  401. sock.close()
  402. if err is not None:
  403. raise err
  404. else:
  405. raise socket.error("getaddrinfo returns an empty list")
  406. else:
  407. compat_socket_create_connection = socket.create_connection
  408. # Fix https://github.com/rg3/youtube-dl/issues/4223
  409. # See http://bugs.python.org/issue9161 for what is broken
  410. def workaround_optparse_bug9161():
  411. op = optparse.OptionParser()
  412. og = optparse.OptionGroup(op, 'foo')
  413. try:
  414. og.add_option('-t')
  415. except TypeError:
  416. real_add_option = optparse.OptionGroup.add_option
  417. def _compat_add_option(self, *args, **kwargs):
  418. enc = lambda v: (
  419. v.encode('ascii', 'replace') if isinstance(v, compat_str)
  420. else v)
  421. bargs = [enc(a) for a in args]
  422. bkwargs = dict(
  423. (k, enc(v)) for k, v in kwargs.items())
  424. return real_add_option(self, *bargs, **bkwargs)
  425. optparse.OptionGroup.add_option = _compat_add_option
  426. if hasattr(shutil, 'get_terminal_size'): # Python >= 3.3
  427. compat_get_terminal_size = shutil.get_terminal_size
  428. else:
  429. _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
  430. def compat_get_terminal_size(fallback=(80, 24)):
  431. columns = compat_getenv('COLUMNS')
  432. if columns:
  433. columns = int(columns)
  434. else:
  435. columns = None
  436. lines = compat_getenv('LINES')
  437. if lines:
  438. lines = int(lines)
  439. else:
  440. lines = None
  441. if columns is None or lines is None or columns <= 0 or lines <= 0:
  442. try:
  443. sp = subprocess.Popen(
  444. ['stty', 'size'],
  445. stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  446. out, err = sp.communicate()
  447. _lines, _columns = map(int, out.split())
  448. except Exception:
  449. _columns, _lines = _terminal_size(*fallback)
  450. if columns is None or columns <= 0:
  451. columns = _columns
  452. if lines is None or lines <= 0:
  453. lines = _lines
  454. return _terminal_size(columns, lines)
  455. try:
  456. itertools.count(start=0, step=1)
  457. compat_itertools_count = itertools.count
  458. except TypeError: # Python 2.6
  459. def compat_itertools_count(start=0, step=1):
  460. n = start
  461. while True:
  462. yield n
  463. n += step
  464. if sys.version_info >= (3, 0):
  465. from tokenize import tokenize as compat_tokenize_tokenize
  466. else:
  467. from tokenize import generate_tokens as compat_tokenize_tokenize
  468. __all__ = [
  469. 'compat_HTTPError',
  470. 'compat_basestring',
  471. 'compat_chr',
  472. 'compat_cookiejar',
  473. 'compat_cookies',
  474. 'compat_etree_fromstring',
  475. 'compat_expanduser',
  476. 'compat_get_terminal_size',
  477. 'compat_getenv',
  478. 'compat_getpass',
  479. 'compat_html_entities',
  480. 'compat_http_client',
  481. 'compat_http_server',
  482. 'compat_itertools_count',
  483. 'compat_kwargs',
  484. 'compat_ord',
  485. 'compat_parse_qs',
  486. 'compat_print',
  487. 'compat_shlex_split',
  488. 'compat_socket_create_connection',
  489. 'compat_str',
  490. 'compat_subprocess_get_DEVNULL',
  491. 'compat_tokenize_tokenize',
  492. 'compat_urllib_error',
  493. 'compat_urllib_parse',
  494. 'compat_urllib_parse_unquote',
  495. 'compat_urllib_parse_unquote_plus',
  496. 'compat_urllib_parse_unquote_to_bytes',
  497. 'compat_urllib_parse_urlparse',
  498. 'compat_urllib_request',
  499. 'compat_urllib_request_DataHandler',
  500. 'compat_urllib_response',
  501. 'compat_urlparse',
  502. 'compat_urlretrieve',
  503. 'compat_xml_parse_error',
  504. 'shlex_quote',
  505. 'subprocess_check_output',
  506. 'workaround_optparse_bug9161',
  507. ]