You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

591 lines
19 KiB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
9 years ago
10 years ago
10 years ago
10 years ago
  1. from __future__ import unicode_literals
  2. import binascii
  3. import collections
  4. import email
  5. import getpass
  6. import io
  7. import optparse
  8. import os
  9. import re
  10. import shlex
  11. import shutil
  12. import socket
  13. import subprocess
  14. import sys
  15. import itertools
  16. import xml.etree.ElementTree
  17. try:
  18. import urllib.request as compat_urllib_request
  19. except ImportError: # Python 2
  20. import urllib2 as compat_urllib_request
  21. try:
  22. import urllib.error as compat_urllib_error
  23. except ImportError: # Python 2
  24. import urllib2 as compat_urllib_error
  25. try:
  26. import urllib.parse as compat_urllib_parse
  27. except ImportError: # Python 2
  28. import urllib as compat_urllib_parse
  29. try:
  30. from urllib.parse import urlparse as compat_urllib_parse_urlparse
  31. except ImportError: # Python 2
  32. from urlparse import urlparse as compat_urllib_parse_urlparse
  33. try:
  34. import urllib.parse as compat_urlparse
  35. except ImportError: # Python 2
  36. import urlparse as compat_urlparse
  37. try:
  38. import urllib.response as compat_urllib_response
  39. except ImportError: # Python 2
  40. import urllib as compat_urllib_response
  41. try:
  42. import http.cookiejar as compat_cookiejar
  43. except ImportError: # Python 2
  44. import cookielib as compat_cookiejar
  45. try:
  46. import http.cookies as compat_cookies
  47. except ImportError: # Python 2
  48. import Cookie as compat_cookies
  49. try:
  50. import html.entities as compat_html_entities
  51. except ImportError: # Python 2
  52. import htmlentitydefs as compat_html_entities
  53. try:
  54. import http.client as compat_http_client
  55. except ImportError: # Python 2
  56. import httplib as compat_http_client
  57. try:
  58. from urllib.error import HTTPError as compat_HTTPError
  59. except ImportError: # Python 2
  60. from urllib2 import HTTPError as compat_HTTPError
  61. try:
  62. from urllib.request import urlretrieve as compat_urlretrieve
  63. except ImportError: # Python 2
  64. from urllib import urlretrieve as compat_urlretrieve
  65. try:
  66. from html.parser import HTMLParser as compat_HTMLParser
  67. except ImportError: # Python 2
  68. from HTMLParser import HTMLParser as compat_HTMLParser
  69. try:
  70. from subprocess import DEVNULL
  71. compat_subprocess_get_DEVNULL = lambda: DEVNULL
  72. except ImportError:
  73. compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
  74. try:
  75. import http.server as compat_http_server
  76. except ImportError:
  77. import BaseHTTPServer as compat_http_server
  78. try:
  79. compat_str = unicode # Python 2
  80. except NameError:
  81. compat_str = str
  82. try:
  83. from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
  84. from urllib.parse import unquote as compat_urllib_parse_unquote
  85. from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
  86. except ImportError: # Python 2
  87. _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
  88. else re.compile('([\x00-\x7f]+)'))
  89. # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
  90. # implementations from cpython 3.4.3's stdlib. Python 2's version
  91. # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
  92. def compat_urllib_parse_unquote_to_bytes(string):
  93. """unquote_to_bytes('abc%20def') -> b'abc def'."""
  94. # Note: strings are encoded as UTF-8. This is only an issue if it contains
  95. # unescaped non-ASCII characters, which URIs should not.
  96. if not string:
  97. # Is it a string-like object?
  98. string.split
  99. return b''
  100. if isinstance(string, compat_str):
  101. string = string.encode('utf-8')
  102. bits = string.split(b'%')
  103. if len(bits) == 1:
  104. return string
  105. res = [bits[0]]
  106. append = res.append
  107. for item in bits[1:]:
  108. try:
  109. append(compat_urllib_parse._hextochr[item[:2]])
  110. append(item[2:])
  111. except KeyError:
  112. append(b'%')
  113. append(item)
  114. return b''.join(res)
  115. def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
  116. """Replace %xx escapes by their single-character equivalent. The optional
  117. encoding and errors parameters specify how to decode percent-encoded
  118. sequences into Unicode characters, as accepted by the bytes.decode()
  119. method.
  120. By default, percent-encoded sequences are decoded with UTF-8, and invalid
  121. sequences are replaced by a placeholder character.
  122. unquote('abc%20def') -> 'abc def'.
  123. """
  124. if '%' not in string:
  125. string.split
  126. return string
  127. if encoding is None:
  128. encoding = 'utf-8'
  129. if errors is None:
  130. errors = 'replace'
  131. bits = _asciire.split(string)
  132. res = [bits[0]]
  133. append = res.append
  134. for i in range(1, len(bits), 2):
  135. append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors))
  136. append(bits[i + 1])
  137. return ''.join(res)
  138. def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'):
  139. """Like unquote(), but also replace plus signs by spaces, as required for
  140. unquoting HTML form values.
  141. unquote_plus('%7e/abc+def') -> '~/abc def'
  142. """
  143. string = string.replace('+', ' ')
  144. return compat_urllib_parse_unquote(string, encoding, errors)
  145. try:
  146. from urllib.request import DataHandler as compat_urllib_request_DataHandler
  147. except ImportError: # Python < 3.4
  148. # Ported from CPython 98774:1733b3bd46db, Lib/urllib/request.py
  149. class compat_urllib_request_DataHandler(compat_urllib_request.BaseHandler):
  150. def data_open(self, req):
  151. # data URLs as specified in RFC 2397.
  152. #
  153. # ignores POSTed data
  154. #
  155. # syntax:
  156. # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
  157. # mediatype := [ type "/" subtype ] *( ";" parameter )
  158. # data := *urlchar
  159. # parameter := attribute "=" value
  160. url = req.get_full_url()
  161. scheme, data = url.split(':', 1)
  162. mediatype, data = data.split(',', 1)
  163. # even base64 encoded data URLs might be quoted so unquote in any case:
  164. data = compat_urllib_parse_unquote_to_bytes(data)
  165. if mediatype.endswith(';base64'):
  166. data = binascii.a2b_base64(data)
  167. mediatype = mediatype[:-7]
  168. if not mediatype:
  169. mediatype = 'text/plain;charset=US-ASCII'
  170. headers = email.message_from_string(
  171. 'Content-type: %s\nContent-length: %d\n' % (mediatype, len(data)))
  172. return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
  173. try:
  174. compat_basestring = basestring # Python 2
  175. except NameError:
  176. compat_basestring = str
  177. try:
  178. compat_chr = unichr # Python 2
  179. except NameError:
  180. compat_chr = chr
  181. try:
  182. from xml.etree.ElementTree import ParseError as compat_xml_parse_error
  183. except ImportError: # Python 2.6
  184. from xml.parsers.expat import ExpatError as compat_xml_parse_error
  185. if sys.version_info[0] >= 3:
  186. compat_etree_fromstring = xml.etree.ElementTree.fromstring
  187. else:
  188. # python 2.x tries to encode unicode strings with ascii (see the
  189. # XMLParser._fixtext method)
  190. etree = xml.etree.ElementTree
  191. try:
  192. _etree_iter = etree.Element.iter
  193. except AttributeError: # Python <=2.6
  194. def _etree_iter(root):
  195. for el in root.findall('*'):
  196. yield el
  197. for sub in _etree_iter(el):
  198. yield sub
  199. # on 2.6 XML doesn't have a parser argument, function copied from CPython
  200. # 2.7 source
  201. def _XML(text, parser=None):
  202. if not parser:
  203. parser = etree.XMLParser(target=etree.TreeBuilder())
  204. parser.feed(text)
  205. return parser.close()
  206. def _element_factory(*args, **kwargs):
  207. el = etree.Element(*args, **kwargs)
  208. for k, v in el.items():
  209. if isinstance(v, bytes):
  210. el.set(k, v.decode('utf-8'))
  211. return el
  212. def compat_etree_fromstring(text):
  213. doc = _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
  214. for el in _etree_iter(doc):
  215. if el.text is not None and isinstance(el.text, bytes):
  216. el.text = el.text.decode('utf-8')
  217. return doc
  218. try:
  219. from urllib.parse import parse_qs as compat_parse_qs
  220. except ImportError: # Python 2
  221. # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
  222. # Python 2's version is apparently totally broken
  223. def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
  224. encoding='utf-8', errors='replace'):
  225. qs, _coerce_result = qs, compat_str
  226. pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
  227. r = []
  228. for name_value in pairs:
  229. if not name_value and not strict_parsing:
  230. continue
  231. nv = name_value.split('=', 1)
  232. if len(nv) != 2:
  233. if strict_parsing:
  234. raise ValueError('bad query field: %r' % (name_value,))
  235. # Handle case of a control-name with no equal sign
  236. if keep_blank_values:
  237. nv.append('')
  238. else:
  239. continue
  240. if len(nv[1]) or keep_blank_values:
  241. name = nv[0].replace('+', ' ')
  242. name = compat_urllib_parse_unquote(
  243. name, encoding=encoding, errors=errors)
  244. name = _coerce_result(name)
  245. value = nv[1].replace('+', ' ')
  246. value = compat_urllib_parse_unquote(
  247. value, encoding=encoding, errors=errors)
  248. value = _coerce_result(value)
  249. r.append((name, value))
  250. return r
  251. def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
  252. encoding='utf-8', errors='replace'):
  253. parsed_result = {}
  254. pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
  255. encoding=encoding, errors=errors)
  256. for name, value in pairs:
  257. if name in parsed_result:
  258. parsed_result[name].append(value)
  259. else:
  260. parsed_result[name] = [value]
  261. return parsed_result
  262. try:
  263. from shlex import quote as shlex_quote
  264. except ImportError: # Python < 3.3
  265. def shlex_quote(s):
  266. if re.match(r'^[-_\w./]+$', s):
  267. return s
  268. else:
  269. return "'" + s.replace("'", "'\"'\"'") + "'"
  270. if sys.version_info >= (2, 7, 3):
  271. compat_shlex_split = shlex.split
  272. else:
  273. # Working around shlex issue with unicode strings on some python 2
  274. # versions (see http://bugs.python.org/issue1548891)
  275. def compat_shlex_split(s, comments=False, posix=True):
  276. if isinstance(s, compat_str):
  277. s = s.encode('utf-8')
  278. return shlex.split(s, comments, posix)
  279. def compat_ord(c):
  280. if type(c) is int:
  281. return c
  282. else:
  283. return ord(c)
  284. compat_os_name = os._name if os.name == 'java' else os.name
  285. if sys.version_info >= (3, 0):
  286. compat_getenv = os.getenv
  287. compat_expanduser = os.path.expanduser
  288. else:
  289. # Environment variables should be decoded with filesystem encoding.
  290. # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
  291. def compat_getenv(key, default=None):
  292. from .utils import get_filesystem_encoding
  293. env = os.getenv(key, default)
  294. if env:
  295. env = env.decode(get_filesystem_encoding())
  296. return env
  297. # HACK: The default implementations of os.path.expanduser from cpython do not decode
  298. # environment variables with filesystem encoding. We will work around this by
  299. # providing adjusted implementations.
  300. # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
  301. # for different platforms with correct environment variables decoding.
  302. if compat_os_name == 'posix':
  303. def compat_expanduser(path):
  304. """Expand ~ and ~user constructions. If user or $HOME is unknown,
  305. do nothing."""
  306. if not path.startswith('~'):
  307. return path
  308. i = path.find('/', 1)
  309. if i < 0:
  310. i = len(path)
  311. if i == 1:
  312. if 'HOME' not in os.environ:
  313. import pwd
  314. userhome = pwd.getpwuid(os.getuid()).pw_dir
  315. else:
  316. userhome = compat_getenv('HOME')
  317. else:
  318. import pwd
  319. try:
  320. pwent = pwd.getpwnam(path[1:i])
  321. except KeyError:
  322. return path
  323. userhome = pwent.pw_dir
  324. userhome = userhome.rstrip('/')
  325. return (userhome + path[i:]) or '/'
  326. elif compat_os_name == 'nt' or compat_os_name == 'ce':
  327. def compat_expanduser(path):
  328. """Expand ~ and ~user constructs.
  329. If user or $HOME is unknown, do nothing."""
  330. if path[:1] != '~':
  331. return path
  332. i, n = 1, len(path)
  333. while i < n and path[i] not in '/\\':
  334. i = i + 1
  335. if 'HOME' in os.environ:
  336. userhome = compat_getenv('HOME')
  337. elif 'USERPROFILE' in os.environ:
  338. userhome = compat_getenv('USERPROFILE')
  339. elif 'HOMEPATH' not in os.environ:
  340. return path
  341. else:
  342. try:
  343. drive = compat_getenv('HOMEDRIVE')
  344. except KeyError:
  345. drive = ''
  346. userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
  347. if i != 1: # ~user
  348. userhome = os.path.join(os.path.dirname(userhome), path[1:i])
  349. return userhome + path[i:]
  350. else:
  351. compat_expanduser = os.path.expanduser
  352. if sys.version_info < (3, 0):
  353. def compat_print(s):
  354. from .utils import preferredencoding
  355. print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
  356. else:
  357. def compat_print(s):
  358. assert isinstance(s, compat_str)
  359. print(s)
  360. try:
  361. subprocess_check_output = subprocess.check_output
  362. except AttributeError:
  363. def subprocess_check_output(*args, **kwargs):
  364. assert 'input' not in kwargs
  365. p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
  366. output, _ = p.communicate()
  367. ret = p.poll()
  368. if ret:
  369. raise subprocess.CalledProcessError(ret, p.args, output=output)
  370. return output
  371. if sys.version_info < (3, 0) and sys.platform == 'win32':
  372. def compat_getpass(prompt, *args, **kwargs):
  373. if isinstance(prompt, compat_str):
  374. from .utils import preferredencoding
  375. prompt = prompt.encode(preferredencoding())
  376. return getpass.getpass(prompt, *args, **kwargs)
  377. else:
  378. compat_getpass = getpass.getpass
  379. # Python < 2.6.5 require kwargs to be bytes
  380. try:
  381. def _testfunc(x):
  382. pass
  383. _testfunc(**{'x': 0})
  384. except TypeError:
  385. def compat_kwargs(kwargs):
  386. return dict((bytes(k), v) for k, v in kwargs.items())
  387. else:
  388. compat_kwargs = lambda kwargs: kwargs
  389. if sys.version_info < (2, 7):
  390. def compat_socket_create_connection(address, timeout, source_address=None):
  391. host, port = address
  392. err = None
  393. for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
  394. af, socktype, proto, canonname, sa = res
  395. sock = None
  396. try:
  397. sock = socket.socket(af, socktype, proto)
  398. sock.settimeout(timeout)
  399. if source_address:
  400. sock.bind(source_address)
  401. sock.connect(sa)
  402. return sock
  403. except socket.error as _:
  404. err = _
  405. if sock is not None:
  406. sock.close()
  407. if err is not None:
  408. raise err
  409. else:
  410. raise socket.error('getaddrinfo returns an empty list')
  411. else:
  412. compat_socket_create_connection = socket.create_connection
  413. # Fix https://github.com/rg3/youtube-dl/issues/4223
  414. # See http://bugs.python.org/issue9161 for what is broken
  415. def workaround_optparse_bug9161():
  416. op = optparse.OptionParser()
  417. og = optparse.OptionGroup(op, 'foo')
  418. try:
  419. og.add_option('-t')
  420. except TypeError:
  421. real_add_option = optparse.OptionGroup.add_option
  422. def _compat_add_option(self, *args, **kwargs):
  423. enc = lambda v: (
  424. v.encode('ascii', 'replace') if isinstance(v, compat_str)
  425. else v)
  426. bargs = [enc(a) for a in args]
  427. bkwargs = dict(
  428. (k, enc(v)) for k, v in kwargs.items())
  429. return real_add_option(self, *bargs, **bkwargs)
  430. optparse.OptionGroup.add_option = _compat_add_option
  431. if hasattr(shutil, 'get_terminal_size'): # Python >= 3.3
  432. compat_get_terminal_size = shutil.get_terminal_size
  433. else:
  434. _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
  435. def compat_get_terminal_size(fallback=(80, 24)):
  436. columns = compat_getenv('COLUMNS')
  437. if columns:
  438. columns = int(columns)
  439. else:
  440. columns = None
  441. lines = compat_getenv('LINES')
  442. if lines:
  443. lines = int(lines)
  444. else:
  445. lines = None
  446. if columns is None or lines is None or columns <= 0 or lines <= 0:
  447. try:
  448. sp = subprocess.Popen(
  449. ['stty', 'size'],
  450. stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  451. out, err = sp.communicate()
  452. _lines, _columns = map(int, out.split())
  453. except Exception:
  454. _columns, _lines = _terminal_size(*fallback)
  455. if columns is None or columns <= 0:
  456. columns = _columns
  457. if lines is None or lines <= 0:
  458. lines = _lines
  459. return _terminal_size(columns, lines)
  460. try:
  461. itertools.count(start=0, step=1)
  462. compat_itertools_count = itertools.count
  463. except TypeError: # Python 2.6
  464. def compat_itertools_count(start=0, step=1):
  465. n = start
  466. while True:
  467. yield n
  468. n += step
  469. if sys.version_info >= (3, 0):
  470. from tokenize import tokenize as compat_tokenize_tokenize
  471. else:
  472. from tokenize import generate_tokens as compat_tokenize_tokenize
  473. __all__ = [
  474. 'compat_HTMLParser',
  475. 'compat_HTTPError',
  476. 'compat_basestring',
  477. 'compat_chr',
  478. 'compat_cookiejar',
  479. 'compat_cookies',
  480. 'compat_etree_fromstring',
  481. 'compat_expanduser',
  482. 'compat_get_terminal_size',
  483. 'compat_getenv',
  484. 'compat_getpass',
  485. 'compat_html_entities',
  486. 'compat_http_client',
  487. 'compat_http_server',
  488. 'compat_itertools_count',
  489. 'compat_kwargs',
  490. 'compat_ord',
  491. 'compat_os_name',
  492. 'compat_parse_qs',
  493. 'compat_print',
  494. 'compat_shlex_split',
  495. 'compat_socket_create_connection',
  496. 'compat_str',
  497. 'compat_subprocess_get_DEVNULL',
  498. 'compat_tokenize_tokenize',
  499. 'compat_urllib_error',
  500. 'compat_urllib_parse',
  501. 'compat_urllib_parse_unquote',
  502. 'compat_urllib_parse_unquote_plus',
  503. 'compat_urllib_parse_unquote_to_bytes',
  504. 'compat_urllib_parse_urlparse',
  505. 'compat_urllib_request',
  506. 'compat_urllib_request_DataHandler',
  507. 'compat_urllib_response',
  508. 'compat_urlparse',
  509. 'compat_urlretrieve',
  510. 'compat_xml_parse_error',
  511. 'shlex_quote',
  512. 'subprocess_check_output',
  513. 'workaround_optparse_bug9161',
  514. ]