You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

386 lines
12 KiB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
  1. from __future__ import unicode_literals
  2. import getpass
  3. import optparse
  4. import os
  5. import re
  6. import socket
  7. import subprocess
  8. import sys
  9. try:
  10. import urllib.request as compat_urllib_request
  11. except ImportError: # Python 2
  12. import urllib2 as compat_urllib_request
  13. try:
  14. import urllib.error as compat_urllib_error
  15. except ImportError: # Python 2
  16. import urllib2 as compat_urllib_error
  17. try:
  18. import urllib.parse as compat_urllib_parse
  19. except ImportError: # Python 2
  20. import urllib as compat_urllib_parse
  21. try:
  22. from urllib.parse import urlparse as compat_urllib_parse_urlparse
  23. except ImportError: # Python 2
  24. from urlparse import urlparse as compat_urllib_parse_urlparse
  25. try:
  26. import urllib.parse as compat_urlparse
  27. except ImportError: # Python 2
  28. import urlparse as compat_urlparse
  29. try:
  30. import http.cookiejar as compat_cookiejar
  31. except ImportError: # Python 2
  32. import cookielib as compat_cookiejar
  33. try:
  34. import html.entities as compat_html_entities
  35. except ImportError: # Python 2
  36. import htmlentitydefs as compat_html_entities
  37. try:
  38. import html.parser as compat_html_parser
  39. except ImportError: # Python 2
  40. import HTMLParser as compat_html_parser
  41. try:
  42. import http.client as compat_http_client
  43. except ImportError: # Python 2
  44. import httplib as compat_http_client
  45. try:
  46. from urllib.error import HTTPError as compat_HTTPError
  47. except ImportError: # Python 2
  48. from urllib2 import HTTPError as compat_HTTPError
  49. try:
  50. from urllib.request import urlretrieve as compat_urlretrieve
  51. except ImportError: # Python 2
  52. from urllib import urlretrieve as compat_urlretrieve
  53. try:
  54. from subprocess import DEVNULL
  55. compat_subprocess_get_DEVNULL = lambda: DEVNULL
  56. except ImportError:
  57. compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
  58. try:
  59. from urllib.parse import unquote as compat_urllib_parse_unquote
  60. except ImportError:
  61. def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
  62. if string == '':
  63. return string
  64. res = string.split('%')
  65. if len(res) == 1:
  66. return string
  67. if encoding is None:
  68. encoding = 'utf-8'
  69. if errors is None:
  70. errors = 'replace'
  71. # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
  72. pct_sequence = b''
  73. string = res[0]
  74. for item in res[1:]:
  75. try:
  76. if not item:
  77. raise ValueError
  78. pct_sequence += item[:2].decode('hex')
  79. rest = item[2:]
  80. if not rest:
  81. # This segment was just a single percent-encoded character.
  82. # May be part of a sequence of code units, so delay decoding.
  83. # (Stored in pct_sequence).
  84. continue
  85. except ValueError:
  86. rest = '%' + item
  87. # Encountered non-percent-encoded characters. Flush the current
  88. # pct_sequence.
  89. string += pct_sequence.decode(encoding, errors) + rest
  90. pct_sequence = b''
  91. if pct_sequence:
  92. # Flush the final pct_sequence
  93. string += pct_sequence.decode(encoding, errors)
  94. return string
  95. try:
  96. from urllib.parse import parse_qs as compat_parse_qs
  97. except ImportError: # Python 2
  98. # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
  99. # Python 2's version is apparently totally broken
  100. def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
  101. encoding='utf-8', errors='replace'):
  102. qs, _coerce_result = qs, unicode
  103. pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
  104. r = []
  105. for name_value in pairs:
  106. if not name_value and not strict_parsing:
  107. continue
  108. nv = name_value.split('=', 1)
  109. if len(nv) != 2:
  110. if strict_parsing:
  111. raise ValueError("bad query field: %r" % (name_value,))
  112. # Handle case of a control-name with no equal sign
  113. if keep_blank_values:
  114. nv.append('')
  115. else:
  116. continue
  117. if len(nv[1]) or keep_blank_values:
  118. name = nv[0].replace('+', ' ')
  119. name = compat_urllib_parse_unquote(
  120. name, encoding=encoding, errors=errors)
  121. name = _coerce_result(name)
  122. value = nv[1].replace('+', ' ')
  123. value = compat_urllib_parse_unquote(
  124. value, encoding=encoding, errors=errors)
  125. value = _coerce_result(value)
  126. r.append((name, value))
  127. return r
  128. def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
  129. encoding='utf-8', errors='replace'):
  130. parsed_result = {}
  131. pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
  132. encoding=encoding, errors=errors)
  133. for name, value in pairs:
  134. if name in parsed_result:
  135. parsed_result[name].append(value)
  136. else:
  137. parsed_result[name] = [value]
  138. return parsed_result
  139. try:
  140. compat_str = unicode # Python 2
  141. except NameError:
  142. compat_str = str
  143. try:
  144. compat_chr = unichr # Python 2
  145. except NameError:
  146. compat_chr = chr
  147. try:
  148. from xml.etree.ElementTree import ParseError as compat_xml_parse_error
  149. except ImportError: # Python 2.6
  150. from xml.parsers.expat import ExpatError as compat_xml_parse_error
  151. try:
  152. from shlex import quote as shlex_quote
  153. except ImportError: # Python < 3.3
  154. def shlex_quote(s):
  155. if re.match(r'^[-_\w./]+$', s):
  156. return s
  157. else:
  158. return "'" + s.replace("'", "'\"'\"'") + "'"
  159. def compat_ord(c):
  160. if type(c) is int:
  161. return c
  162. else:
  163. return ord(c)
  164. if sys.version_info >= (3, 0):
  165. compat_getenv = os.getenv
  166. compat_expanduser = os.path.expanduser
  167. else:
  168. # Environment variables should be decoded with filesystem encoding.
  169. # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
  170. def compat_getenv(key, default=None):
  171. from .utils import get_filesystem_encoding
  172. env = os.getenv(key, default)
  173. if env:
  174. env = env.decode(get_filesystem_encoding())
  175. return env
  176. # HACK: The default implementations of os.path.expanduser from cpython do not decode
  177. # environment variables with filesystem encoding. We will work around this by
  178. # providing adjusted implementations.
  179. # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
  180. # for different platforms with correct environment variables decoding.
  181. if os.name == 'posix':
  182. def compat_expanduser(path):
  183. """Expand ~ and ~user constructions. If user or $HOME is unknown,
  184. do nothing."""
  185. if not path.startswith('~'):
  186. return path
  187. i = path.find('/', 1)
  188. if i < 0:
  189. i = len(path)
  190. if i == 1:
  191. if 'HOME' not in os.environ:
  192. import pwd
  193. userhome = pwd.getpwuid(os.getuid()).pw_dir
  194. else:
  195. userhome = compat_getenv('HOME')
  196. else:
  197. import pwd
  198. try:
  199. pwent = pwd.getpwnam(path[1:i])
  200. except KeyError:
  201. return path
  202. userhome = pwent.pw_dir
  203. userhome = userhome.rstrip('/')
  204. return (userhome + path[i:]) or '/'
  205. elif os.name == 'nt' or os.name == 'ce':
  206. def compat_expanduser(path):
  207. """Expand ~ and ~user constructs.
  208. If user or $HOME is unknown, do nothing."""
  209. if path[:1] != '~':
  210. return path
  211. i, n = 1, len(path)
  212. while i < n and path[i] not in '/\\':
  213. i = i + 1
  214. if 'HOME' in os.environ:
  215. userhome = compat_getenv('HOME')
  216. elif 'USERPROFILE' in os.environ:
  217. userhome = compat_getenv('USERPROFILE')
  218. elif 'HOMEPATH' not in os.environ:
  219. return path
  220. else:
  221. try:
  222. drive = compat_getenv('HOMEDRIVE')
  223. except KeyError:
  224. drive = ''
  225. userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
  226. if i != 1: # ~user
  227. userhome = os.path.join(os.path.dirname(userhome), path[1:i])
  228. return userhome + path[i:]
  229. else:
  230. compat_expanduser = os.path.expanduser
  231. if sys.version_info < (3, 0):
  232. def compat_print(s):
  233. from .utils import preferredencoding
  234. print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
  235. else:
  236. def compat_print(s):
  237. assert isinstance(s, compat_str)
  238. print(s)
  239. try:
  240. subprocess_check_output = subprocess.check_output
  241. except AttributeError:
  242. def subprocess_check_output(*args, **kwargs):
  243. assert 'input' not in kwargs
  244. p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
  245. output, _ = p.communicate()
  246. ret = p.poll()
  247. if ret:
  248. raise subprocess.CalledProcessError(ret, p.args, output=output)
  249. return output
  250. if sys.version_info < (3, 0) and sys.platform == 'win32':
  251. def compat_getpass(prompt, *args, **kwargs):
  252. if isinstance(prompt, compat_str):
  253. from .utils import preferredencoding
  254. prompt = prompt.encode(preferredencoding())
  255. return getpass.getpass(prompt, *args, **kwargs)
  256. else:
  257. compat_getpass = getpass.getpass
  258. # Old 2.6 and 2.7 releases require kwargs to be bytes
  259. try:
  260. def _testfunc(x):
  261. pass
  262. _testfunc(**{'x': 0})
  263. except TypeError:
  264. def compat_kwargs(kwargs):
  265. return dict((bytes(k), v) for k, v in kwargs.items())
  266. else:
  267. compat_kwargs = lambda kwargs: kwargs
  268. if sys.version_info < (2, 7):
  269. def compat_socket_create_connection(address, timeout, source_address=None):
  270. host, port = address
  271. err = None
  272. for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
  273. af, socktype, proto, canonname, sa = res
  274. sock = None
  275. try:
  276. sock = socket.socket(af, socktype, proto)
  277. sock.settimeout(timeout)
  278. if source_address:
  279. sock.bind(source_address)
  280. sock.connect(sa)
  281. return sock
  282. except socket.error as _:
  283. err = _
  284. if sock is not None:
  285. sock.close()
  286. if err is not None:
  287. raise err
  288. else:
  289. raise error("getaddrinfo returns an empty list")
  290. else:
  291. compat_socket_create_connection = socket.create_connection
  292. # Fix https://github.com/rg3/youtube-dl/issues/4223
  293. # See http://bugs.python.org/issue9161 for what is broken
  294. def workaround_optparse_bug9161():
  295. op = optparse.OptionParser()
  296. og = optparse.OptionGroup(op, 'foo')
  297. try:
  298. og.add_option('-t')
  299. except TypeError:
  300. real_add_option = optparse.OptionGroup.add_option
  301. def _compat_add_option(self, *args, **kwargs):
  302. enc = lambda v: (
  303. v.encode('ascii', 'replace') if isinstance(v, compat_str)
  304. else v)
  305. bargs = [enc(a) for a in args]
  306. bkwargs = dict(
  307. (k, enc(v)) for k, v in kwargs.items())
  308. return real_add_option(self, *bargs, **bkwargs)
  309. optparse.OptionGroup.add_option = _compat_add_option
  310. __all__ = [
  311. 'compat_HTTPError',
  312. 'compat_chr',
  313. 'compat_cookiejar',
  314. 'compat_expanduser',
  315. 'compat_getenv',
  316. 'compat_getpass',
  317. 'compat_html_entities',
  318. 'compat_html_parser',
  319. 'compat_http_client',
  320. 'compat_kwargs',
  321. 'compat_ord',
  322. 'compat_parse_qs',
  323. 'compat_print',
  324. 'compat_str',
  325. 'compat_socket_create_connection',
  326. 'compat_subprocess_get_DEVNULL',
  327. 'compat_urllib_error',
  328. 'compat_urllib_parse',
  329. 'compat_urllib_parse_unquote',
  330. 'compat_urllib_parse_urlparse',
  331. 'compat_urllib_request',
  332. 'compat_urlparse',
  333. 'compat_urlretrieve',
  334. 'compat_xml_parse_error',
  335. 'shlex_quote',
  336. 'subprocess_check_output',
  337. 'workaround_optparse_bug9161',
  338. ]