You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

350 lines
11 KiB

  1. from __future__ import unicode_literals
  2. import getpass
  3. import optparse
  4. import os
  5. import subprocess
  6. import sys
  7. try:
  8. import urllib.request as compat_urllib_request
  9. except ImportError: # Python 2
  10. import urllib2 as compat_urllib_request
  11. try:
  12. import urllib.error as compat_urllib_error
  13. except ImportError: # Python 2
  14. import urllib2 as compat_urllib_error
  15. try:
  16. import urllib.parse as compat_urllib_parse
  17. except ImportError: # Python 2
  18. import urllib as compat_urllib_parse
  19. try:
  20. from urllib.parse import urlparse as compat_urllib_parse_urlparse
  21. except ImportError: # Python 2
  22. from urlparse import urlparse as compat_urllib_parse_urlparse
  23. try:
  24. import urllib.parse as compat_urlparse
  25. except ImportError: # Python 2
  26. import urlparse as compat_urlparse
  27. try:
  28. import http.cookiejar as compat_cookiejar
  29. except ImportError: # Python 2
  30. import cookielib as compat_cookiejar
  31. try:
  32. import html.entities as compat_html_entities
  33. except ImportError: # Python 2
  34. import htmlentitydefs as compat_html_entities
  35. try:
  36. import html.parser as compat_html_parser
  37. except ImportError: # Python 2
  38. import HTMLParser as compat_html_parser
  39. try:
  40. import http.client as compat_http_client
  41. except ImportError: # Python 2
  42. import httplib as compat_http_client
  43. try:
  44. from urllib.error import HTTPError as compat_HTTPError
  45. except ImportError: # Python 2
  46. from urllib2 import HTTPError as compat_HTTPError
  47. try:
  48. from urllib.request import urlretrieve as compat_urlretrieve
  49. except ImportError: # Python 2
  50. from urllib import urlretrieve as compat_urlretrieve
  51. try:
  52. from subprocess import DEVNULL
  53. compat_subprocess_get_DEVNULL = lambda: DEVNULL
  54. except ImportError:
  55. compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
  56. try:
  57. from urllib.parse import unquote as compat_urllib_parse_unquote
  58. except ImportError:
  59. def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
  60. if string == '':
  61. return string
  62. res = string.split('%')
  63. if len(res) == 1:
  64. return string
  65. if encoding is None:
  66. encoding = 'utf-8'
  67. if errors is None:
  68. errors = 'replace'
  69. # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
  70. pct_sequence = b''
  71. string = res[0]
  72. for item in res[1:]:
  73. try:
  74. if not item:
  75. raise ValueError
  76. pct_sequence += item[:2].decode('hex')
  77. rest = item[2:]
  78. if not rest:
  79. # This segment was just a single percent-encoded character.
  80. # May be part of a sequence of code units, so delay decoding.
  81. # (Stored in pct_sequence).
  82. continue
  83. except ValueError:
  84. rest = '%' + item
  85. # Encountered non-percent-encoded characters. Flush the current
  86. # pct_sequence.
  87. string += pct_sequence.decode(encoding, errors) + rest
  88. pct_sequence = b''
  89. if pct_sequence:
  90. # Flush the final pct_sequence
  91. string += pct_sequence.decode(encoding, errors)
  92. return string
  93. try:
  94. from urllib.parse import parse_qs as compat_parse_qs
  95. except ImportError: # Python 2
  96. # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
  97. # Python 2's version is apparently totally broken
  98. def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
  99. encoding='utf-8', errors='replace'):
  100. qs, _coerce_result = qs, unicode
  101. pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
  102. r = []
  103. for name_value in pairs:
  104. if not name_value and not strict_parsing:
  105. continue
  106. nv = name_value.split('=', 1)
  107. if len(nv) != 2:
  108. if strict_parsing:
  109. raise ValueError("bad query field: %r" % (name_value,))
  110. # Handle case of a control-name with no equal sign
  111. if keep_blank_values:
  112. nv.append('')
  113. else:
  114. continue
  115. if len(nv[1]) or keep_blank_values:
  116. name = nv[0].replace('+', ' ')
  117. name = compat_urllib_parse_unquote(
  118. name, encoding=encoding, errors=errors)
  119. name = _coerce_result(name)
  120. value = nv[1].replace('+', ' ')
  121. value = compat_urllib_parse_unquote(
  122. value, encoding=encoding, errors=errors)
  123. value = _coerce_result(value)
  124. r.append((name, value))
  125. return r
  126. def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
  127. encoding='utf-8', errors='replace'):
  128. parsed_result = {}
  129. pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
  130. encoding=encoding, errors=errors)
  131. for name, value in pairs:
  132. if name in parsed_result:
  133. parsed_result[name].append(value)
  134. else:
  135. parsed_result[name] = [value]
  136. return parsed_result
  137. try:
  138. compat_str = unicode # Python 2
  139. except NameError:
  140. compat_str = str
  141. try:
  142. compat_chr = unichr # Python 2
  143. except NameError:
  144. compat_chr = chr
  145. try:
  146. from xml.etree.ElementTree import ParseError as compat_xml_parse_error
  147. except ImportError: # Python 2.6
  148. from xml.parsers.expat import ExpatError as compat_xml_parse_error
  149. try:
  150. from shlex import quote as shlex_quote
  151. except ImportError: # Python < 3.3
  152. def shlex_quote(s):
  153. return "'" + s.replace("'", "'\"'\"'") + "'"
  154. def compat_ord(c):
  155. if type(c) is int: return c
  156. else: return ord(c)
  157. if sys.version_info >= (3, 0):
  158. compat_getenv = os.getenv
  159. compat_expanduser = os.path.expanduser
  160. else:
  161. # Environment variables should be decoded with filesystem encoding.
  162. # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
  163. def compat_getenv(key, default=None):
  164. from .utils import get_filesystem_encoding
  165. env = os.getenv(key, default)
  166. if env:
  167. env = env.decode(get_filesystem_encoding())
  168. return env
  169. # HACK: The default implementations of os.path.expanduser from cpython do not decode
  170. # environment variables with filesystem encoding. We will work around this by
  171. # providing adjusted implementations.
  172. # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
  173. # for different platforms with correct environment variables decoding.
  174. if os.name == 'posix':
  175. def compat_expanduser(path):
  176. """Expand ~ and ~user constructions. If user or $HOME is unknown,
  177. do nothing."""
  178. if not path.startswith('~'):
  179. return path
  180. i = path.find('/', 1)
  181. if i < 0:
  182. i = len(path)
  183. if i == 1:
  184. if 'HOME' not in os.environ:
  185. import pwd
  186. userhome = pwd.getpwuid(os.getuid()).pw_dir
  187. else:
  188. userhome = compat_getenv('HOME')
  189. else:
  190. import pwd
  191. try:
  192. pwent = pwd.getpwnam(path[1:i])
  193. except KeyError:
  194. return path
  195. userhome = pwent.pw_dir
  196. userhome = userhome.rstrip('/')
  197. return (userhome + path[i:]) or '/'
  198. elif os.name == 'nt' or os.name == 'ce':
  199. def compat_expanduser(path):
  200. """Expand ~ and ~user constructs.
  201. If user or $HOME is unknown, do nothing."""
  202. if path[:1] != '~':
  203. return path
  204. i, n = 1, len(path)
  205. while i < n and path[i] not in '/\\':
  206. i = i + 1
  207. if 'HOME' in os.environ:
  208. userhome = compat_getenv('HOME')
  209. elif 'USERPROFILE' in os.environ:
  210. userhome = compat_getenv('USERPROFILE')
  211. elif not 'HOMEPATH' in os.environ:
  212. return path
  213. else:
  214. try:
  215. drive = compat_getenv('HOMEDRIVE')
  216. except KeyError:
  217. drive = ''
  218. userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
  219. if i != 1: #~user
  220. userhome = os.path.join(os.path.dirname(userhome), path[1:i])
  221. return userhome + path[i:]
  222. else:
  223. compat_expanduser = os.path.expanduser
  224. if sys.version_info < (3, 0):
  225. def compat_print(s):
  226. from .utils import preferredencoding
  227. print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
  228. else:
  229. def compat_print(s):
  230. assert type(s) == type(u'')
  231. print(s)
  232. try:
  233. subprocess_check_output = subprocess.check_output
  234. except AttributeError:
  235. def subprocess_check_output(*args, **kwargs):
  236. assert 'input' not in kwargs
  237. p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
  238. output, _ = p.communicate()
  239. ret = p.poll()
  240. if ret:
  241. raise subprocess.CalledProcessError(ret, p.args, output=output)
  242. return output
  243. if sys.version_info < (3, 0) and sys.platform == 'win32':
  244. def compat_getpass(prompt, *args, **kwargs):
  245. if isinstance(prompt, compat_str):
  246. from .utils import preferredencoding
  247. prompt = prompt.encode(preferredencoding())
  248. return getpass.getpass(prompt, *args, **kwargs)
  249. else:
  250. compat_getpass = getpass.getpass
  251. # Old 2.6 and 2.7 releases require kwargs to be bytes
  252. try:
  253. (lambda x: x)(**{'x': 0})
  254. except TypeError:
  255. def compat_kwargs(kwargs):
  256. return dict((bytes(k), v) for k, v in kwargs.items())
  257. else:
  258. compat_kwargs = lambda kwargs: kwargs
  259. # Fix https://github.com/rg3/youtube-dl/issues/4223
  260. # See http://bugs.python.org/issue9161 for what is broken
  261. def workaround_optparse_bug9161():
  262. op = optparse.OptionParser()
  263. og = optparse.OptionGroup(op, 'foo')
  264. try:
  265. og.add_option('-t')
  266. except TypeError as te:
  267. real_add_option = optparse.OptionGroup.add_option
  268. def _compat_add_option(self, *args, **kwargs):
  269. enc = lambda v: (
  270. v.encode('ascii', 'replace') if isinstance(v, compat_str)
  271. else v)
  272. bargs = [enc(a) for a in args]
  273. bkwargs = dict(
  274. (k, enc(v)) for k, v in kwargs.items())
  275. return real_add_option(self, *bargs, **bkwargs)
  276. optparse.OptionGroup.add_option = _compat_add_option
  277. __all__ = [
  278. 'compat_HTTPError',
  279. 'compat_chr',
  280. 'compat_cookiejar',
  281. 'compat_expanduser',
  282. 'compat_getenv',
  283. 'compat_getpass',
  284. 'compat_html_entities',
  285. 'compat_html_parser',
  286. 'compat_http_client',
  287. 'compat_kwargs',
  288. 'compat_ord',
  289. 'compat_parse_qs',
  290. 'compat_print',
  291. 'compat_str',
  292. 'compat_subprocess_get_DEVNULL',
  293. 'compat_urllib_error',
  294. 'compat_urllib_parse',
  295. 'compat_urllib_parse_unquote',
  296. 'compat_urllib_parse_urlparse',
  297. 'compat_urllib_request',
  298. 'compat_urlparse',
  299. 'compat_urlretrieve',
  300. 'compat_xml_parse_error',
  301. 'shlex_quote',
  302. 'subprocess_check_output',
  303. 'workaround_optparse_bug9161',
  304. ]