You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

356 lines
11 KiB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
  1. from __future__ import unicode_literals
  2. import getpass
  3. import optparse
  4. import os
  5. import re
  6. import subprocess
  7. import sys
  8. try:
  9. import urllib.request as compat_urllib_request
  10. except ImportError: # Python 2
  11. import urllib2 as compat_urllib_request
  12. try:
  13. import urllib.error as compat_urllib_error
  14. except ImportError: # Python 2
  15. import urllib2 as compat_urllib_error
  16. try:
  17. import urllib.parse as compat_urllib_parse
  18. except ImportError: # Python 2
  19. import urllib as compat_urllib_parse
  20. try:
  21. from urllib.parse import urlparse as compat_urllib_parse_urlparse
  22. except ImportError: # Python 2
  23. from urlparse import urlparse as compat_urllib_parse_urlparse
  24. try:
  25. import urllib.parse as compat_urlparse
  26. except ImportError: # Python 2
  27. import urlparse as compat_urlparse
  28. try:
  29. import http.cookiejar as compat_cookiejar
  30. except ImportError: # Python 2
  31. import cookielib as compat_cookiejar
  32. try:
  33. import html.entities as compat_html_entities
  34. except ImportError: # Python 2
  35. import htmlentitydefs as compat_html_entities
  36. try:
  37. import html.parser as compat_html_parser
  38. except ImportError: # Python 2
  39. import HTMLParser as compat_html_parser
  40. try:
  41. import http.client as compat_http_client
  42. except ImportError: # Python 2
  43. import httplib as compat_http_client
  44. try:
  45. from urllib.error import HTTPError as compat_HTTPError
  46. except ImportError: # Python 2
  47. from urllib2 import HTTPError as compat_HTTPError
  48. try:
  49. from urllib.request import urlretrieve as compat_urlretrieve
  50. except ImportError: # Python 2
  51. from urllib import urlretrieve as compat_urlretrieve
  52. try:
  53. from subprocess import DEVNULL
  54. compat_subprocess_get_DEVNULL = lambda: DEVNULL
  55. except ImportError:
  56. compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
  57. try:
  58. from urllib.parse import unquote as compat_urllib_parse_unquote
  59. except ImportError:
  60. def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
  61. if string == '':
  62. return string
  63. res = string.split('%')
  64. if len(res) == 1:
  65. return string
  66. if encoding is None:
  67. encoding = 'utf-8'
  68. if errors is None:
  69. errors = 'replace'
  70. # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
  71. pct_sequence = b''
  72. string = res[0]
  73. for item in res[1:]:
  74. try:
  75. if not item:
  76. raise ValueError
  77. pct_sequence += item[:2].decode('hex')
  78. rest = item[2:]
  79. if not rest:
  80. # This segment was just a single percent-encoded character.
  81. # May be part of a sequence of code units, so delay decoding.
  82. # (Stored in pct_sequence).
  83. continue
  84. except ValueError:
  85. rest = '%' + item
  86. # Encountered non-percent-encoded characters. Flush the current
  87. # pct_sequence.
  88. string += pct_sequence.decode(encoding, errors) + rest
  89. pct_sequence = b''
  90. if pct_sequence:
  91. # Flush the final pct_sequence
  92. string += pct_sequence.decode(encoding, errors)
  93. return string
  94. try:
  95. from urllib.parse import parse_qs as compat_parse_qs
  96. except ImportError: # Python 2
  97. # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
  98. # Python 2's version is apparently totally broken
  99. def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
  100. encoding='utf-8', errors='replace'):
  101. qs, _coerce_result = qs, unicode
  102. pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
  103. r = []
  104. for name_value in pairs:
  105. if not name_value and not strict_parsing:
  106. continue
  107. nv = name_value.split('=', 1)
  108. if len(nv) != 2:
  109. if strict_parsing:
  110. raise ValueError("bad query field: %r" % (name_value,))
  111. # Handle case of a control-name with no equal sign
  112. if keep_blank_values:
  113. nv.append('')
  114. else:
  115. continue
  116. if len(nv[1]) or keep_blank_values:
  117. name = nv[0].replace('+', ' ')
  118. name = compat_urllib_parse_unquote(
  119. name, encoding=encoding, errors=errors)
  120. name = _coerce_result(name)
  121. value = nv[1].replace('+', ' ')
  122. value = compat_urllib_parse_unquote(
  123. value, encoding=encoding, errors=errors)
  124. value = _coerce_result(value)
  125. r.append((name, value))
  126. return r
  127. def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
  128. encoding='utf-8', errors='replace'):
  129. parsed_result = {}
  130. pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
  131. encoding=encoding, errors=errors)
  132. for name, value in pairs:
  133. if name in parsed_result:
  134. parsed_result[name].append(value)
  135. else:
  136. parsed_result[name] = [value]
  137. return parsed_result
  138. try:
  139. compat_str = unicode # Python 2
  140. except NameError:
  141. compat_str = str
  142. try:
  143. compat_chr = unichr # Python 2
  144. except NameError:
  145. compat_chr = chr
  146. try:
  147. from xml.etree.ElementTree import ParseError as compat_xml_parse_error
  148. except ImportError: # Python 2.6
  149. from xml.parsers.expat import ExpatError as compat_xml_parse_error
  150. try:
  151. from shlex import quote as shlex_quote
  152. except ImportError: # Python < 3.3
  153. def shlex_quote(s):
  154. if re.match(r'^[-_\w./]+$', s):
  155. return s
  156. else:
  157. return "'" + s.replace("'", "'\"'\"'") + "'"
  158. def compat_ord(c):
  159. if type(c) is int:
  160. return c
  161. else:
  162. return ord(c)
  163. if sys.version_info >= (3, 0):
  164. compat_getenv = os.getenv
  165. compat_expanduser = os.path.expanduser
  166. else:
  167. # Environment variables should be decoded with filesystem encoding.
  168. # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
  169. def compat_getenv(key, default=None):
  170. from .utils import get_filesystem_encoding
  171. env = os.getenv(key, default)
  172. if env:
  173. env = env.decode(get_filesystem_encoding())
  174. return env
  175. # HACK: The default implementations of os.path.expanduser from cpython do not decode
  176. # environment variables with filesystem encoding. We will work around this by
  177. # providing adjusted implementations.
  178. # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
  179. # for different platforms with correct environment variables decoding.
  180. if os.name == 'posix':
  181. def compat_expanduser(path):
  182. """Expand ~ and ~user constructions. If user or $HOME is unknown,
  183. do nothing."""
  184. if not path.startswith('~'):
  185. return path
  186. i = path.find('/', 1)
  187. if i < 0:
  188. i = len(path)
  189. if i == 1:
  190. if 'HOME' not in os.environ:
  191. import pwd
  192. userhome = pwd.getpwuid(os.getuid()).pw_dir
  193. else:
  194. userhome = compat_getenv('HOME')
  195. else:
  196. import pwd
  197. try:
  198. pwent = pwd.getpwnam(path[1:i])
  199. except KeyError:
  200. return path
  201. userhome = pwent.pw_dir
  202. userhome = userhome.rstrip('/')
  203. return (userhome + path[i:]) or '/'
  204. elif os.name == 'nt' or os.name == 'ce':
  205. def compat_expanduser(path):
  206. """Expand ~ and ~user constructs.
  207. If user or $HOME is unknown, do nothing."""
  208. if path[:1] != '~':
  209. return path
  210. i, n = 1, len(path)
  211. while i < n and path[i] not in '/\\':
  212. i = i + 1
  213. if 'HOME' in os.environ:
  214. userhome = compat_getenv('HOME')
  215. elif 'USERPROFILE' in os.environ:
  216. userhome = compat_getenv('USERPROFILE')
  217. elif not 'HOMEPATH' in os.environ:
  218. return path
  219. else:
  220. try:
  221. drive = compat_getenv('HOMEDRIVE')
  222. except KeyError:
  223. drive = ''
  224. userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
  225. if i != 1: # ~user
  226. userhome = os.path.join(os.path.dirname(userhome), path[1:i])
  227. return userhome + path[i:]
  228. else:
  229. compat_expanduser = os.path.expanduser
  230. if sys.version_info < (3, 0):
  231. def compat_print(s):
  232. from .utils import preferredencoding
  233. print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
  234. else:
  235. def compat_print(s):
  236. assert isinstance(s, compat_str)
  237. print(s)
  238. try:
  239. subprocess_check_output = subprocess.check_output
  240. except AttributeError:
  241. def subprocess_check_output(*args, **kwargs):
  242. assert 'input' not in kwargs
  243. p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
  244. output, _ = p.communicate()
  245. ret = p.poll()
  246. if ret:
  247. raise subprocess.CalledProcessError(ret, p.args, output=output)
  248. return output
  249. if sys.version_info < (3, 0) and sys.platform == 'win32':
  250. def compat_getpass(prompt, *args, **kwargs):
  251. if isinstance(prompt, compat_str):
  252. from .utils import preferredencoding
  253. prompt = prompt.encode(preferredencoding())
  254. return getpass.getpass(prompt, *args, **kwargs)
  255. else:
  256. compat_getpass = getpass.getpass
  257. # Old 2.6 and 2.7 releases require kwargs to be bytes
  258. try:
  259. (lambda x: x)(**{'x': 0})
  260. except TypeError:
  261. def compat_kwargs(kwargs):
  262. return dict((bytes(k), v) for k, v in kwargs.items())
  263. else:
  264. compat_kwargs = lambda kwargs: kwargs
  265. # Fix https://github.com/rg3/youtube-dl/issues/4223
  266. # See http://bugs.python.org/issue9161 for what is broken
  267. def workaround_optparse_bug9161():
  268. op = optparse.OptionParser()
  269. og = optparse.OptionGroup(op, 'foo')
  270. try:
  271. og.add_option('-t')
  272. except TypeError:
  273. real_add_option = optparse.OptionGroup.add_option
  274. def _compat_add_option(self, *args, **kwargs):
  275. enc = lambda v: (
  276. v.encode('ascii', 'replace') if isinstance(v, compat_str)
  277. else v)
  278. bargs = [enc(a) for a in args]
  279. bkwargs = dict(
  280. (k, enc(v)) for k, v in kwargs.items())
  281. return real_add_option(self, *bargs, **bkwargs)
  282. optparse.OptionGroup.add_option = _compat_add_option
  283. __all__ = [
  284. 'compat_HTTPError',
  285. 'compat_chr',
  286. 'compat_cookiejar',
  287. 'compat_expanduser',
  288. 'compat_getenv',
  289. 'compat_getpass',
  290. 'compat_html_entities',
  291. 'compat_html_parser',
  292. 'compat_http_client',
  293. 'compat_kwargs',
  294. 'compat_ord',
  295. 'compat_parse_qs',
  296. 'compat_print',
  297. 'compat_str',
  298. 'compat_subprocess_get_DEVNULL',
  299. 'compat_urllib_error',
  300. 'compat_urllib_parse',
  301. 'compat_urllib_parse_unquote',
  302. 'compat_urllib_parse_urlparse',
  303. 'compat_urllib_request',
  304. 'compat_urlparse',
  305. 'compat_urlretrieve',
  306. 'compat_xml_parse_error',
  307. 'shlex_quote',
  308. 'subprocess_check_output',
  309. 'workaround_optparse_bug9161',
  310. ]