You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

354 lines
9.8 KiB

  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. import gzip
  4. import htmlentitydefs
  5. import HTMLParser
  6. import locale
  7. import os
  8. import re
  9. import sys
  10. import zlib
  11. import urllib2
  12. import email.utils
  13. import json
  14. try:
  15. import cStringIO as StringIO
  16. except ImportError:
  17. import StringIO
  18. std_headers = {
  19. 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
  20. 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  21. 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  22. 'Accept-Encoding': 'gzip, deflate',
  23. 'Accept-Language': 'en-us,en;q=0.5',
  24. }
  25. def preferredencoding():
  26. """Get preferred encoding.
  27. Returns the best encoding scheme for the system, based on
  28. locale.getpreferredencoding() and some further tweaks.
  29. """
  30. def yield_preferredencoding():
  31. try:
  32. pref = locale.getpreferredencoding()
  33. u'TEST'.encode(pref)
  34. except:
  35. pref = 'UTF-8'
  36. while True:
  37. yield pref
  38. return yield_preferredencoding().next()
  39. def htmlentity_transform(matchobj):
  40. """Transforms an HTML entity to a Unicode character.
  41. This function receives a match object and is intended to be used with
  42. the re.sub() function.
  43. """
  44. entity = matchobj.group(1)
  45. # Known non-numeric HTML entity
  46. if entity in htmlentitydefs.name2codepoint:
  47. return unichr(htmlentitydefs.name2codepoint[entity])
  48. # Unicode character
  49. mobj = re.match(ur'(?u)#(x?\d+)', entity)
  50. if mobj is not None:
  51. numstr = mobj.group(1)
  52. if numstr.startswith(u'x'):
  53. base = 16
  54. numstr = u'0%s' % numstr
  55. else:
  56. base = 10
  57. return unichr(long(numstr, base))
  58. # Unknown entity in name, return its literal representation
  59. return (u'&%s;' % entity)
  60. HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
  61. class IDParser(HTMLParser.HTMLParser):
  62. """Modified HTMLParser that isolates a tag with the specified id"""
  63. def __init__(self, id):
  64. self.id = id
  65. self.result = None
  66. self.started = False
  67. self.depth = {}
  68. self.html = None
  69. self.watch_startpos = False
  70. self.error_count = 0
  71. HTMLParser.HTMLParser.__init__(self)
  72. def error(self, message):
  73. print >> sys.stderr, self.getpos()
  74. if self.error_count > 10 or self.started:
  75. raise HTMLParser.HTMLParseError(message, self.getpos())
  76. self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
  77. self.error_count += 1
  78. self.goahead(1)
  79. def loads(self, html):
  80. self.html = html
  81. self.feed(html)
  82. self.close()
  83. def handle_starttag(self, tag, attrs):
  84. attrs = dict(attrs)
  85. if self.started:
  86. self.find_startpos(None)
  87. if 'id' in attrs and attrs['id'] == self.id:
  88. self.result = [tag]
  89. self.started = True
  90. self.watch_startpos = True
  91. if self.started:
  92. if not tag in self.depth: self.depth[tag] = 0
  93. self.depth[tag] += 1
  94. def handle_endtag(self, tag):
  95. if self.started:
  96. if tag in self.depth: self.depth[tag] -= 1
  97. if self.depth[self.result[0]] == 0:
  98. self.started = False
  99. self.result.append(self.getpos())
  100. def find_startpos(self, x):
  101. """Needed to put the start position of the result (self.result[1])
  102. after the opening tag with the requested id"""
  103. if self.watch_startpos:
  104. self.watch_startpos = False
  105. self.result.append(self.getpos())
  106. handle_entityref = handle_charref = handle_data = handle_comment = \
  107. handle_decl = handle_pi = unknown_decl = find_startpos
  108. def get_result(self):
  109. if self.result == None: return None
  110. if len(self.result) != 3: return None
  111. lines = self.html.split('\n')
  112. lines = lines[self.result[1][0]-1:self.result[2][0]]
  113. lines[0] = lines[0][self.result[1][1]:]
  114. if len(lines) == 1:
  115. lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
  116. lines[-1] = lines[-1][:self.result[2][1]]
  117. return '\n'.join(lines).strip()
  118. def get_element_by_id(id, html):
  119. """Return the content of the tag with the specified id in the passed HTML document"""
  120. parser = IDParser(id)
  121. try:
  122. parser.loads(html)
  123. except HTMLParser.HTMLParseError:
  124. pass
  125. return parser.get_result()
  126. def clean_html(html):
  127. """Clean an HTML snippet into a readable string"""
  128. # Newline vs <br />
  129. html = html.replace('\n', ' ')
  130. html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
  131. # Strip html tags
  132. html = re.sub('<.*?>', '', html)
  133. # Replace html entities
  134. html = unescapeHTML(html)
  135. return html
  136. def sanitize_open(filename, open_mode):
  137. """Try to open the given filename, and slightly tweak it if this fails.
  138. Attempts to open the given filename. If this fails, it tries to change
  139. the filename slightly, step by step, until it's either able to open it
  140. or it fails and raises a final exception, like the standard open()
  141. function.
  142. It returns the tuple (stream, definitive_file_name).
  143. """
  144. try:
  145. if filename == u'-':
  146. if sys.platform == 'win32':
  147. import msvcrt
  148. msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  149. return (sys.stdout, filename)
  150. stream = open(encodeFilename(filename), open_mode)
  151. return (stream, filename)
  152. except (IOError, OSError), err:
  153. # In case of error, try to remove win32 forbidden chars
  154. filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
  155. # An exception here should be caught in the caller
  156. stream = open(encodeFilename(filename), open_mode)
  157. return (stream, filename)
  158. def timeconvert(timestr):
  159. """Convert RFC 2822 defined time string into system timestamp"""
  160. timestamp = None
  161. timetuple = email.utils.parsedate_tz(timestr)
  162. if timetuple is not None:
  163. timestamp = email.utils.mktime_tz(timetuple)
  164. return timestamp
  165. def sanitize_filename(s):
  166. """Sanitizes a string so it could be used as part of a filename."""
  167. def replace_insane(char):
  168. if char in u' .\\/|?*<>:"' or ord(char) < 32:
  169. return '_'
  170. return char
  171. return u''.join(map(replace_insane, s)).strip('_')
  172. def orderedSet(iterable):
  173. """ Remove all duplicates from the input iterable """
  174. res = []
  175. for el in iterable:
  176. if el not in res:
  177. res.append(el)
  178. return res
  179. def unescapeHTML(s):
  180. """
  181. @param s a string (of type unicode)
  182. """
  183. assert type(s) == type(u'')
  184. result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
  185. return result
  186. def encodeFilename(s):
  187. """
  188. @param s The name of the file (of type unicode)
  189. """
  190. assert type(s) == type(u'')
  191. if sys.platform == 'win32' and sys.getwindowsversion().major >= 5:
  192. # Pass u'' directly to use Unicode APIs on Windows 2000 and up
  193. # (Detecting Windows NT 4 is tricky because 'major >= 4' would
  194. # match Windows 9x series as well. Besides, NT 4 is obsolete.)
  195. return s
  196. else:
  197. return s.encode(sys.getfilesystemencoding(), 'ignore')
  198. class DownloadError(Exception):
  199. """Download Error exception.
  200. This exception may be thrown by FileDownloader objects if they are not
  201. configured to continue on errors. They will contain the appropriate
  202. error message.
  203. """
  204. pass
  205. class SameFileError(Exception):
  206. """Same File exception.
  207. This exception will be thrown by FileDownloader objects if they detect
  208. multiple files would have to be downloaded to the same file on disk.
  209. """
  210. pass
  211. class PostProcessingError(Exception):
  212. """Post Processing exception.
  213. This exception may be raised by PostProcessor's .run() method to
  214. indicate an error in the postprocessing task.
  215. """
  216. pass
  217. class MaxDownloadsReached(Exception):
  218. """ --max-downloads limit has been reached. """
  219. pass
  220. class UnavailableVideoError(Exception):
  221. """Unavailable Format exception.
  222. This exception will be thrown when a video is requested
  223. in a format that is not available for that video.
  224. """
  225. pass
  226. class ContentTooShortError(Exception):
  227. """Content Too Short exception.
  228. This exception may be raised by FileDownloader objects when a file they
  229. download is too small for what the server announced first, indicating
  230. the connection was probably interrupted.
  231. """
  232. # Both in bytes
  233. downloaded = None
  234. expected = None
  235. def __init__(self, downloaded, expected):
  236. self.downloaded = downloaded
  237. self.expected = expected
  238. class Trouble(Exception):
  239. """Trouble helper exception
  240. This is an exception to be handled with
  241. FileDownloader.trouble
  242. """
  243. class YoutubeDLHandler(urllib2.HTTPHandler):
  244. """Handler for HTTP requests and responses.
  245. This class, when installed with an OpenerDirector, automatically adds
  246. the standard headers to every HTTP request and handles gzipped and
  247. deflated responses from web servers. If compression is to be avoided in
  248. a particular request, the original request in the program code only has
  249. to include the HTTP header "Youtubedl-No-Compression", which will be
  250. removed before making the real request.
  251. Part of this code was copied from:
  252. http://techknack.net/python-urllib2-handlers/
  253. Andrew Rowls, the author of that code, agreed to release it to the
  254. public domain.
  255. """
  256. @staticmethod
  257. def deflate(data):
  258. try:
  259. return zlib.decompress(data, -zlib.MAX_WBITS)
  260. except zlib.error:
  261. return zlib.decompress(data)
  262. @staticmethod
  263. def addinfourl_wrapper(stream, headers, url, code):
  264. if hasattr(urllib2.addinfourl, 'getcode'):
  265. return urllib2.addinfourl(stream, headers, url, code)
  266. ret = urllib2.addinfourl(stream, headers, url)
  267. ret.code = code
  268. return ret
  269. def http_request(self, req):
  270. for h in std_headers:
  271. if h in req.headers:
  272. del req.headers[h]
  273. req.add_header(h, std_headers[h])
  274. if 'Youtubedl-no-compression' in req.headers:
  275. if 'Accept-encoding' in req.headers:
  276. del req.headers['Accept-encoding']
  277. del req.headers['Youtubedl-no-compression']
  278. return req
  279. def http_response(self, req, resp):
  280. old_resp = resp
  281. # gzip
  282. if resp.headers.get('Content-encoding', '') == 'gzip':
  283. gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
  284. resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
  285. resp.msg = old_resp.msg
  286. # deflate
  287. if resp.headers.get('Content-encoding', '') == 'deflate':
  288. gz = StringIO.StringIO(self.deflate(resp.read()))
  289. resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
  290. resp.msg = old_resp.msg
  291. return resp