You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

332 lines
14 KiB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import json
  5. import base64
  6. import zlib
  7. import xml.etree.ElementTree
  8. from hashlib import sha1
  9. from math import pow, sqrt, floor
  10. from .common import InfoExtractor
  11. from ..compat import (
  12. compat_urllib_parse,
  13. compat_urllib_request,
  14. )
  15. from ..utils import (
  16. ExtractorError,
  17. bytes_to_intlist,
  18. intlist_to_bytes,
  19. unified_strdate,
  20. urlencode_postdata,
  21. )
  22. from ..aes import (
  23. aes_cbc_decrypt,
  24. inc,
  25. )
  26. class CrunchyrollIE(InfoExtractor):
  27. _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
  28. _NETRC_MACHINE = 'crunchyroll'
  29. _TESTS = [{
  30. 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
  31. 'info_dict': {
  32. 'id': '645513',
  33. 'ext': 'flv',
  34. 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
  35. 'description': 'md5:2d17137920c64f2f49981a7797d275ef',
  36. 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
  37. 'uploader': 'Yomiuri Telecasting Corporation (YTV)',
  38. 'upload_date': '20131013',
  39. 'url': 're:(?!.*&amp)',
  40. },
  41. 'params': {
  42. # rtmp
  43. 'skip_download': True,
  44. },
  45. }, {
  46. 'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
  47. 'only_matching': True,
  48. }]
  49. _FORMAT_IDS = {
  50. '360': ('60', '106'),
  51. '480': ('61', '106'),
  52. '720': ('62', '106'),
  53. '1080': ('80', '108'),
  54. }
  55. def _login(self):
  56. (username, password) = self._get_login_info()
  57. if username is None:
  58. return
  59. self.report_login()
  60. login_url = 'https://www.crunchyroll.com/?a=formhandler'
  61. data = urlencode_postdata({
  62. 'formname': 'RpcApiUser_Login',
  63. 'name': username,
  64. 'password': password,
  65. })
  66. login_request = compat_urllib_request.Request(login_url, data)
  67. login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
  68. self._download_webpage(login_request, None, False, 'Wrong login info')
  69. def _real_initialize(self):
  70. self._login()
  71. def _decrypt_subtitles(self, data, iv, id):
  72. data = bytes_to_intlist(data)
  73. iv = bytes_to_intlist(iv)
  74. id = int(id)
  75. def obfuscate_key_aux(count, modulo, start):
  76. output = list(start)
  77. for _ in range(count):
  78. output.append(output[-1] + output[-2])
  79. # cut off start values
  80. output = output[2:]
  81. output = list(map(lambda x: x % modulo + 33, output))
  82. return output
  83. def obfuscate_key(key):
  84. num1 = int(floor(pow(2, 25) * sqrt(6.9)))
  85. num2 = (num1 ^ key) << 5
  86. num3 = key ^ num1
  87. num4 = num3 ^ (num3 >> 3) ^ num2
  88. prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2)))
  89. shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode('ascii')).digest())
  90. # Extend 160 Bit hash to 256 Bit
  91. return shaHash + [0] * 12
  92. key = obfuscate_key(id)
  93. class Counter:
  94. __value = iv
  95. def next_value(self):
  96. temp = self.__value
  97. self.__value = inc(self.__value)
  98. return temp
  99. decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
  100. return zlib.decompress(decrypted_data)
  101. def _convert_subtitles_to_srt(self, sub_root):
  102. output = ''
  103. for i, event in enumerate(sub_root.findall('./events/event'), 1):
  104. start = event.attrib['start'].replace('.', ',')
  105. end = event.attrib['end'].replace('.', ',')
  106. text = event.attrib['text'].replace('\\N', '\n')
  107. output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
  108. return output
  109. def _convert_subtitles_to_ass(self, sub_root):
  110. output = ''
  111. def ass_bool(strvalue):
  112. assvalue = '0'
  113. if strvalue == '1':
  114. assvalue = '-1'
  115. return assvalue
  116. output = '[Script Info]\n'
  117. output += 'Title: %s\n' % sub_root.attrib["title"]
  118. output += 'ScriptType: v4.00+\n'
  119. output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"]
  120. output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"]
  121. output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"]
  122. output += """ScaledBorderAndShadow: yes
  123. [V4+ Styles]
  124. Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
  125. """
  126. for style in sub_root.findall('./styles/style'):
  127. output += 'Style: ' + style.attrib["name"]
  128. output += ',' + style.attrib["font_name"]
  129. output += ',' + style.attrib["font_size"]
  130. output += ',' + style.attrib["primary_colour"]
  131. output += ',' + style.attrib["secondary_colour"]
  132. output += ',' + style.attrib["outline_colour"]
  133. output += ',' + style.attrib["back_colour"]
  134. output += ',' + ass_bool(style.attrib["bold"])
  135. output += ',' + ass_bool(style.attrib["italic"])
  136. output += ',' + ass_bool(style.attrib["underline"])
  137. output += ',' + ass_bool(style.attrib["strikeout"])
  138. output += ',' + style.attrib["scale_x"]
  139. output += ',' + style.attrib["scale_y"]
  140. output += ',' + style.attrib["spacing"]
  141. output += ',' + style.attrib["angle"]
  142. output += ',' + style.attrib["border_style"]
  143. output += ',' + style.attrib["outline"]
  144. output += ',' + style.attrib["shadow"]
  145. output += ',' + style.attrib["alignment"]
  146. output += ',' + style.attrib["margin_l"]
  147. output += ',' + style.attrib["margin_r"]
  148. output += ',' + style.attrib["margin_v"]
  149. output += ',' + style.attrib["encoding"]
  150. output += '\n'
  151. output += """
  152. [Events]
  153. Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
  154. """
  155. for event in sub_root.findall('./events/event'):
  156. output += 'Dialogue: 0'
  157. output += ',' + event.attrib["start"]
  158. output += ',' + event.attrib["end"]
  159. output += ',' + event.attrib["style"]
  160. output += ',' + event.attrib["name"]
  161. output += ',' + event.attrib["margin_l"]
  162. output += ',' + event.attrib["margin_r"]
  163. output += ',' + event.attrib["margin_v"]
  164. output += ',' + event.attrib["effect"]
  165. output += ',' + event.attrib["text"]
  166. output += '\n'
  167. return output
  168. def _get_subtitles(self, video_id, webpage):
  169. subtitles = {}
  170. for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
  171. sub_page = self._download_webpage(
  172. 'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
  173. video_id, note='Downloading subtitles for ' + sub_name)
  174. id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
  175. iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
  176. data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
  177. if not id or not iv or not data:
  178. continue
  179. id = int(id)
  180. iv = base64.b64decode(iv)
  181. data = base64.b64decode(data)
  182. subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
  183. lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
  184. if not lang_code:
  185. continue
  186. sub_root = xml.etree.ElementTree.fromstring(subtitle)
  187. subtitles[lang_code] = [
  188. {
  189. 'ext': 'srt',
  190. 'data': self._convert_subtitles_to_srt(sub_root),
  191. },
  192. {
  193. 'ext': 'ass',
  194. 'data': self._convert_subtitles_to_ass(sub_root),
  195. },
  196. ]
  197. return subtitles
  198. def _real_extract(self, url):
  199. mobj = re.match(self._VALID_URL, url)
  200. video_id = mobj.group('video_id')
  201. if mobj.group('prefix') == 'm':
  202. mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage')
  203. webpage_url = self._search_regex(r'<link rel="canonical" href="([^"]+)" />', mobile_webpage, 'webpage_url')
  204. else:
  205. webpage_url = 'http://www.' + mobj.group('url')
  206. webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
  207. note_m = self._html_search_regex(r'<div class="showmedia-trailer-notice">(.+?)</div>', webpage, 'trailer-notice', default='')
  208. if note_m:
  209. raise ExtractorError(note_m)
  210. mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage)
  211. if mobj:
  212. msg = json.loads(mobj.group('msg'))
  213. if msg.get('type') == 'error':
  214. raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
  215. video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
  216. video_title = re.sub(r' {2,}', ' ', video_title)
  217. video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
  218. if not video_description:
  219. video_description = None
  220. video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
  221. if video_upload_date:
  222. video_upload_date = unified_strdate(video_upload_date)
  223. video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
  224. playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
  225. playerdata_req = compat_urllib_request.Request(playerdata_url)
  226. playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
  227. playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
  228. playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
  229. stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id')
  230. video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
  231. formats = []
  232. for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
  233. stream_quality, stream_format = self._FORMAT_IDS[fmt]
  234. video_format = fmt + 'p'
  235. streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')
  236. # urlencode doesn't work!
  237. streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality=' + stream_quality + '&media%5Fid=' + stream_id + '&video%5Fformat=' + stream_format
  238. streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
  239. streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
  240. streamdata = self._download_xml(
  241. streamdata_req, video_id,
  242. note='Downloading media info for %s' % video_format)
  243. video_url = streamdata.find('.//host').text
  244. video_play_path = streamdata.find('.//file').text
  245. formats.append({
  246. 'url': video_url,
  247. 'play_path': video_play_path,
  248. 'ext': 'flv',
  249. 'format': video_format,
  250. 'format_id': video_format,
  251. })
  252. subtitles = self.extract_subtitles(video_id, webpage)
  253. return {
  254. 'id': video_id,
  255. 'title': video_title,
  256. 'description': video_description,
  257. 'thumbnail': video_thumbnail,
  258. 'uploader': video_uploader,
  259. 'upload_date': video_upload_date,
  260. 'subtitles': subtitles,
  261. 'formats': formats,
  262. }
  263. class CrunchyrollShowPlaylistIE(InfoExtractor):
  264. IE_NAME = "crunchyroll:playlist"
  265. _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
  266. _TESTS = [{
  267. 'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
  268. 'info_dict': {
  269. 'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
  270. 'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
  271. },
  272. 'playlist_count': 13,
  273. }]
  274. def _real_extract(self, url):
  275. show_id = self._match_id(url)
  276. webpage = self._download_webpage(url, show_id)
  277. title = self._html_search_regex(
  278. r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
  279. webpage, 'title')
  280. episode_paths = re.findall(
  281. r'(?s)<li id="showview_videos_media_[0-9]+"[^>]+>.*?<a href="([^"]+)"',
  282. webpage)
  283. entries = [
  284. self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll')
  285. for ep in episode_paths
  286. ]
  287. entries.reverse()
  288. return {
  289. '_type': 'playlist',
  290. 'id': show_id,
  291. 'title': title,
  292. 'entries': entries,
  293. }