You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

283 lines
12 KiB

  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import json
  5. import base64
  6. import zlib
  7. import xml.etree.ElementTree
  8. from hashlib import sha1
  9. from math import pow, sqrt, floor
  10. from .common import InfoExtractor
  11. from ..utils import (
  12. ExtractorError,
  13. compat_urllib_parse,
  14. compat_urllib_request,
  15. bytes_to_intlist,
  16. intlist_to_bytes,
  17. unified_strdate,
  18. clean_html,
  19. urlencode_postdata,
  20. )
  21. from ..aes import (
  22. aes_cbc_decrypt,
  23. inc,
  24. )
  25. class CrunchyrollIE(InfoExtractor):
  26. _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
  27. _TEST = {
  28. 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
  29. #'md5': 'b1639fd6ddfaa43788c85f6d1dddd412',
  30. 'info_dict': {
  31. 'id': '645513',
  32. 'ext': 'flv',
  33. 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
  34. 'description': 'md5:2d17137920c64f2f49981a7797d275ef',
  35. 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
  36. 'uploader': 'Yomiuri Telecasting Corporation (YTV)',
  37. 'upload_date': '20131013',
  38. },
  39. 'params': {
  40. # rtmp
  41. 'skip_download': True,
  42. },
  43. }
  44. _FORMAT_IDS = {
  45. '360': ('60', '106'),
  46. '480': ('61', '106'),
  47. '720': ('62', '106'),
  48. '1080': ('80', '108'),
  49. }
  50. def _login(self):
  51. (username, password) = self._get_login_info()
  52. if username is None:
  53. return
  54. self.report_login()
  55. login_url = 'https://www.crunchyroll.com/?a=formhandler'
  56. data = urlencode_postdata({
  57. 'formname': 'RpcApiUser_Login',
  58. 'name': username,
  59. 'password': password,
  60. })
  61. login_request = compat_urllib_request.Request(login_url, data)
  62. login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
  63. self._download_webpage(login_request, None, False, 'Wrong login info')
  64. def _real_initialize(self):
  65. self._login()
  66. def _decrypt_subtitles(self, data, iv, id):
  67. data = bytes_to_intlist(data)
  68. iv = bytes_to_intlist(iv)
  69. id = int(id)
  70. def obfuscate_key_aux(count, modulo, start):
  71. output = list(start)
  72. for _ in range(count):
  73. output.append(output[-1] + output[-2])
  74. # cut off start values
  75. output = output[2:]
  76. output = list(map(lambda x: x % modulo + 33, output))
  77. return output
  78. def obfuscate_key(key):
  79. num1 = int(floor(pow(2, 25) * sqrt(6.9)))
  80. num2 = (num1 ^ key) << 5
  81. num3 = key ^ num1
  82. num4 = num3 ^ (num3 >> 3) ^ num2
  83. prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2)))
  84. shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode('ascii')).digest())
  85. # Extend 160 Bit hash to 256 Bit
  86. return shaHash + [0] * 12
  87. key = obfuscate_key(id)
  88. class Counter:
  89. __value = iv
  90. def next_value(self):
  91. temp = self.__value
  92. self.__value = inc(self.__value)
  93. return temp
  94. decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
  95. return zlib.decompress(decrypted_data)
  96. def _convert_subtitles_to_srt(self, subtitles):
  97. output = ''
  98. for i, (start, end, text) in enumerate(re.findall(r'<event [^>]*?start="([^"]+)" [^>]*?end="([^"]+)" [^>]*?text="([^"]+)"[^>]*?>', subtitles), 1):
  99. start = start.replace('.', ',')
  100. end = end.replace('.', ',')
  101. text = clean_html(text)
  102. text = text.replace('\\N', '\n')
  103. if not text:
  104. continue
  105. output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
  106. return output
  107. def _convert_subtitles_to_ass(self, subtitles):
  108. output = ''
  109. def ass_bool(strvalue):
  110. assvalue = '0'
  111. if strvalue == '1':
  112. assvalue = '-1'
  113. return assvalue
  114. sub_root = xml.etree.ElementTree.fromstring(subtitles)
  115. if not sub_root:
  116. return output
  117. output = '[Script Info]\n'
  118. output += 'Title: %s\n' % sub_root.attrib["title"]
  119. output += 'ScriptType: v4.00+\n'
  120. output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"]
  121. output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"]
  122. output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"]
  123. output += """ScaledBorderAndShadow: yes
  124. [V4+ Styles]
  125. Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
  126. """
  127. for style in sub_root.findall('./styles/style'):
  128. output += 'Style: ' + style.attrib["name"]
  129. output += ',' + style.attrib["font_name"]
  130. output += ',' + style.attrib["font_size"]
  131. output += ',' + style.attrib["primary_colour"]
  132. output += ',' + style.attrib["secondary_colour"]
  133. output += ',' + style.attrib["outline_colour"]
  134. output += ',' + style.attrib["back_colour"]
  135. output += ',' + ass_bool(style.attrib["bold"])
  136. output += ',' + ass_bool(style.attrib["italic"])
  137. output += ',' + ass_bool(style.attrib["underline"])
  138. output += ',' + ass_bool(style.attrib["strikeout"])
  139. output += ',' + style.attrib["scale_x"]
  140. output += ',' + style.attrib["scale_y"]
  141. output += ',' + style.attrib["spacing"]
  142. output += ',' + style.attrib["angle"]
  143. output += ',' + style.attrib["border_style"]
  144. output += ',' + style.attrib["outline"]
  145. output += ',' + style.attrib["shadow"]
  146. output += ',' + style.attrib["alignment"]
  147. output += ',' + style.attrib["margin_l"]
  148. output += ',' + style.attrib["margin_r"]
  149. output += ',' + style.attrib["margin_v"]
  150. output += ',' + style.attrib["encoding"]
  151. output += '\n'
  152. output += """
  153. [Events]
  154. Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
  155. """
  156. for event in sub_root.findall('./events/event'):
  157. output += 'Dialogue: 0'
  158. output += ',' + event.attrib["start"]
  159. output += ',' + event.attrib["end"]
  160. output += ',' + event.attrib["style"]
  161. output += ',' + event.attrib["name"]
  162. output += ',' + event.attrib["margin_l"]
  163. output += ',' + event.attrib["margin_r"]
  164. output += ',' + event.attrib["margin_v"]
  165. output += ',' + event.attrib["effect"]
  166. output += ',' + event.attrib["text"]
  167. output += '\n'
  168. return output
  169. def _real_extract(self,url):
  170. mobj = re.match(self._VALID_URL, url)
  171. video_id = mobj.group('video_id')
  172. if mobj.group('prefix') == 'm':
  173. mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage')
  174. webpage_url = self._search_regex(r'<link rel="canonical" href="([^"]+)" />', mobile_webpage, 'webpage_url')
  175. else:
  176. webpage_url = 'http://www.' + mobj.group('url')
  177. webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
  178. note_m = self._html_search_regex(r'<div class="showmedia-trailer-notice">(.+?)</div>', webpage, 'trailer-notice', default='')
  179. if note_m:
  180. raise ExtractorError(note_m)
  181. mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage)
  182. if mobj:
  183. msg = json.loads(mobj.group('msg'))
  184. if msg.get('type') == 'error':
  185. raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
  186. video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
  187. video_title = re.sub(r' {2,}', ' ', video_title)
  188. video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
  189. if not video_description:
  190. video_description = None
  191. video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
  192. if video_upload_date:
  193. video_upload_date = unified_strdate(video_upload_date)
  194. video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
  195. playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
  196. playerdata_req = compat_urllib_request.Request(playerdata_url)
  197. playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
  198. playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
  199. playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
  200. stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id')
  201. video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
  202. formats = []
  203. for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage):
  204. stream_quality, stream_format = self._FORMAT_IDS[fmt]
  205. video_format = fmt+'p'
  206. streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')
  207. # urlencode doesn't work!
  208. streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality='+stream_quality+'&media%5Fid='+stream_id+'&video%5Fformat='+stream_format
  209. streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
  210. streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
  211. streamdata = self._download_webpage(streamdata_req, video_id, note='Downloading media info for '+video_format)
  212. video_url = self._search_regex(r'<host>([^<]+)', streamdata, 'video_url')
  213. video_play_path = self._search_regex(r'<file>([^<]+)', streamdata, 'video_play_path')
  214. formats.append({
  215. 'url': video_url,
  216. 'play_path': video_play_path,
  217. 'ext': 'flv',
  218. 'format': video_format,
  219. 'format_id': video_format,
  220. })
  221. subtitles = {}
  222. sub_format = self._downloader.params.get('subtitlesformat', 'srt')
  223. for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
  224. sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\
  225. video_id, note='Downloading subtitles for '+sub_name)
  226. id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
  227. iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
  228. data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
  229. if not id or not iv or not data:
  230. continue
  231. id = int(id)
  232. iv = base64.b64decode(iv)
  233. data = base64.b64decode(data)
  234. subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
  235. lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
  236. if not lang_code:
  237. continue
  238. if sub_format == 'ass':
  239. subtitles[lang_code] = self._convert_subtitles_to_ass(subtitle)
  240. else:
  241. subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)
  242. return {
  243. 'id': video_id,
  244. 'title': video_title,
  245. 'description': video_description,
  246. 'thumbnail': video_thumbnail,
  247. 'uploader': video_uploader,
  248. 'upload_date': video_upload_date,
  249. 'subtitles': subtitles,
  250. 'formats': formats,
  251. }