You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

325 lines
14 KiB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import json
  5. import base64
  6. import zlib
  7. import xml.etree.ElementTree
  8. from hashlib import sha1
  9. from math import pow, sqrt, floor
  10. from .subtitles import SubtitlesInfoExtractor
  11. from ..compat import (
  12. compat_urllib_parse,
  13. compat_urllib_request,
  14. )
  15. from ..utils import (
  16. ExtractorError,
  17. bytes_to_intlist,
  18. intlist_to_bytes,
  19. unified_strdate,
  20. urlencode_postdata,
  21. )
  22. from ..aes import (
  23. aes_cbc_decrypt,
  24. inc,
  25. )
  26. from .common import InfoExtractor
  27. class CrunchyrollIE(SubtitlesInfoExtractor):
  28. _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
  29. _TEST = {
  30. 'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
  31. #'md5': 'b1639fd6ddfaa43788c85f6d1dddd412',
  32. 'info_dict': {
  33. 'id': '645513',
  34. 'ext': 'flv',
  35. 'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
  36. 'description': 'md5:2d17137920c64f2f49981a7797d275ef',
  37. 'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
  38. 'uploader': 'Yomiuri Telecasting Corporation (YTV)',
  39. 'upload_date': '20131013',
  40. 'url': 're:(?!.*&amp)',
  41. },
  42. 'params': {
  43. # rtmp
  44. 'skip_download': True,
  45. },
  46. }
  47. _FORMAT_IDS = {
  48. '360': ('60', '106'),
  49. '480': ('61', '106'),
  50. '720': ('62', '106'),
  51. '1080': ('80', '108'),
  52. }
  53. def _login(self):
  54. (username, password) = self._get_login_info()
  55. if username is None:
  56. return
  57. self.report_login()
  58. login_url = 'https://www.crunchyroll.com/?a=formhandler'
  59. data = urlencode_postdata({
  60. 'formname': 'RpcApiUser_Login',
  61. 'name': username,
  62. 'password': password,
  63. })
  64. login_request = compat_urllib_request.Request(login_url, data)
  65. login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
  66. self._download_webpage(login_request, None, False, 'Wrong login info')
  67. def _real_initialize(self):
  68. self._login()
  69. def _decrypt_subtitles(self, data, iv, id):
  70. data = bytes_to_intlist(data)
  71. iv = bytes_to_intlist(iv)
  72. id = int(id)
  73. def obfuscate_key_aux(count, modulo, start):
  74. output = list(start)
  75. for _ in range(count):
  76. output.append(output[-1] + output[-2])
  77. # cut off start values
  78. output = output[2:]
  79. output = list(map(lambda x: x % modulo + 33, output))
  80. return output
  81. def obfuscate_key(key):
  82. num1 = int(floor(pow(2, 25) * sqrt(6.9)))
  83. num2 = (num1 ^ key) << 5
  84. num3 = key ^ num1
  85. num4 = num3 ^ (num3 >> 3) ^ num2
  86. prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2)))
  87. shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode('ascii')).digest())
  88. # Extend 160 Bit hash to 256 Bit
  89. return shaHash + [0] * 12
  90. key = obfuscate_key(id)
  91. class Counter:
  92. __value = iv
  93. def next_value(self):
  94. temp = self.__value
  95. self.__value = inc(self.__value)
  96. return temp
  97. decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
  98. return zlib.decompress(decrypted_data)
  99. def _convert_subtitles_to_srt(self, sub_root):
  100. output = ''
  101. for i, event in enumerate(sub_root.findall('./events/event'), 1):
  102. start = event.attrib['start'].replace('.', ',')
  103. end = event.attrib['end'].replace('.', ',')
  104. text = event.attrib['text'].replace('\\N', '\n')
  105. output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
  106. return output
  107. def _convert_subtitles_to_ass(self, sub_root):
  108. output = ''
  109. def ass_bool(strvalue):
  110. assvalue = '0'
  111. if strvalue == '1':
  112. assvalue = '-1'
  113. return assvalue
  114. output = '[Script Info]\n'
  115. output += 'Title: %s\n' % sub_root.attrib["title"]
  116. output += 'ScriptType: v4.00+\n'
  117. output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"]
  118. output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"]
  119. output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"]
  120. output += """ScaledBorderAndShadow: yes
  121. [V4+ Styles]
  122. Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
  123. """
  124. for style in sub_root.findall('./styles/style'):
  125. output += 'Style: ' + style.attrib["name"]
  126. output += ',' + style.attrib["font_name"]
  127. output += ',' + style.attrib["font_size"]
  128. output += ',' + style.attrib["primary_colour"]
  129. output += ',' + style.attrib["secondary_colour"]
  130. output += ',' + style.attrib["outline_colour"]
  131. output += ',' + style.attrib["back_colour"]
  132. output += ',' + ass_bool(style.attrib["bold"])
  133. output += ',' + ass_bool(style.attrib["italic"])
  134. output += ',' + ass_bool(style.attrib["underline"])
  135. output += ',' + ass_bool(style.attrib["strikeout"])
  136. output += ',' + style.attrib["scale_x"]
  137. output += ',' + style.attrib["scale_y"]
  138. output += ',' + style.attrib["spacing"]
  139. output += ',' + style.attrib["angle"]
  140. output += ',' + style.attrib["border_style"]
  141. output += ',' + style.attrib["outline"]
  142. output += ',' + style.attrib["shadow"]
  143. output += ',' + style.attrib["alignment"]
  144. output += ',' + style.attrib["margin_l"]
  145. output += ',' + style.attrib["margin_r"]
  146. output += ',' + style.attrib["margin_v"]
  147. output += ',' + style.attrib["encoding"]
  148. output += '\n'
  149. output += """
  150. [Events]
  151. Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
  152. """
  153. for event in sub_root.findall('./events/event'):
  154. output += 'Dialogue: 0'
  155. output += ',' + event.attrib["start"]
  156. output += ',' + event.attrib["end"]
  157. output += ',' + event.attrib["style"]
  158. output += ',' + event.attrib["name"]
  159. output += ',' + event.attrib["margin_l"]
  160. output += ',' + event.attrib["margin_r"]
  161. output += ',' + event.attrib["margin_v"]
  162. output += ',' + event.attrib["effect"]
  163. output += ',' + event.attrib["text"]
  164. output += '\n'
  165. return output
  166. def _real_extract(self, url):
  167. mobj = re.match(self._VALID_URL, url)
  168. video_id = mobj.group('video_id')
  169. if mobj.group('prefix') == 'm':
  170. mobile_webpage = self._download_webpage(url, video_id, 'Downloading mobile webpage')
  171. webpage_url = self._search_regex(r'<link rel="canonical" href="([^"]+)" />', mobile_webpage, 'webpage_url')
  172. else:
  173. webpage_url = 'http://www.' + mobj.group('url')
  174. webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
  175. note_m = self._html_search_regex(r'<div class="showmedia-trailer-notice">(.+?)</div>', webpage, 'trailer-notice', default='')
  176. if note_m:
  177. raise ExtractorError(note_m)
  178. mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage)
  179. if mobj:
  180. msg = json.loads(mobj.group('msg'))
  181. if msg.get('type') == 'error':
  182. raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
  183. video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
  184. video_title = re.sub(r' {2,}', ' ', video_title)
  185. video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
  186. if not video_description:
  187. video_description = None
  188. video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
  189. if video_upload_date:
  190. video_upload_date = unified_strdate(video_upload_date)
  191. video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
  192. playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
  193. playerdata_req = compat_urllib_request.Request(playerdata_url)
  194. playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
  195. playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
  196. playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
  197. stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id')
  198. video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
  199. formats = []
  200. for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage):
  201. stream_quality, stream_format = self._FORMAT_IDS[fmt]
  202. video_format = fmt + 'p'
  203. streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')
  204. # urlencode doesn't work!
  205. streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality=' + stream_quality + '&media%5Fid=' + stream_id + '&video%5Fformat=' + stream_format
  206. streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
  207. streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
  208. streamdata = self._download_xml(
  209. streamdata_req, video_id,
  210. note='Downloading media info for %s' % video_format)
  211. video_url = streamdata.find('.//host').text
  212. video_play_path = streamdata.find('.//file').text
  213. formats.append({
  214. 'url': video_url,
  215. 'play_path': video_play_path,
  216. 'ext': 'flv',
  217. 'format': video_format,
  218. 'format_id': video_format,
  219. })
  220. subtitles = {}
  221. sub_format = self._downloader.params.get('subtitlesformat', 'srt')
  222. for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
  223. sub_page = self._download_webpage(
  224. 'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
  225. video_id, note='Downloading subtitles for ' + sub_name)
  226. id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
  227. iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
  228. data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
  229. if not id or not iv or not data:
  230. continue
  231. id = int(id)
  232. iv = base64.b64decode(iv)
  233. data = base64.b64decode(data)
  234. subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
  235. lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
  236. if not lang_code:
  237. continue
  238. sub_root = xml.etree.ElementTree.fromstring(subtitle)
  239. if sub_format == 'ass':
  240. subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root)
  241. else:
  242. subtitles[lang_code] = self._convert_subtitles_to_srt(sub_root)
  243. if self._downloader.params.get('listsubtitles', False):
  244. self._list_available_subtitles(video_id, subtitles)
  245. return
  246. return {
  247. 'id': video_id,
  248. 'title': video_title,
  249. 'description': video_description,
  250. 'thumbnail': video_thumbnail,
  251. 'uploader': video_uploader,
  252. 'upload_date': video_upload_date,
  253. 'subtitles': subtitles,
  254. 'formats': formats,
  255. }
  256. class CrunchyrollShowPlaylistIE(InfoExtractor):
  257. IE_NAME = "crunchyroll:playlist"
  258. _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
  259. _TESTS = [{
  260. 'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
  261. 'info_dict': {
  262. 'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
  263. 'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
  264. },
  265. 'playlist_count': 13,
  266. }]
  267. def _real_extract(self, url):
  268. show_id = self._match_id(url)
  269. webpage = self._download_webpage(url, show_id)
  270. title = self._html_search_regex(
  271. r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
  272. webpage, 'title')
  273. episode_paths = re.findall(
  274. r'(?s)<li id="showview_videos_media_[0-9]+"[^>]+>.*?<a href="([^"]+)"',
  275. webpage)
  276. entries = [
  277. self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll')
  278. for ep in episode_paths
  279. ]
  280. entries.reverse()
  281. return {
  282. '_type': 'playlist',
  283. 'id': show_id,
  284. 'title': title,
  285. 'entries': entries,
  286. }