You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

283 lines
10 KiB

10 years ago
10 years ago
  1. from __future__ import unicode_literals
  2. import re
  3. import json
  4. from .common import InfoExtractor
  5. from ..compat import compat_urlparse
  6. from ..utils import (
  7. int_or_none,
  8. parse_duration,
  9. unified_strdate,
  10. )
  11. class AppleTrailersIE(InfoExtractor):
  12. IE_NAME = 'appletrailers'
  13. _VALID_URL = r'https?://(?:www\.|movie)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
  14. _TESTS = [{
  15. 'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
  16. 'info_dict': {
  17. 'id': '5111',
  18. 'title': 'Man of Steel',
  19. },
  20. 'playlist': [
  21. {
  22. 'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
  23. 'info_dict': {
  24. 'id': 'manofsteel-trailer4',
  25. 'ext': 'mov',
  26. 'duration': 111,
  27. 'title': 'Trailer 4',
  28. 'upload_date': '20130523',
  29. 'uploader_id': 'wb',
  30. },
  31. },
  32. {
  33. 'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
  34. 'info_dict': {
  35. 'id': 'manofsteel-trailer3',
  36. 'ext': 'mov',
  37. 'duration': 182,
  38. 'title': 'Trailer 3',
  39. 'upload_date': '20130417',
  40. 'uploader_id': 'wb',
  41. },
  42. },
  43. {
  44. 'md5': 'd0f1e1150989b9924679b441f3404d48',
  45. 'info_dict': {
  46. 'id': 'manofsteel-trailer',
  47. 'ext': 'mov',
  48. 'duration': 148,
  49. 'title': 'Trailer',
  50. 'upload_date': '20121212',
  51. 'uploader_id': 'wb',
  52. },
  53. },
  54. {
  55. 'md5': '5fe08795b943eb2e757fa95cb6def1cb',
  56. 'info_dict': {
  57. 'id': 'manofsteel-teaser',
  58. 'ext': 'mov',
  59. 'duration': 93,
  60. 'title': 'Teaser',
  61. 'upload_date': '20120721',
  62. 'uploader_id': 'wb',
  63. },
  64. },
  65. ]
  66. }, {
  67. 'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
  68. 'info_dict': {
  69. 'id': '4489',
  70. 'title': 'Blackthorn',
  71. },
  72. 'playlist_mincount': 2,
  73. 'expected_warnings': ['Unable to download JSON metadata'],
  74. }, {
  75. # json data only available from http://trailers.apple.com/trailers/feeds/data/15881.json
  76. 'url': 'http://trailers.apple.com/trailers/fox/kungfupanda3/',
  77. 'info_dict': {
  78. 'id': '15881',
  79. 'title': 'Kung Fu Panda 3',
  80. },
  81. 'playlist_mincount': 4,
  82. }, {
  83. 'url': 'http://trailers.apple.com/ca/metropole/autrui/',
  84. 'only_matching': True,
  85. }, {
  86. 'url': 'http://movietrailers.apple.com/trailers/focus_features/kuboandthetwostrings/',
  87. 'only_matching': True,
  88. }]
  89. _JSON_RE = r'iTunes.playURL\((.*?)\);'
  90. def _real_extract(self, url):
  91. mobj = re.match(self._VALID_URL, url)
  92. movie = mobj.group('movie')
  93. uploader_id = mobj.group('company')
  94. webpage = self._download_webpage(url, movie)
  95. film_id = self._search_regex(r"FilmId\s*=\s*'(\d+)'", webpage, 'film id')
  96. film_data = self._download_json(
  97. 'http://trailers.apple.com/trailers/feeds/data/%s.json' % film_id,
  98. film_id, fatal=False)
  99. if film_data:
  100. entries = []
  101. for clip in film_data.get('clips', []):
  102. clip_title = clip['title']
  103. formats = []
  104. for version, version_data in clip.get('versions', {}).items():
  105. for size, size_data in version_data.get('sizes', {}).items():
  106. src = size_data.get('src')
  107. if not src:
  108. continue
  109. formats.append({
  110. 'format_id': '%s-%s' % (version, size),
  111. 'url': re.sub(r'_(\d+p\.mov)', r'_h\1', src),
  112. 'width': int_or_none(size_data.get('width')),
  113. 'height': int_or_none(size_data.get('height')),
  114. 'language': version[:2],
  115. })
  116. self._sort_formats(formats)
  117. entries.append({
  118. 'id': movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', clip_title).lower(),
  119. 'formats': formats,
  120. 'title': clip_title,
  121. 'thumbnail': clip.get('screen') or clip.get('thumb'),
  122. 'duration': parse_duration(clip.get('runtime') or clip.get('faded')),
  123. 'upload_date': unified_strdate(clip.get('posted')),
  124. 'uploader_id': uploader_id,
  125. })
  126. page_data = film_data.get('page', {})
  127. return self.playlist_result(entries, film_id, page_data.get('movie_title'))
  128. playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
  129. def fix_html(s):
  130. s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
  131. s = re.sub(r'<img ([^<]*?)/?>', r'<img \1/>', s)
  132. # The ' in the onClick attributes are not escaped, it couldn't be parsed
  133. # like: http://trailers.apple.com/trailers/wb/gravity/
  134. def _clean_json(m):
  135. return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
  136. s = re.sub(self._JSON_RE, _clean_json, s)
  137. s = '<html>%s</html>' % s
  138. return s
  139. doc = self._download_xml(playlist_url, movie, transform_source=fix_html)
  140. playlist = []
  141. for li in doc.findall('./div/ul/li'):
  142. on_click = li.find('.//a').attrib['onClick']
  143. trailer_info_json = self._search_regex(self._JSON_RE,
  144. on_click, 'trailer info')
  145. trailer_info = json.loads(trailer_info_json)
  146. first_url = trailer_info.get('url')
  147. if not first_url:
  148. continue
  149. title = trailer_info['title']
  150. video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
  151. thumbnail = li.find('.//img').attrib['src']
  152. upload_date = trailer_info['posted'].replace('-', '')
  153. runtime = trailer_info['runtime']
  154. m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
  155. duration = None
  156. if m:
  157. duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
  158. trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
  159. settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
  160. settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')
  161. formats = []
  162. for format in settings['metadata']['sizes']:
  163. # The src is a file pointing to the real video file
  164. format_url = re.sub(r'_(\d*p\.mov)', r'_h\1', format['src'])
  165. formats.append({
  166. 'url': format_url,
  167. 'format': format['type'],
  168. 'width': int_or_none(format['width']),
  169. 'height': int_or_none(format['height']),
  170. })
  171. self._sort_formats(formats)
  172. playlist.append({
  173. '_type': 'video',
  174. 'id': video_id,
  175. 'formats': formats,
  176. 'title': title,
  177. 'duration': duration,
  178. 'thumbnail': thumbnail,
  179. 'upload_date': upload_date,
  180. 'uploader_id': uploader_id,
  181. 'http_headers': {
  182. 'User-Agent': 'QuickTime compatible (youtube-dl)',
  183. },
  184. })
  185. return {
  186. '_type': 'playlist',
  187. 'id': movie,
  188. 'entries': playlist,
  189. }
  190. class AppleTrailersSectionIE(InfoExtractor):
  191. IE_NAME = 'appletrailers:section'
  192. _SECTIONS = {
  193. 'justadded': {
  194. 'feed_path': 'just_added',
  195. 'title': 'Just Added',
  196. },
  197. 'exclusive': {
  198. 'feed_path': 'exclusive',
  199. 'title': 'Exclusive',
  200. },
  201. 'justhd': {
  202. 'feed_path': 'just_hd',
  203. 'title': 'Just HD',
  204. },
  205. 'mostpopular': {
  206. 'feed_path': 'most_pop',
  207. 'title': 'Most Popular',
  208. },
  209. 'moviestudios': {
  210. 'feed_path': 'studios',
  211. 'title': 'Movie Studios',
  212. },
  213. }
  214. _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/#section=(?P<id>%s)' % '|'.join(_SECTIONS)
  215. _TESTS = [{
  216. 'url': 'http://trailers.apple.com/#section=justadded',
  217. 'info_dict': {
  218. 'title': 'Just Added',
  219. 'id': 'justadded',
  220. },
  221. 'playlist_mincount': 80,
  222. }, {
  223. 'url': 'http://trailers.apple.com/#section=exclusive',
  224. 'info_dict': {
  225. 'title': 'Exclusive',
  226. 'id': 'exclusive',
  227. },
  228. 'playlist_mincount': 80,
  229. }, {
  230. 'url': 'http://trailers.apple.com/#section=justhd',
  231. 'info_dict': {
  232. 'title': 'Just HD',
  233. 'id': 'justhd',
  234. },
  235. 'playlist_mincount': 80,
  236. }, {
  237. 'url': 'http://trailers.apple.com/#section=mostpopular',
  238. 'info_dict': {
  239. 'title': 'Most Popular',
  240. 'id': 'mostpopular',
  241. },
  242. 'playlist_mincount': 30,
  243. }, {
  244. 'url': 'http://trailers.apple.com/#section=moviestudios',
  245. 'info_dict': {
  246. 'title': 'Movie Studios',
  247. 'id': 'moviestudios',
  248. },
  249. 'playlist_mincount': 80,
  250. }]
  251. def _real_extract(self, url):
  252. section = self._match_id(url)
  253. section_data = self._download_json(
  254. 'http://trailers.apple.com/trailers/home/feeds/%s.json' % self._SECTIONS[section]['feed_path'],
  255. section)
  256. entries = [
  257. self.url_result('http://trailers.apple.com' + e['location'])
  258. for e in section_data]
  259. return self.playlist_result(entries, section, self._SECTIONS[section]['title'])