You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

263 lines
8.6 KiB

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. from .common import InfoExtractor
  5. from ..compat import (
  6. compat_str,
  7. compat_urlparse,
  8. )
  9. from ..utils import (
  10. ExtractorError,
  11. parse_duration,
  12. url_or_none,
  13. urlencode_postdata,
  14. )
  15. class FrontendMastersBaseIE(InfoExtractor):
  16. _API_BASE = 'https://api.frontendmasters.com/v1/kabuki'
  17. _LOGIN_URL = 'https://frontendmasters.com/login/'
  18. _NETRC_MACHINE = 'frontendmasters'
  19. _QUALITIES = {
  20. 'low': {'width': 480, 'height': 360},
  21. 'mid': {'width': 1280, 'height': 720},
  22. 'high': {'width': 1920, 'height': 1080}
  23. }
  24. def _real_initialize(self):
  25. self._login()
  26. def _login(self):
  27. (username, password) = self._get_login_info()
  28. if username is None:
  29. return
  30. login_page = self._download_webpage(
  31. self._LOGIN_URL, None, 'Downloading login page')
  32. login_form = self._hidden_inputs(login_page)
  33. login_form.update({
  34. 'username': username,
  35. 'password': password
  36. })
  37. post_url = self._search_regex(
  38. r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
  39. 'post_url', default=self._LOGIN_URL, group='url')
  40. if not post_url.startswith('http'):
  41. post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
  42. response = self._download_webpage(
  43. post_url, None, 'Logging in', data=urlencode_postdata(login_form),
  44. headers={'Content-Type': 'application/x-www-form-urlencoded'})
  45. # Successful login
  46. if any(p in response for p in (
  47. 'wp-login.php?action=logout', '>Logout')):
  48. return
  49. error = self._html_search_regex(
  50. r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<',
  51. response, 'error message', default=None, group='error')
  52. if error:
  53. raise ExtractorError('Unable to login: %s' % error, expected=True)
  54. raise ExtractorError('Unable to log in')
  55. class FrontendMastersPageBaseIE(FrontendMastersBaseIE):
  56. def _download_course(self, course_name, url):
  57. return self._download_json(
  58. '%s/courses/%s' % (self._API_BASE, course_name), course_name,
  59. 'Downloading course JSON', headers={'Referer': url})
  60. @staticmethod
  61. def _extract_chapters(course):
  62. chapters = []
  63. lesson_elements = course.get('lessonElements')
  64. if isinstance(lesson_elements, list):
  65. chapters = [url_or_none(e) for e in lesson_elements if url_or_none(e)]
  66. return chapters
  67. @staticmethod
  68. def _extract_lesson(chapters, lesson_id, lesson):
  69. title = lesson.get('title') or lesson_id
  70. display_id = lesson.get('slug')
  71. description = lesson.get('description')
  72. thumbnail = lesson.get('thumbnail')
  73. chapter_number = None
  74. index = lesson.get('index')
  75. element_index = lesson.get('elementIndex')
  76. if (isinstance(index, int) and isinstance(element_index, int) and
  77. index < element_index):
  78. chapter_number = element_index - index
  79. chapter = (chapters[chapter_number - 1]
  80. if chapter_number - 1 < len(chapters) else None)
  81. duration = None
  82. timestamp = lesson.get('timestamp')
  83. if isinstance(timestamp, compat_str):
  84. mobj = re.search(
  85. r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})',
  86. timestamp)
  87. if mobj:
  88. duration = parse_duration(mobj.group('end')) - parse_duration(
  89. mobj.group('start'))
  90. return {
  91. '_type': 'url_transparent',
  92. 'url': 'frontendmasters:%s' % lesson_id,
  93. 'ie_key': FrontendMastersIE.ie_key(),
  94. 'id': lesson_id,
  95. 'display_id': display_id,
  96. 'title': title,
  97. 'description': description,
  98. 'thumbnail': thumbnail,
  99. 'duration': duration,
  100. 'chapter': chapter,
  101. 'chapter_number': chapter_number,
  102. }
  103. class FrontendMastersIE(FrontendMastersBaseIE):
  104. _VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)'
  105. _TESTS = [{
  106. 'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba',
  107. 'md5': '7f161159710d6b7016a4f4af6fcb05e2',
  108. 'info_dict': {
  109. 'id': 'a2qogef6ba',
  110. 'ext': 'mp4',
  111. 'title': 'a2qogef6ba',
  112. },
  113. 'skip': 'Requires FrontendMasters account credentials',
  114. }, {
  115. 'url': 'frontendmasters:a2qogef6ba',
  116. 'only_matching': True,
  117. }]
  118. def _real_extract(self, url):
  119. lesson_id = self._match_id(url)
  120. source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id)
  121. formats = []
  122. for ext in ('webm', 'mp4'):
  123. for quality in ('low', 'mid', 'high'):
  124. resolution = self._QUALITIES[quality].copy()
  125. format_id = '%s-%s' % (ext, quality)
  126. format_url = self._download_json(
  127. source_url, lesson_id,
  128. 'Downloading %s source JSON' % format_id, query={
  129. 'f': ext,
  130. 'r': resolution['height'],
  131. }, headers={
  132. 'Referer': url,
  133. }, fatal=False)['url']
  134. if not format_url:
  135. continue
  136. f = resolution.copy()
  137. f.update({
  138. 'url': format_url,
  139. 'ext': ext,
  140. 'format_id': format_id,
  141. })
  142. formats.append(f)
  143. self._sort_formats(formats)
  144. subtitles = {
  145. 'en': [{
  146. 'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id),
  147. }]
  148. }
  149. return {
  150. 'id': lesson_id,
  151. 'title': lesson_id,
  152. 'formats': formats,
  153. 'subtitles': subtitles
  154. }
  155. class FrontendMastersLessonIE(FrontendMastersPageBaseIE):
  156. _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)'
  157. _TEST = {
  158. 'url': 'https://frontendmasters.com/courses/web-development/tools',
  159. 'info_dict': {
  160. 'id': 'a2qogef6ba',
  161. 'display_id': 'tools',
  162. 'ext': 'mp4',
  163. 'title': 'Tools',
  164. 'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
  165. 'thumbnail': r're:^https?://.*\.jpg$',
  166. 'chapter': 'Introduction',
  167. 'chapter_number': 1,
  168. },
  169. 'params': {
  170. 'skip_download': True,
  171. },
  172. 'skip': 'Requires FrontendMasters account credentials',
  173. }
  174. def _real_extract(self, url):
  175. mobj = re.match(self._VALID_URL, url)
  176. course_name, lesson_name = mobj.group('course_name', 'lesson_name')
  177. course = self._download_course(course_name, url)
  178. lesson_id, lesson = next(
  179. (video_id, data)
  180. for video_id, data in course['lessonData'].items()
  181. if data.get('slug') == lesson_name)
  182. chapters = self._extract_chapters(course)
  183. return self._extract_lesson(chapters, lesson_id, lesson)
  184. class FrontendMastersCourseIE(FrontendMastersPageBaseIE):
  185. _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)'
  186. _TEST = {
  187. 'url': 'https://frontendmasters.com/courses/web-development/',
  188. 'info_dict': {
  189. 'id': 'web-development',
  190. 'title': 'Introduction to Web Development',
  191. 'description': 'md5:9317e6e842098bf725d62360e52d49a6',
  192. },
  193. 'playlist_count': 81,
  194. 'skip': 'Requires FrontendMasters account credentials',
  195. }
  196. @classmethod
  197. def suitable(cls, url):
  198. return False if FrontendMastersLessonIE.suitable(url) else super(
  199. FrontendMastersBaseIE, cls).suitable(url)
  200. def _real_extract(self, url):
  201. course_name = self._match_id(url)
  202. course = self._download_course(course_name, url)
  203. chapters = self._extract_chapters(course)
  204. lessons = sorted(
  205. course['lessonData'].values(), key=lambda data: data['index'])
  206. entries = []
  207. for lesson in lessons:
  208. lesson_name = lesson.get('slug')
  209. if not lesson_name:
  210. continue
  211. lesson_id = lesson.get('hash') or lesson.get('statsId')
  212. entries.append(self._extract_lesson(chapters, lesson_id, lesson))
  213. title = course.get('title')
  214. description = course.get('description')
  215. return self.playlist_result(entries, course_name, title, description)