@ -1,271 +0,0 @@ | |||||
# coding: utf-8 | |||||
from __future__ import unicode_literals | |||||
import collections | |||||
import re | |||||
from .common import InfoExtractor | |||||
from ..compat import ( | |||||
compat_urlparse) | |||||
from ..utils import ( | |||||
ExtractorError, | |||||
urlencode_postdata, | |||||
qualities, unescapeHTML) | |||||
class FrontEndMasterBaseIE(InfoExtractor): | |||||
_API_BASE = 'https://api.frontendmasters.com/v1/kabuki/courses' | |||||
_VIDEO_BASE = 'http://www.frontendmasters.com/courses' | |||||
_CAPTIONS_BASE = 'https://api.frontendmasters.com/v1/kabuki/transcripts' | |||||
_COOKIES_BASE = 'https://api.frontendmasters.com' | |||||
_LOGIN_URL = 'https://frontendmasters.com/login/' | |||||
_QUALITIES_PREFERENCE = ('low', 'medium', 'high') | |||||
_QUALITIES = { | |||||
'low': {'width': 480, 'height': 360}, | |||||
'medium': {'width': 1280, 'height': 720}, | |||||
'high': {'width': 1920, 'height': 1080} | |||||
} | |||||
AllowedQuality = collections.namedtuple('AllowedQuality', | |||||
['ext', 'qualities']) | |||||
_ALLOWED_QUALITIES = [ | |||||
AllowedQuality('webm', ['low', 'medium', 'high']), | |||||
AllowedQuality('mp4', ['low', 'medium', 'high']) | |||||
] | |||||
def _real_initialize(self): | |||||
self._login() | |||||
def _login(self): | |||||
(username, password) = self._get_login_info() | |||||
if username is None: | |||||
return | |||||
login_page = self._download_webpage( | |||||
self._LOGIN_URL, None, 'Downloading login page') | |||||
login_form = self._hidden_inputs(login_page) | |||||
login_form.update({ | |||||
'username': username, | |||||
'password': password | |||||
}) | |||||
post_url = self._search_regex( | |||||
r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, | |||||
'post_url', default=self._LOGIN_URL, group='url') | |||||
if not post_url.startswith('http'): | |||||
post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url) | |||||
response = self._download_webpage( | |||||
post_url, None, 'Logging in', | |||||
data=urlencode_postdata(login_form), | |||||
headers={'Content-Type': 'application/x-www-form-urlencoded'} | |||||
) | |||||
error = self._search_regex( | |||||
r'<div[^>]+class=["\']Message MessageAlert["\'][^>]*>' | |||||
r'([^<]+)' | |||||
r'</div>', | |||||
response, 'error message', default=None) | |||||
if error: | |||||
raise ExtractorError('Unable to login: %s' % unescapeHTML(error), | |||||
expected=True) | |||||
def _download_course(self, course_id, url): | |||||
response = self._download_json( | |||||
'%s/%s' % (self._API_BASE, course_id), course_id, | |||||
'Downloading course JSON', | |||||
headers={ | |||||
'Content-Type': 'application/json;charset=utf-8', | |||||
'Referer': url, | |||||
}) | |||||
return response | |||||
@staticmethod | |||||
def _pair_section_video_element(lesson_elements): | |||||
sections = {} | |||||
current_section = None | |||||
current_section_number = 0 | |||||
for elem in lesson_elements: | |||||
if not isinstance(elem, int): | |||||
elem_name = elem | |||||
if not isinstance(elem_name, str): | |||||
# convert unicode to str | |||||
elem_name = elem.encode('utf-8') | |||||
(current_section, current_section_number) = \ | |||||
(elem_name, current_section_number + 1) | |||||
else: | |||||
if current_section: | |||||
sections[elem] = (current_section, current_section_number) | |||||
return sections | |||||
class FrontEndMasterIE(FrontEndMasterBaseIE): | |||||
IE_NAME = 'frontend-masters' | |||||
_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/' \ | |||||
r'(?P<courseid>[a-z\-]+)/' \ | |||||
r'(?P<id>[a-z\-]+)' | |||||
_NETRC_MACHINE = 'frontendmasters' | |||||
_TEST = { | |||||
'url': 'https://frontendmasters.com/courses/web-development/tools', | |||||
'md5': '7f161159710d6b7016a4f4af6fcb05e2', | |||||
'info_dict': { | |||||
'id': 'tools', | |||||
'title': 'Tools', | |||||
'display_id': 'tools', | |||||
'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7', | |||||
'ext': 'mp4' | |||||
}, | |||||
'skip': 'Requires FrontendMasters account credentials', | |||||
} | |||||
def _get_subtitles(self, video_hash, video_id): | |||||
captions = self._download_webpage( | |||||
'%s/%s.vtt' % (self._CAPTIONS_BASE, video_hash), video_id, | |||||
fatal=False) | |||||
if captions: | |||||
return { | |||||
'en': [{ | |||||
'ext': 'vtt', | |||||
'data': captions | |||||
}] | |||||
} | |||||
def _real_extract(self, url): | |||||
mobj = re.match(self._VALID_URL, url) | |||||
video_id = mobj.group('id') | |||||
course_id = mobj.group('courseid') | |||||
course_json_content = self._download_course(course_id=course_id, | |||||
url=url) | |||||
# Necessary to get mandatory informations like title and video_url | |||||
lesson_index = course_json_content.get('lessonSlugs').index(video_id) | |||||
lesson_hash = course_json_content.get('lessonHashes')[lesson_index] | |||||
lesson_data = course_json_content.get('lessonData')[lesson_hash] | |||||
# This is necessary to get the link for the video | |||||
lesson_source_base = lesson_data['sourceBase'] | |||||
lesson_title = lesson_data['title'] | |||||
# Some optional fields | |||||
lesson_description = lesson_data.get('description') | |||||
lesson_index = lesson_data.get('index') | |||||
lesson_slug = lesson_data.get('slug') | |||||
lesson_thumbnail_url = lesson_data.get('thumbnail') | |||||
lesson_section_elements = course_json_content.get('lessonElements') | |||||
try: | |||||
course_sections_pairing = self._pair_section_video_element( | |||||
lesson_section_elements) | |||||
lesson_section = \ | |||||
course_sections_pairing.get(lesson_index)[0] | |||||
lesson_section_number = \ | |||||
course_sections_pairing.get(lesson_index)[1] | |||||
except Exception: | |||||
lesson_section = None | |||||
lesson_section_number = None | |||||
video_request_url = '%s/source' | |||||
video_request_headers = { | |||||
'origin': 'https://frontendmasters.com', | |||||
'referer': lesson_source_base, | |||||
} | |||||
quality_key = qualities(self._QUALITIES_PREFERENCE) | |||||
formats = [] | |||||
for ext, qualities_ in self._ALLOWED_QUALITIES: | |||||
for quality in qualities_: | |||||
f = self._QUALITIES[quality].copy() | |||||
video_request_params = { | |||||
'r': f['height'], | |||||
'f': ext | |||||
} | |||||
video_response = self._download_json( | |||||
video_request_url % lesson_source_base, video_id, | |||||
query=video_request_params, headers=video_request_headers) | |||||
video_url = video_response.get('url') | |||||
clip_f = f.copy() | |||||
clip_f.update({ | |||||
'url': video_url, | |||||
'ext': ext, | |||||
'format_id': '%s-%s' % (ext, quality), | |||||
'quality': quality_key(quality), | |||||
'height': f['height'] | |||||
}) | |||||
formats.append(clip_f) | |||||
self._sort_formats(formats) | |||||
subtitles = self.extract_subtitles(lesson_hash, video_id) | |||||
return { | |||||
'id': video_id, | |||||
'display_id': lesson_slug, | |||||
'title': lesson_title, | |||||
'description': lesson_description, | |||||
'chapter': lesson_section, | |||||
'chapter_number': lesson_section_number, | |||||
'thumbnail': lesson_thumbnail_url, | |||||
'formats': formats, | |||||
'subtitles': subtitles | |||||
} | |||||
class FrontEndMasterCourseIE(FrontEndMasterBaseIE): | |||||
IE_NAME = 'frontend-masters:course' | |||||
_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<courseid>[a-z\-]+)/?$' | |||||
_NETRC_MACHINE = 'frontendmasters' | |||||
_TEST = { | |||||
'url': 'https://frontendmasters.com/courses/javascript-basics/', | |||||
'info_dict': { | |||||
'id': 'javascript-basics', | |||||
'title': 'Introduction to JavaScript Programming', | |||||
'description': 'md5:269412fbb76d86954761599ad8e4cbc9' | |||||
}, | |||||
'playlist_count': 19, | |||||
'skip': 'Requires FrontendMasters account credentials' | |||||
} | |||||
@classmethod | |||||
def suitable(cls, url): | |||||
return False if FrontEndMasterIE.suitable(url) else super(FrontEndMasterBaseIE, cls).suitable(url) | |||||
def _real_extract(self, url): | |||||
mobj = re.match(self._VALID_URL, url) | |||||
course_id = mobj.group('courseid') | |||||
course_json_content = self._download_course(course_id=course_id, | |||||
url=url) | |||||
title = course_json_content.get('title') | |||||
description = course_json_content.get('description') | |||||
course_display_id = course_json_content.get('slug') | |||||
videos_data = course_json_content.get('lessonData').values() | |||||
videos_data = sorted(videos_data, key=lambda video: video.get('index')) | |||||
entries = [] | |||||
for video in videos_data: | |||||
video_slug = video.get('slug') | |||||
clip_url = '%s/%s/%s' % ( | |||||
self._VIDEO_BASE, course_display_id, video_slug) | |||||
entries.append({ | |||||
'_type': 'url_transparent', | |||||
'url': clip_url, | |||||
'ie_key': FrontEndMasterIE.ie_key() | |||||
}) | |||||
return self.playlist_result(entries, course_id, title, description) |
@ -0,0 +1,262 @@ | |||||
# coding: utf-8 | |||||
from __future__ import unicode_literals | |||||
import re | |||||
from .common import InfoExtractor | |||||
from ..compat import ( | |||||
compat_str, | |||||
compat_urlparse, | |||||
) | |||||
from ..utils import ( | |||||
ExtractorError, | |||||
parse_duration, | |||||
urlencode_postdata, | |||||
) | |||||
class FrontendMastersBaseIE(InfoExtractor): | |||||
_API_BASE = 'https://api.frontendmasters.com/v1/kabuki' | |||||
_LOGIN_URL = 'https://frontendmasters.com/login/' | |||||
_NETRC_MACHINE = 'frontendmasters' | |||||
_QUALITIES = { | |||||
'low': {'width': 480, 'height': 360}, | |||||
'mid': {'width': 1280, 'height': 720}, | |||||
'high': {'width': 1920, 'height': 1080} | |||||
} | |||||
def _real_initialize(self): | |||||
self._login() | |||||
def _login(self): | |||||
(username, password) = self._get_login_info() | |||||
if username is None: | |||||
return | |||||
login_page = self._download_webpage( | |||||
self._LOGIN_URL, None, 'Downloading login page') | |||||
login_form = self._hidden_inputs(login_page) | |||||
login_form.update({ | |||||
'username': username, | |||||
'password': password | |||||
}) | |||||
post_url = self._search_regex( | |||||
r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, | |||||
'post_url', default=self._LOGIN_URL, group='url') | |||||
if not post_url.startswith('http'): | |||||
post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url) | |||||
response = self._download_webpage( | |||||
post_url, None, 'Logging in', data=urlencode_postdata(login_form), | |||||
headers={'Content-Type': 'application/x-www-form-urlencoded'}) | |||||
# Successful login | |||||
if any(p in response for p in ( | |||||
'wp-login.php?action=logout', '>Logout')): | |||||
return | |||||
error = self._html_search_regex( | |||||
r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<', | |||||
response, 'error message', default=None, group='error') | |||||
if error: | |||||
raise ExtractorError('Unable to login: %s' % error, expected=True) | |||||
raise ExtractorError('Unable to log in') | |||||
class FrontendMastersPageBaseIE(FrontendMastersBaseIE): | |||||
def _download_course(self, course_name, url): | |||||
return self._download_json( | |||||
'%s/courses/%s' % (self._API_BASE, course_name), course_name, | |||||
'Downloading course JSON', headers={'Referer': url}) | |||||
@staticmethod | |||||
def _extract_chapters(course): | |||||
chapters = [] | |||||
lesson_elements = course.get('lessonElements') | |||||
if isinstance(lesson_elements, list): | |||||
chapters = [e for e in lesson_elements if isinstance(e, compat_str)] | |||||
return chapters | |||||
@staticmethod | |||||
def _extract_lesson(chapters, lesson_id, lesson): | |||||
title = lesson.get('title') or lesson_id | |||||
display_id = lesson.get('slug') | |||||
description = lesson.get('description') | |||||
thumbnail = lesson.get('thumbnail') | |||||
chapter_number = None | |||||
index = lesson.get('index') | |||||
element_index = lesson.get('elementIndex') | |||||
if (isinstance(index, int) and isinstance(element_index, int) and | |||||
index < element_index): | |||||
chapter_number = element_index - index | |||||
chapter = (chapters[chapter_number - 1] | |||||
if chapter_number - 1 < len(chapters) else None) | |||||
duration = None | |||||
timestamp = lesson.get('timestamp') | |||||
if isinstance(timestamp, compat_str): | |||||
mobj = re.search( | |||||
r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})', | |||||
timestamp) | |||||
if mobj: | |||||
duration = parse_duration(mobj.group('end')) - parse_duration( | |||||
mobj.group('start')) | |||||
return { | |||||
'_type': 'url_transparent', | |||||
'url': 'frontendmasters:%s' % lesson_id, | |||||
'ie_key': FrontendMastersIE.ie_key(), | |||||
'id': lesson_id, | |||||
'display_id': display_id, | |||||
'title': title, | |||||
'description': description, | |||||
'thumbnail': thumbnail, | |||||
'duration': duration, | |||||
'chapter': chapter, | |||||
'chapter_number': chapter_number, | |||||
} | |||||
class FrontendMastersIE(FrontendMastersBaseIE): | |||||
_VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)' | |||||
_TESTS = [{ | |||||
'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba', | |||||
'md5': '7f161159710d6b7016a4f4af6fcb05e2', | |||||
'info_dict': { | |||||
'id': 'a2qogef6ba', | |||||
'ext': 'mp4', | |||||
'title': 'a2qogef6ba', | |||||
}, | |||||
'skip': 'Requires FrontendMasters account credentials', | |||||
}, { | |||||
'url': 'frontendmasters:a2qogef6ba', | |||||
'only_matching': True, | |||||
}] | |||||
def _real_extract(self, url): | |||||
lesson_id = self._match_id(url) | |||||
source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id) | |||||
formats = [] | |||||
for ext in ('webm', 'mp4'): | |||||
for quality in ('low', 'mid', 'high'): | |||||
resolution = self._QUALITIES[quality].copy() | |||||
format_id = '%s-%s' % (ext, quality) | |||||
format_url = self._download_json( | |||||
source_url, lesson_id, | |||||
'Downloading %s source JSON' % format_id, query={ | |||||
'f': ext, | |||||
'r': resolution['height'], | |||||
}, headers={ | |||||
'Referer': url, | |||||
}, fatal=False)['url'] | |||||
if not format_url: | |||||
continue | |||||
f = resolution.copy() | |||||
f.update({ | |||||
'url': format_url, | |||||
'ext': ext, | |||||
'format_id': format_id, | |||||
}) | |||||
formats.append(f) | |||||
self._sort_formats(formats) | |||||
subtitles = { | |||||
'en': [{ | |||||
'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id), | |||||
}] | |||||
} | |||||
return { | |||||
'id': lesson_id, | |||||
'title': lesson_id, | |||||
'formats': formats, | |||||
'subtitles': subtitles | |||||
} | |||||
class FrontendMastersLessonIE(FrontendMastersPageBaseIE): | |||||
_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)' | |||||
_TEST = { | |||||
'url': 'https://frontendmasters.com/courses/web-development/tools', | |||||
'info_dict': { | |||||
'id': 'a2qogef6ba', | |||||
'display_id': 'tools', | |||||
'ext': 'mp4', | |||||
'title': 'Tools', | |||||
'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7', | |||||
'thumbnail': r're:^https?://.*\.jpg$', | |||||
'chapter': 'Introduction', | |||||
'chapter_number': 1, | |||||
}, | |||||
'params': { | |||||
'skip_download': True, | |||||
}, | |||||
'skip': 'Requires FrontendMasters account credentials', | |||||
} | |||||
def _real_extract(self, url): | |||||
mobj = re.match(self._VALID_URL, url) | |||||
course_name, lesson_name = mobj.group('course_name', 'lesson_name') | |||||
course = self._download_course(course_name, url) | |||||
lesson_id, lesson = next( | |||||
(video_id, data) | |||||
for video_id, data in course['lessonData'].items() | |||||
if data.get('slug') == lesson_name) | |||||
chapters = self._extract_chapters(course) | |||||
return self._extract_lesson(chapters, lesson_id, lesson) | |||||
class FrontendMastersCourseIE(FrontendMastersPageBaseIE): | |||||
_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)' | |||||
_TEST = { | |||||
'url': 'https://frontendmasters.com/courses/web-development/', | |||||
'info_dict': { | |||||
'id': 'web-development', | |||||
'title': 'Introduction to Web Development', | |||||
'description': 'md5:9317e6e842098bf725d62360e52d49a6', | |||||
}, | |||||
'playlist_count': 81, | |||||
'skip': 'Requires FrontendMasters account credentials', | |||||
} | |||||
@classmethod | |||||
def suitable(cls, url): | |||||
return False if FrontendMastersLessonIE.suitable(url) else super( | |||||
FrontendMastersBaseIE, cls).suitable(url) | |||||
def _real_extract(self, url): | |||||
course_name = self._match_id(url) | |||||
course = self._download_course(course_name, url) | |||||
chapters = self._extract_chapters(course) | |||||
lessons = sorted( | |||||
course['lessonData'].values(), key=lambda data: data['index']) | |||||
entries = [] | |||||
for lesson in lessons: | |||||
lesson_name = lesson.get('slug') | |||||
if not lesson_name: | |||||
continue | |||||
lesson_id = lesson.get('hash') or lesson.get('statsId') | |||||
entries.append(self._extract_lesson(chapters, lesson_id, lesson)) | |||||
title = course.get('title') | |||||
description = course.get('description') | |||||
return self.playlist_result(entries, course_name, title, description) |