[frontendmasters] Add extractor

youtube_dl/extractor/extractors.py

@@ -390,6 +390,10 @@ from .francetv import (
 from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
 from .freshlive import FreshLiveIE
+from .frontendmaster import (
+    FrontEndMasterIE,
+    FrontEndMasterCourseIE
+)
 from .funimation import FunimationIE
 from .funk import (
     FunkMixIE,

youtube_dl/extractor/frontendmaster.py

@@ -0,0 +1,271 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import collections
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse)
+from ..utils import (
+    ExtractorError,
+    urlencode_postdata,
+    qualities, unescapeHTML)
+
+
+class FrontEndMasterBaseIE(InfoExtractor):
+    _API_BASE = 'https://api.frontendmasters.com/v1/kabuki/courses'
+    _VIDEO_BASE = 'http://www.frontendmasters.com/courses'
+    _CAPTIONS_BASE = 'https://api.frontendmasters.com/v1/kabuki/transcripts'
+    _COOKIES_BASE = 'https://api.frontendmasters.com'
+    _LOGIN_URL = 'https://frontendmasters.com/login/'
+
+    _QUALITIES_PREFERENCE = ('low', 'medium', 'high')
+    _QUALITIES = {
+        'low': {'width': 480, 'height': 360},
+        'medium': {'width': 1280, 'height': 720},
+        'high': {'width': 1920, 'height': 1080}
+    }
+
+    AllowedQuality = collections.namedtuple('AllowedQuality',
+                                            ['ext', 'qualities'])
+    _ALLOWED_QUALITIES = [
+        AllowedQuality('webm', ['low', 'medium', 'high']),
+        AllowedQuality('mp4', ['low', 'medium', 'high'])
+    ]
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'username': username,
+            'password': password
+        })
+
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post_url', default=self._LOGIN_URL, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
+        response = self._download_webpage(
+            post_url, None, 'Logging in',
+            data=urlencode_postdata(login_form),
+            headers={'Content-Type': 'application/x-www-form-urlencoded'}
+        )
+
+        error = self._search_regex(
+            r'<div[^>]+class=["\']Message MessageAlert["\'][^>]*>'
+            r'([^<]+)'
+            r'</div>',
+            response, 'error message', default=None)
+
+        if error:
+            raise ExtractorError('Unable to login: %s' % unescapeHTML(error),
+                                 expected=True)
+
+    def _download_course(self, course_id, url):
+        response = self._download_json(
+            '%s/%s' % (self._API_BASE, course_id), course_id,
+            'Downloading course JSON',
+            headers={
+                'Content-Type': 'application/json;charset=utf-8',
+                'Referer': url,
+            })
+        return response
+
+    @staticmethod
+    def _pair_section_video_element(lesson_elements):
+        sections = {}
+        current_section = None
+        current_section_number = 0
+        for elem in lesson_elements:
+            if not isinstance(elem, int):
+                elem_name = elem
+                if not isinstance(elem_name, str):
+                    # convert unicode to str
+                    elem_name = elem.encode('utf-8')
+                (current_section, current_section_number) = \
+                    (elem_name, current_section_number + 1)
+            else:
+                if current_section:
+                    sections[elem] = (current_section, current_section_number)
+
+        return sections
+
+
+class FrontEndMasterIE(FrontEndMasterBaseIE):
+    IE_NAME = 'frontend-masters'
+    _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/' \
+                 r'(?P<courseid>[a-z\-]+)/' \
+                 r'(?P<id>[a-z\-]+)'
+
+    _NETRC_MACHINE = 'frontendmasters'
+
+    _TEST = {
+        'url': 'https://frontendmasters.com/courses/web-development/tools',
+        'md5': '7f161159710d6b7016a4f4af6fcb05e2',
+        'info_dict': {
+            'id': 'tools',
+            'title': 'Tools',
+            'display_id': 'tools',
+            'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
+            'ext': 'mp4'
+        },
+        'skip': 'Requires FrontendMasters account credentials',
+    }
+
+    def _get_subtitles(self, video_hash, video_id):
+        captions = self._download_webpage(
+            '%s/%s.vtt' % (self._CAPTIONS_BASE, video_hash), video_id,
+            fatal=False)
+        if captions:
+            return {
+                'en': [{
+                    'ext': 'vtt',
+                    'data': captions
+                }]
+            }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        course_id = mobj.group('courseid')
+
+        course_json_content = self._download_course(course_id=course_id,
+                                                    url=url)
+
+        # Necessary to get mandatory informations like title and video_url
+        lesson_index = course_json_content.get('lessonSlugs').index(video_id)
+        lesson_hash = course_json_content.get('lessonHashes')[lesson_index]
+        lesson_data = course_json_content.get('lessonData')[lesson_hash]
+        # This is necessary to get the link for the video
+        lesson_source_base = lesson_data['sourceBase']
+
+        lesson_title = lesson_data['title']
+
+        # Some optional fields
+        lesson_description = lesson_data.get('description')
+        lesson_index = lesson_data.get('index')
+        lesson_slug = lesson_data.get('slug')
+        lesson_thumbnail_url = lesson_data.get('thumbnail')
+        lesson_section_elements = course_json_content.get('lessonElements')
+
+        try:
+            course_sections_pairing = self._pair_section_video_element(
+                lesson_section_elements)
+
+            lesson_section = \
+                course_sections_pairing.get(lesson_index)[0]
+
+            lesson_section_number = \
+                course_sections_pairing.get(lesson_index)[1]
+        except Exception:
+            lesson_section = None
+            lesson_section_number = None
+
+        video_request_url = '%s/source'
+        video_request_headers = {
+            'origin': 'https://frontendmasters.com',
+            'referer': lesson_source_base,
+        }
+
+        quality_key = qualities(self._QUALITIES_PREFERENCE)
+
+        formats = []
+        for ext, qualities_ in self._ALLOWED_QUALITIES:
+            for quality in qualities_:
+                f = self._QUALITIES[quality].copy()
+                video_request_params = {
+                    'r': f['height'],
+                    'f': ext
+                }
+                video_response = self._download_json(
+                    video_request_url % lesson_source_base, video_id,
+                    query=video_request_params, headers=video_request_headers)
+
+                video_url = video_response.get('url')
+                clip_f = f.copy()
+                clip_f.update({
+                    'url': video_url,
+                    'ext': ext,
+                    'format_id': '%s-%s' % (ext, quality),
+                    'quality': quality_key(quality),
+                    'height': f['height']
+                })
+                formats.append(clip_f)
+
+        self._sort_formats(formats)
+
+        subtitles = self.extract_subtitles(lesson_hash, video_id)
+
+        return {
+            'id': video_id,
+            'display_id': lesson_slug,
+            'title': lesson_title,
+            'description': lesson_description,
+            'chapter': lesson_section,
+            'chapter_number': lesson_section_number,
+            'thumbnail': lesson_thumbnail_url,
+            'formats': formats,
+            'subtitles': subtitles
+        }
+
+
+class FrontEndMasterCourseIE(FrontEndMasterBaseIE):
+    IE_NAME = 'frontend-masters:course'
+    _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<courseid>[a-z\-]+)/?$'
+
+    _NETRC_MACHINE = 'frontendmasters'
+
+    _TEST = {
+        'url': 'https://frontendmasters.com/courses/javascript-basics/',
+        'info_dict': {
+            'id': 'javascript-basics',
+            'title': 'Introduction to JavaScript Programming',
+            'description': 'md5:269412fbb76d86954761599ad8e4cbc9'
+        },
+        'playlist_count': 19,
+        'skip': 'Requires FrontendMasters account credentials'
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if FrontEndMasterIE.suitable(url) else super(FrontEndMasterBaseIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        course_id = mobj.group('courseid')
+        course_json_content = self._download_course(course_id=course_id,
+                                                    url=url)
+
+        title = course_json_content.get('title')
+        description = course_json_content.get('description')
+        course_display_id = course_json_content.get('slug')
+
+        videos_data = course_json_content.get('lessonData').values()
+        videos_data = sorted(videos_data, key=lambda video: video.get('index'))
+
+        entries = []
+        for video in videos_data:
+            video_slug = video.get('slug')
+            clip_url = '%s/%s/%s' % (
+                self._VIDEO_BASE, course_display_id, video_slug)
+            entries.append({
+                '_type': 'url_transparent',
+                'url': clip_url,
+                'ie_key': FrontEndMasterIE.ie_key()
+            })
+
+        return self.playlist_result(entries, course_id, title, description)