Philipp Hagemeister
10 years ago
11 changed files with 393 additions and 17 deletions
Split View
Diff Options
-
+4 -0youtube_dl/extractor/__init__.py
-
+1 -1youtube_dl/extractor/dfb.py
-
+7 -0youtube_dl/extractor/generic.py
-
+134 -0youtube_dl/extractor/howstuffworks.py
-
+80 -0youtube_dl/extractor/jove.py
-
+60 -0youtube_dl/extractor/mitele.py
-
+40 -13youtube_dl/extractor/pbs.py
-
+52 -0youtube_dl/extractor/rtlnl.py
-
+1 -1youtube_dl/extractor/teamcoco.py
-
+13 -2youtube_dl/extractor/yahoo.py
-
+1 -0youtube_dl/utils.py
+ 4
- 0
youtube_dl/extractor/__init__.py
View File
+ 1
- 1
youtube_dl/extractor/dfb.py
View File
+ 7
- 0
youtube_dl/extractor/generic.py
View File
+ 134
- 0
youtube_dl/extractor/howstuffworks.py
View File
| @ -0,0 +1,134 @@ | |||
| from __future__ import unicode_literals | |||
| import re | |||
| import json | |||
| import random | |||
| import string | |||
| from .common import InfoExtractor | |||
| from ..utils import find_xpath_attr | |||
| class HowStuffWorksIE(InfoExtractor): | |||
| _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm' | |||
| _TESTS = [ | |||
| { | |||
| 'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm', | |||
| 'info_dict': { | |||
| 'id': '450221', | |||
| 'display_id': 'cool-jobs-iditarod-musher', | |||
| 'ext': 'flv', | |||
| 'title': 'Cool Jobs - Iditarod Musher', | |||
| 'description': 'md5:82bb58438a88027b8186a1fccb365f90', | |||
| 'thumbnail': 're:^https?://.*\.jpg$', | |||
| }, | |||
| 'params': { | |||
| # md5 is not consistent | |||
| 'skip_download': True | |||
| } | |||
| }, | |||
| { | |||
| 'url': 'http://adventure.howstuffworks.com/39516-deadliest-catch-jakes-farewell-pots-video.htm', | |||
| 'info_dict': { | |||
| 'id': '553470', | |||
| 'display_id': 'deadliest-catch-jakes-farewell-pots', | |||
| 'ext': 'mp4', | |||
| 'title': 'Deadliest Catch: Jake\'s Farewell Pots', | |||
| 'description': 'md5:9632c346d5e43ee238028c9cefd8dbbc', | |||
| 'thumbnail': 're:^https?://.*\.jpg$', | |||
| }, | |||
| 'params': { | |||
| # md5 is not consistent | |||
| 'skip_download': True | |||
| } | |||
| }, | |||
| { | |||
| 'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm', | |||
| 'info_dict': { | |||
| 'id': '440011', | |||
| 'display_id': 'sword-swallowing-1-by-dan-meyer', | |||
| 'ext': 'flv', | |||
| 'title': 'Sword Swallowing #1 by Dan Meyer', | |||
| 'description': 'md5:b2409e88172913e2e7d3d1159b0ef735', | |||
| 'thumbnail': 're:^https?://.*\.jpg$', | |||
| }, | |||
| 'params': { | |||
| # md5 is not consistent | |||
| 'skip_download': True | |||
| } | |||
| }, | |||
| ] | |||
| def _real_extract(self, url): | |||
| mobj = re.match(self._VALID_URL, url) | |||
| display_id = mobj.group('id') | |||
| webpage = self._download_webpage(url, display_id) | |||
| content_id = self._search_regex(r'var siteSectionId="(\d+)";', webpage, 'content id') | |||
| mp4 = self._search_regex( | |||
| r'''(?xs)var\s+clip\s*=\s*{\s* | |||
| .+?\s* | |||
| content_id\s*:\s*%s\s*,\s* | |||
| .+?\s* | |||
| mp4\s*:\s*\[(.*?),?\]\s* | |||
| };\s* | |||
| videoData\.push\(clip\);''' % content_id, | |||
| webpage, 'mp4', fatal=False, default=None) | |||
| smil = self._download_xml( | |||
| 'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % content_id, | |||
| content_id, 'Downloading video SMIL') | |||
| http_base = find_xpath_attr( | |||
| smil, | |||
| './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'), | |||
| 'name', | |||
| 'httpBase').get('content') | |||
| def random_string(str_len=0): | |||
| return ''.join([random.choice(string.ascii_uppercase) for _ in range(str_len)]) | |||
| URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=%s&g=%s' % (random_string(5), random_string(12)) | |||
| formats = [] | |||
| if mp4: | |||
| for video in json.loads('[%s]' % mp4): | |||
| bitrate = video['bitrate'] | |||
| fmt = { | |||
| 'url': video['src'].replace('http://pmd.video.howstuffworks.com', http_base) + URL_SUFFIX, | |||
| 'format_id': bitrate, | |||
| } | |||
| m = re.search(r'(?P<vbr>\d+)[Kk]', bitrate) | |||
| if m: | |||
| fmt['vbr'] = int(m.group('vbr')) | |||
| formats.append(fmt) | |||
| else: | |||
| for video in smil.findall( | |||
| './/{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')): | |||
| vbr = int(video.attrib['system-bitrate']) / 1000 | |||
| formats.append({ | |||
| 'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX), | |||
| 'format_id': '%dk' % vbr, | |||
| 'vbr': vbr, | |||
| }) | |||
| self._sort_formats(formats) | |||
| title = self._og_search_title(webpage) | |||
| TITLE_SUFFIX = ' : HowStuffWorks' | |||
| if title.endswith(TITLE_SUFFIX): | |||
| title = title[:-len(TITLE_SUFFIX)] | |||
| description = self._og_search_description(webpage) | |||
| thumbnail = self._og_search_thumbnail(webpage) | |||
| return { | |||
| 'id': content_id, | |||
| 'display_id': display_id, | |||
| 'title': title, | |||
| 'description': description, | |||
| 'thumbnail': thumbnail, | |||
| 'formats': formats, | |||
| } | |||
+ 80
- 0
youtube_dl/extractor/jove.py
View File
| @ -0,0 +1,80 @@ | |||
| from __future__ import unicode_literals | |||
| import re | |||
| from .common import InfoExtractor | |||
| from ..utils import ( | |||
| ExtractorError, | |||
| unified_strdate | |||
| ) | |||
| class JoveIE(InfoExtractor): | |||
| _VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P<id>[0-9]+)' | |||
| _CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}' | |||
| _TESTS = [ | |||
| { | |||
| 'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current', | |||
| 'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b', | |||
| 'info_dict': { | |||
| 'id': '2744', | |||
| 'ext': 'mp4', | |||
| 'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation', | |||
| 'description': 'md5:015dd4509649c0908bc27f049e0262c6', | |||
| 'thumbnail': 're:^https?://.*\.png$', | |||
| 'upload_date': '20110523', | |||
| } | |||
| }, | |||
| { | |||
| 'url': 'http://www.jove.com/video/51796/culturing-caenorhabditis-elegans-axenic-liquid-media-creation', | |||
| 'md5': '914aeb356f416811d911996434811beb', | |||
| 'info_dict': { | |||
| 'id': '51796', | |||
| 'ext': 'mp4', | |||
| 'title': 'Culturing Caenorhabditis elegans in Axenic Liquid Media and Creation of Transgenic Worms by Microparticle Bombardment', | |||
| 'description': 'md5:35ff029261900583970c4023b70f1dc9', | |||
| 'thumbnail': 're:^https?://.*\.png$', | |||
| 'upload_date': '20140802', | |||
| } | |||
| }, | |||
| ] | |||
| def _real_extract(self, url): | |||
| mobj = re.match(self._VALID_URL, url) | |||
| video_id = mobj.group('id') | |||
| webpage = self._download_webpage(url, video_id) | |||
| chapters_id = self._html_search_regex( | |||
| r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id') | |||
| chapters_xml = self._download_xml( | |||
| self._CHAPTERS_URL.format(video_id=chapters_id), | |||
| video_id, note='Downloading chapters XML', | |||
| errnote='Failed to download chapters XML') | |||
| video_url = chapters_xml.attrib.get('video') | |||
| if not video_url: | |||
| raise ExtractorError('Failed to get the video URL') | |||
| title = self._html_search_meta('citation_title', webpage, 'title') | |||
| thumbnail = self._og_search_thumbnail(webpage) | |||
| description = self._html_search_regex( | |||
| r'<div id="section_body_summary"><p class="jove_content">(.+?)</p>', | |||
| webpage, 'description', fatal=False) | |||
| publish_date = unified_strdate(self._html_search_meta( | |||
| 'citation_publication_date', webpage, 'publish date', fatal=False)) | |||
| comment_count = self._html_search_regex( | |||
| r'<meta name="num_comments" content="(\d+) Comments?"', | |||
| webpage, 'comment count', fatal=False) | |||
| return { | |||
| 'id': video_id, | |||
| 'title': title, | |||
| 'url': video_url, | |||
| 'thumbnail': thumbnail, | |||
| 'description': description, | |||
| 'upload_date': publish_date, | |||
| 'comment_count': comment_count, | |||
| } | |||
+ 60
- 0
youtube_dl/extractor/mitele.py
View File
| @ -0,0 +1,60 @@ | |||
| from __future__ import unicode_literals | |||
| import re | |||
| import json | |||
| from .common import InfoExtractor | |||
| from ..utils import ( | |||
| compat_urllib_parse, | |||
| get_element_by_attribute, | |||
| parse_duration, | |||
| strip_jsonp, | |||
| ) | |||
| class MiTeleIE(InfoExtractor): | |||
| IE_NAME = 'mitele.es' | |||
| _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<episode>[^/]+)/' | |||
| _TEST = { | |||
| 'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/', | |||
| 'md5': '6a75fe9d0d3275bead0cb683c616fddb', | |||
| 'info_dict': { | |||
| 'id': '0fce117d', | |||
| 'ext': 'mp4', | |||
| 'title': 'Programa 144 - Tor, la web invisible', | |||
| 'description': 'md5:3b6fce7eaa41b2d97358726378d9369f', | |||
| 'display_id': 'programa-144', | |||
| 'duration': 2913, | |||
| }, | |||
| } | |||
| def _real_extract(self, url): | |||
| mobj = re.match(self._VALID_URL, url) | |||
| episode = mobj.group('episode') | |||
| webpage = self._download_webpage(url, episode) | |||
| embed_data_json = self._search_regex( | |||
| r'MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data', | |||
| flags=re.DOTALL | |||
| ).replace('\'', '"') | |||
| embed_data = json.loads(embed_data_json) | |||
| info_url = embed_data['flashvars']['host'] | |||
| info_el = self._download_xml(info_url, episode).find('./video/info') | |||
| video_link = info_el.find('videoUrl/link').text | |||
| token_query = compat_urllib_parse.urlencode({'id': video_link}) | |||
| token_info = self._download_json( | |||
| 'http://token.mitele.es/?' + token_query, episode, | |||
| transform_source=strip_jsonp | |||
| ) | |||
| return { | |||
| 'id': embed_data['videoId'], | |||
| 'display_id': episode, | |||
| 'title': info_el.find('title').text, | |||
| 'url': token_info['tokenizedUrl'], | |||
| 'description': get_element_by_attribute('class', 'text', webpage), | |||
| 'thumbnail': info_el.find('thumb').text, | |||
| 'duration': parse_duration(info_el.find('duration').text), | |||
| } | |||
+ 40
- 13
youtube_dl/extractor/pbs.py
View File
+ 52
- 0
youtube_dl/extractor/rtlnl.py
View File
| @ -0,0 +1,52 @@ | |||
| from __future__ import unicode_literals | |||
| import re | |||
| from .common import InfoExtractor | |||
| class RtlXlIE(InfoExtractor): | |||
| IE_NAME = 'rtlxl.nl' | |||
| _VALID_URL = r'https?://www\.rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)' | |||
| _TEST = { | |||
| 'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677', | |||
| 'info_dict': { | |||
| 'id': '6e4203a6-0a5e-3596-8424-c599a59e0677', | |||
| 'ext': 'flv', | |||
| 'title': 'RTL Nieuws - Laat', | |||
| 'description': 'Dagelijks het laatste nieuws uit binnen- en ' | |||
| 'buitenland. Voor nog meer nieuws kunt u ook gebruikmaken van ' | |||
| 'onze mobiele apps.', | |||
| 'timestamp': 1408051800, | |||
| 'upload_date': '20140814', | |||
| }, | |||
| 'params': { | |||
| # We download the first bytes of the first fragment, it can't be | |||
| # processed by the f4m downloader beacuse it isn't complete | |||
| 'skip_download': True, | |||
| }, | |||
| } | |||
| def _real_extract(self, url): | |||
| mobj = re.match(self._VALID_URL, url) | |||
| uuid = mobj.group('uuid') | |||
| info = self._download_json( | |||
| 'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid, | |||
| uuid) | |||
| meta = info['meta'] | |||
| material = info['material'][0] | |||
| episode_info = info['episodes'][0] | |||
| f4m_url = 'http://manifest.us.rtl.nl' + material['videopath'] | |||
| progname = info['abstracts'][0]['name'] | |||
| subtitle = material['title'] or info['episodes'][0]['name'] | |||
| return { | |||
| 'id': uuid, | |||
| 'title': '%s - %s' % (progname, subtitle), | |||
| 'formats': self._extract_f4m_formats(f4m_url, uuid), | |||
| 'timestamp': material['original_date'], | |||
| 'description': episode_info['synopsis'], | |||
| } | |||