From adccf33632c51def397cdfb08c1271de6d6ec95e Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 2 Oct 2015 21:58:20 +0100 Subject: [PATCH 1/3] [ign] add support for pcmag and extract all formats and more metadata --- youtube_dl/extractor/__init__.py | 6 +- youtube_dl/extractor/ign.py | 117 +++++++++++++++++++++++-------- 2 files changed, 93 insertions(+), 30 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index a73a1317e..191661390 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -231,7 +231,11 @@ from .howstuffworks import HowStuffWorksIE from .huffpost import HuffPostIE from .hypem import HypemIE from .iconosquare import IconosquareIE -from .ign import IGNIE, OneUPIE +from .ign import ( + IGNIE, + OneUPIE, + PCMagIE, +) from .imdb import ( ImdbIE, ImdbListIE diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py index bf2d2041b..fa4e67394 100644 --- a/youtube_dl/extractor/ign.py +++ b/youtube_dl/extractor/ign.py @@ -3,6 +3,10 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from ..utils import ( + int_or_none, + parse_iso8601, +) class IGNIE(InfoExtractor): @@ -11,25 +15,23 @@ class IGNIE(InfoExtractor): Some videos of it.ign.com are also supported """ - _VALID_URL = r'https?://.+?\.ign\.com/(?Pvideos|show_videos|articles|(?:[^/]*/feature))(/.+)?/(?P.+)' + _VALID_URL = r'https?://.+?\.ign\.com/(?:[^/]+/)?(?Pvideos|show_videos|articles|feature|(?:[^/]+/\d+/video))(/.+)?/(?P.+)' IE_NAME = 'ign.com' - _CONFIG_URL_TEMPLATE = 'http://www.ign.com/videos/configs/id/%s.config' - _DESCRIPTION_RE = [ - r'(.+?)', - r'id="my_show_video">.*?

(.*?)

', - r']+?["\']((?:https?:)?//.+?\.ign\.com.+?/embed.+?)["\']' _TESTS = [ { 'url': 'http://www.ign.com/videos/2013/06/05/the-last-of-us-review', - 'md5': 'eac8bdc1890980122c3b66f14bdd02e9', + 'md5': 'febda82c4bafecd2d44b6e1a18a595f8', 'info_dict': { 'id': '8f862beef863986b2785559b9e1aa599', 'ext': 'mp4', 'title': 'The Last of Us Review', 'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c', + 'timestamp': 1370440800, + 'upload_date': '20130605', } }, { @@ -44,6 +46,8 @@ class IGNIE(InfoExtractor): 'ext': 'mp4', 'title': 'GTA 5 Video Review', 'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.', + 'timestamp': 1379339880, + 'upload_date': '20130916', }, }, { @@ -52,6 +56,8 @@ class IGNIE(InfoExtractor): 'ext': 'mp4', 'title': '26 Twisted Moments from GTA 5 in Slow Motion', 'description': 'The twisted beauty of GTA 5 in stunning slow motion.', + 'timestamp': 1386878820, + 'upload_date': '20131212', }, }, ], @@ -66,10 +72,9 @@ class IGNIE(InfoExtractor): 'id': '078fdd005f6d3c02f63d795faa1b984f', 'ext': 'mp4', 'title': 'Rewind Theater - Wild Trailer Gamescom 2014', - 'description': ( - 'Giant skeletons, bloody hunts, and captivating' - ' natural beauty take our breath away.' - ), + 'description': 'Brian and Jared explore Michel Ancel\'s captivating new preview.', + 'timestamp': 1408047180, + 'upload_date': '20140814', }, }, ] @@ -82,7 +87,7 @@ class IGNIE(InfoExtractor): r'video)/id/(?P.+)\.html' IE_NAME = '1up.com' - _DESCRIPTION_RE = r'
(.+?)
' - _TESTS = [{ 'url': 'http://gamevideos.1up.com/video/id/34976.html', - 'md5': '68a54ce4ebc772e4b71e3123d413163d', + 'md5': 'c9cc69e07acb675c31a16719f909e347', 'info_dict': { 'id': '34976', 'ext': 'mp4', 'title': 'Sniper Elite V2 - Trailer', - 'description': 'md5:5d289b722f5a6d940ca3136e9dae89cf', + 'description': 'md5:bf0516c5ee32a3217aa703e9b1bc7826', + 'timestamp': 1313099220, + 'upload_date': '20110811', } }] @@ -143,3 +171,34 @@ class OneUPIE(IGNIE): result = super(OneUPIE, self)._real_extract(url) result['id'] = mobj.group('name_or_id') return result + + +class PCMagIE(IGNIE): + _VALID_URL = r'https?://(?:www\.)?pcmag\.com/(?Pvideos|article2)(/.+)?/(?P.+)' + IE_NAME = 'pcmag' + + _EMBED_RE = r'iframe.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content.html?[^"]*url=([^"]+)["&]' + + _TESTS = [{ + 'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data', + 'md5': '212d6154fd0361a2781075f1febbe9ad', + 'info_dict': { + 'id': 'ee10d774b508c9b8ec07e763b9125b91', + 'ext': 'mp4', + 'title': '010615_What\'s New Now: Is GoGo Snooping on Your Data?', + 'description': 'md5:a7071ae64d2f68cc821c729d4ded6bb3', + 'timestamp': 1420571160, + 'upload_date': '20150106', + } + },{ + 'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp', + 'md5': '94130c1ca07ba0adb6088350681f16c1', + 'info_dict': { + 'id': '042e560ba94823d43afcb12ddf7142ca', + 'ext': 'mp4', + 'title': 'HTC\'s Weird New Re Camera - What\'s New Now', + 'description': 'md5:53433c45df96d2ea5d0fda18be2ca908', + 'timestamp': 1412953920, + 'upload_date': '20141010', + } + }] From db8e38b8cff2e67a9ff51104c4a7b33c20650204 Mon Sep 17 00:00:00 2001 From: remitamine Date: Wed, 14 Oct 2015 11:55:03 +0100 Subject: [PATCH 2/3] [ign] add tests for me.ign specific language urls --- youtube_dl/extractor/ign.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py index fa4e67394..fb2753738 100644 --- a/youtube_dl/extractor/ign.py +++ b/youtube_dl/extractor/ign.py @@ -77,6 +77,14 @@ class IGNIE(InfoExtractor): 'upload_date': '20140814', }, }, + { + 'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s', + 'only_matching': True, + }, + { + 'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds', + 'only_matching': True, + }, ] def _find_video_id(self, webpage): From e3e166d8cfa2be039b320d0c733b8233e95d3dcf Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 11 Dec 2015 00:08:54 +0100 Subject: [PATCH 3/3] [ign] improve extraction and extract uploader_id --- youtube_dl/extractor/ign.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py index fb2753738..6160f03d1 100644 --- a/youtube_dl/extractor/ign.py +++ b/youtube_dl/extractor/ign.py @@ -32,6 +32,7 @@ class IGNIE(InfoExtractor): 'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c', 'timestamp': 1370440800, 'upload_date': '20130605', + 'uploader_id': 'cberidon@ign.com', } }, { @@ -48,6 +49,7 @@ class IGNIE(InfoExtractor): 'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.', 'timestamp': 1379339880, 'upload_date': '20130916', + 'uploader_id': 'danieljkrupa@gmail.com', }, }, { @@ -58,6 +60,7 @@ class IGNIE(InfoExtractor): 'description': 'The twisted beauty of GTA 5 in stunning slow motion.', 'timestamp': 1386878820, 'upload_date': '20131212', + 'uploader_id': 'togilvie@ign.com', }, }, ], @@ -75,6 +78,7 @@ class IGNIE(InfoExtractor): 'description': 'Brian and Jared explore Michel Ancel\'s captivating new preview.', 'timestamp': 1408047180, 'upload_date': '20140814', + 'uploader_id': 'jamesduggan1990@gmail.com', }, }, { @@ -104,8 +108,8 @@ class IGNIE(InfoExtractor): webpage = self._download_webpage(url, name_or_id) if page_type != 'video': multiple_urls = re.findall( - ']*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]', - webpage) + r']*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]', + webpage) if multiple_urls: entries = [self.url_result(u, ie='IGN') for u in multiple_urls] return { @@ -125,10 +129,14 @@ class IGNIE(InfoExtractor): formats = [] m3u8_url = api_data['refs'].get('m3uUrl') if m3u8_url: - formats.extend(self._extract_m3u8_formats(m3u8_url, video_id)) + m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False) + if m3u8_formats: + formats.extend(m3u8_formats) f4m_url = api_data['refs'].get('f4mUrl') if f4m_url: - formats.extend(self._extract_f4m_formats(f4m_url, video_id)) + f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False) + if f4m_formats: + formats.extend(f4m_formats) for asset in api_data['assets']: formats.append({ 'url': asset['url'], @@ -139,9 +147,9 @@ class IGNIE(InfoExtractor): }) self._sort_formats(formats) - thumbnails = [] - for thumbnail in api_data['thumbnails']: - thumbnails.append({'url': thumbnail['url']}) + thumbnails = [{ + 'url': thumbnail['url'] + } for thumbnail in api_data.get('thumbnails', [])] metadata = api_data['metadata'] @@ -152,6 +160,7 @@ class IGNIE(InfoExtractor): 'timestamp': parse_iso8601(metadata.get('publishDate')), 'duration': int_or_none(metadata.get('duration')), 'display_id': metadata.get('slug') or video_id, + 'uploader_id': metadata.get('creator'), 'thumbnails': thumbnails, 'formats': formats, } @@ -171,6 +180,7 @@ class OneUPIE(IGNIE): 'description': 'md5:bf0516c5ee32a3217aa703e9b1bc7826', 'timestamp': 1313099220, 'upload_date': '20110811', + 'uploader_id': 'IGN', } }] @@ -197,6 +207,7 @@ class PCMagIE(IGNIE): 'description': 'md5:a7071ae64d2f68cc821c729d4ded6bb3', 'timestamp': 1420571160, 'upload_date': '20150106', + 'uploader_id': 'cozzipix@gmail.com', } },{ 'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp', @@ -208,5 +219,6 @@ class PCMagIE(IGNIE): 'description': 'md5:53433c45df96d2ea5d0fda18be2ca908', 'timestamp': 1412953920, 'upload_date': '20141010', + 'uploader_id': 'chris_snyder@pcmag.com', } }]