|
@ -2,158 +2,141 @@ |
|
|
from __future__ import unicode_literals |
|
|
from __future__ import unicode_literals |
|
|
|
|
|
|
|
|
import base64 |
|
|
import base64 |
|
|
|
|
|
|
|
|
from .common import InfoExtractor |
|
|
from .common import InfoExtractor |
|
|
from ..compat import ( |
|
|
|
|
|
compat_urlparse, |
|
|
|
|
|
compat_urllib_parse_unquote, |
|
|
|
|
|
compat_parse_qs |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
from ..compat import compat_urllib_parse_unquote |
|
|
from ..utils import ( |
|
|
from ..utils import ( |
|
|
ExtractorError, |
|
|
ExtractorError, |
|
|
parse_iso8601, |
|
|
parse_iso8601, |
|
|
parse_duration |
|
|
|
|
|
|
|
|
parse_duration, |
|
|
) |
|
|
) |
|
|
|
|
|
|
|
|
# ref: http://stackoverflow.com/questions/475074/regex-to-parse-or-validate-base64-data |
|
|
|
|
|
REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?' |
|
|
|
|
|
|
|
|
|
|
|
CST_ZONE = +8 # China Standard Time |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class XuiteIE(InfoExtractor): |
|
|
class XuiteIE(InfoExtractor): |
|
|
_VALID_URL = r'http://vlog.xuite.net/play/(?P<id>%s)(/.*)?' % REGEX_BASE64 |
|
|
|
|
|
|
|
|
_REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?' |
|
|
|
|
|
_VALID_URL = r'http://vlog.xuite.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64 |
|
|
_TESTS = [{ |
|
|
_TESTS = [{ |
|
|
# Audio |
|
|
# Audio |
|
|
'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2', |
|
|
'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2', |
|
|
'md5': '63a42c705772aa53fd4c1a0027f86adf', |
|
|
'md5': '63a42c705772aa53fd4c1a0027f86adf', |
|
|
'info_dict': { |
|
|
'info_dict': { |
|
|
'id': 'RGkzc1ZULTM4NjA5MTQuZmx2', |
|
|
|
|
|
|
|
|
'id': '3860914', |
|
|
'ext': 'mp3', |
|
|
'ext': 'mp3', |
|
|
'upload_date': '20110902', |
|
|
|
|
|
'uploader_id': '15973816', |
|
|
|
|
|
'uploader': '阿能', |
|
|
|
|
|
'timestamp': 1314932940, |
|
|
|
|
|
'title': '孤單南半球-歐德陽', |
|
|
'title': '孤單南半球-歐德陽', |
|
|
'thumbnail': 're:^https?://.*\.jpg$', |
|
|
'thumbnail': 're:^https?://.*\.jpg$', |
|
|
|
|
|
'duration': 247.246, |
|
|
|
|
|
'timestamp': 1314932940, |
|
|
|
|
|
'upload_date': '20110902', |
|
|
|
|
|
'uploader': '阿能', |
|
|
|
|
|
'uploader_id': '15973816', |
|
|
'categories': ['個人短片'], |
|
|
'categories': ['個人短片'], |
|
|
'duration': 247.246 |
|
|
|
|
|
} |
|
|
|
|
|
}, { |
|
|
|
|
|
# Audio with alternative form of url |
|
|
|
|
|
'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9', |
|
|
|
|
|
'md5': 'c91645f324de53d82ebb80930e1a73d2', |
|
|
|
|
|
'info_dict': { |
|
|
|
|
|
'id': 'S1dDUjdyLTMyOTc3NjcuZmx2', |
|
|
|
|
|
'ext': 'mp3', |
|
|
|
|
|
'upload_date': '20101226', |
|
|
|
|
|
'uploader_id': '10102699', |
|
|
|
|
|
'uploader': '蠍', |
|
|
|
|
|
'timestamp': 1293367080, |
|
|
|
|
|
'title': '孫燕姿-眼淚成詩', |
|
|
|
|
|
'thumbnail': 're:^https?://.*\.jpg$', |
|
|
|
|
|
'categories': ['個人短片'], |
|
|
|
|
|
'duration': 223.19 |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
}, |
|
|
}, { |
|
|
}, { |
|
|
# Video with only one format |
|
|
# Video with only one format |
|
|
'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2', |
|
|
'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2', |
|
|
'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e', |
|
|
'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e', |
|
|
'info_dict': { |
|
|
'info_dict': { |
|
|
'id': 'TkRZNjhULTM0NDE2MjkuZmx2', |
|
|
|
|
|
|
|
|
'id': '3441629', |
|
|
'ext': 'mp4', |
|
|
'ext': 'mp4', |
|
|
'upload_date': '20110306', |
|
|
|
|
|
'uploader_id': '10400126', |
|
|
|
|
|
'uploader': 'Valen', |
|
|
|
|
|
'timestamp': 1299383640, |
|
|
|
|
|
'title': '孫燕姿 - 眼淚成詩', |
|
|
'title': '孫燕姿 - 眼淚成詩', |
|
|
'thumbnail': 're:^https?://.*\.jpg$', |
|
|
'thumbnail': 're:^https?://.*\.jpg$', |
|
|
|
|
|
'duration': 217.399, |
|
|
|
|
|
'timestamp': 1299383640, |
|
|
|
|
|
'upload_date': '20110306', |
|
|
|
|
|
'uploader': 'Valen', |
|
|
|
|
|
'uploader_id': '10400126', |
|
|
'categories': ['影視娛樂'], |
|
|
'categories': ['影視娛樂'], |
|
|
'duration': 217.399 |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
}, |
|
|
}, { |
|
|
}, { |
|
|
# Video with two formats |
|
|
# Video with two formats |
|
|
'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==', |
|
|
'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==', |
|
|
'md5': '1166e0f461efe55b62e26a2d2a68e6de', |
|
|
'md5': '1166e0f461efe55b62e26a2d2a68e6de', |
|
|
'info_dict': { |
|
|
'info_dict': { |
|
|
'id': 'bWo1N1pLLTIxMzAxMTcwLmZsdg==', |
|
|
|
|
|
|
|
|
'id': '21301170', |
|
|
'ext': 'mp4', |
|
|
'ext': 'mp4', |
|
|
'upload_date': '20150117', |
|
|
|
|
|
'uploader_id': '242127761', |
|
|
|
|
|
'uploader': '我只是想認真點', |
|
|
|
|
|
'timestamp': 1421481240, |
|
|
|
|
|
'title': '暗殺教室 02', |
|
|
'title': '暗殺教室 02', |
|
|
|
|
|
'description': '字幕:【極影字幕社】', |
|
|
'thumbnail': 're:^https?://.*\.jpg$', |
|
|
'thumbnail': 're:^https?://.*\.jpg$', |
|
|
|
|
|
'duration': 1384.907, |
|
|
|
|
|
'timestamp': 1421481240, |
|
|
|
|
|
'upload_date': '20150117', |
|
|
|
|
|
'uploader': '我只是想認真點', |
|
|
|
|
|
'uploader_id': '242127761', |
|
|
'categories': ['電玩動漫'], |
|
|
'categories': ['電玩動漫'], |
|
|
'duration': 1384.907 |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
}, |
|
|
|
|
|
}, { |
|
|
|
|
|
'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9', |
|
|
|
|
|
'only_matching': True, |
|
|
}] |
|
|
}] |
|
|
|
|
|
|
|
|
def _flv_config(self, media_id): |
|
|
|
|
|
|
|
|
def _extract_flv_config(self, media_id): |
|
|
base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8') |
|
|
base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8') |
|
|
flv_config_url = 'http://vlog.xuite.net/flash/player?media=' + base64_media_id |
|
|
|
|
|
flv_config = self._download_xml(flv_config_url, 'flv config') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
flv_config = self._download_xml( |
|
|
|
|
|
'http://vlog.xuite.net/flash/player?media=%s' % base64_media_id, |
|
|
|
|
|
'flv config') |
|
|
prop_dict = {} |
|
|
prop_dict = {} |
|
|
for prop in flv_config.findall('./property'): |
|
|
for prop in flv_config.findall('./property'): |
|
|
prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8') |
|
|
prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8') |
|
|
|
|
|
|
|
|
|
|
|
# CDATA may be empty in flv config |
|
|
if not prop.text: |
|
|
if not prop.text: |
|
|
continue # CDATA may be empty in flv config |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
continue |
|
|
encoded_content = base64.b64decode(prop.text).decode('utf-8') |
|
|
encoded_content = base64.b64decode(prop.text).decode('utf-8') |
|
|
prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content) |
|
|
prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content) |
|
|
|
|
|
|
|
|
return prop_dict |
|
|
return prop_dict |
|
|
|
|
|
|
|
|
def _type_string(self, media_url): |
|
|
|
|
|
query_string = compat_urlparse.urlparse(media_url).query |
|
|
|
|
|
type_string = compat_parse_qs(query_string)['q'][0] |
|
|
|
|
|
return type_string |
|
|
|
|
|
|
|
|
|
|
|
def _guess_ext(self, media_url): |
|
|
|
|
|
type_string = self._type_string(media_url) |
|
|
|
|
|
if type_string == 'mp3': |
|
|
|
|
|
return 'mp3' |
|
|
|
|
|
elif type_string == '360' or type_string == '720': |
|
|
|
|
|
return 'mp4' |
|
|
|
|
|
else: |
|
|
|
|
|
raise ExtractorError('Unknown type string %s' % type_string) |
|
|
|
|
|
|
|
|
|
|
|
def _real_extract(self, url): |
|
|
def _real_extract(self, url): |
|
|
video_id = self._match_id(url) |
|
|
video_id = self._match_id(url) |
|
|
|
|
|
|
|
|
page = self._download_webpage(url, video_id) |
|
|
|
|
|
media_id = self._html_search_regex(r'data-mediaid="(\d+)"', page, 'media id') |
|
|
|
|
|
flv_config = self._flv_config(media_id) |
|
|
|
|
|
|
|
|
webpage = self._download_webpage(url, video_id) |
|
|
|
|
|
|
|
|
|
|
|
error_msg = self._search_regex( |
|
|
|
|
|
r'<div id="error-message-content">([^<]+)', |
|
|
|
|
|
webpage, 'error message', default=None) |
|
|
|
|
|
if error_msg: |
|
|
|
|
|
raise ExtractorError( |
|
|
|
|
|
'%s returned error: %s' % (self.IE_NAME, error_msg), |
|
|
|
|
|
expected=True) |
|
|
|
|
|
|
|
|
timestamp_local = parse_iso8601(flv_config['publish_datetime'], ' ') |
|
|
|
|
|
timestamp_gmt = timestamp_local - CST_ZONE * 3600 |
|
|
|
|
|
|
|
|
video_id = self._html_search_regex( |
|
|
|
|
|
r'data-mediaid="(\d+)"', webpage, 'media id') |
|
|
|
|
|
flv_config = self._extract_flv_config(video_id) |
|
|
|
|
|
|
|
|
ret_attrs = { |
|
|
|
|
|
|
|
|
FORMATS = { |
|
|
|
|
|
'audio': 'mp3', |
|
|
|
|
|
'video': 'mp4', |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
formats = [] |
|
|
|
|
|
for format_tag in ('src', 'hq_src'): |
|
|
|
|
|
video_url = flv_config.get(format_tag) |
|
|
|
|
|
if not video_url: |
|
|
|
|
|
continue |
|
|
|
|
|
format_id = self._search_regex( |
|
|
|
|
|
r'\bq=(.+?)\b', video_url, 'format id', default=format_tag) |
|
|
|
|
|
formats.append({ |
|
|
|
|
|
'url': video_url, |
|
|
|
|
|
'ext': FORMATS.get(flv_config['type'], 'mp4'), |
|
|
|
|
|
'format_id': format_id, |
|
|
|
|
|
'height': int(format_id) if format_id.isnumeric() else None, |
|
|
|
|
|
}) |
|
|
|
|
|
self._sort_formats(formats) |
|
|
|
|
|
|
|
|
|
|
|
timestamp = flv_config.get('publish_datetime') |
|
|
|
|
|
if timestamp: |
|
|
|
|
|
timestamp = parse_iso8601(timestamp + ' +0800', ' ') |
|
|
|
|
|
|
|
|
|
|
|
category = flv_config.get('category') |
|
|
|
|
|
categories = [category] if category else [] |
|
|
|
|
|
|
|
|
|
|
|
return { |
|
|
'id': video_id, |
|
|
'id': video_id, |
|
|
'title': flv_config['title'], |
|
|
'title': flv_config['title'], |
|
|
'thumbnail': flv_config['thumb'], |
|
|
|
|
|
'uploader': flv_config['author_name'], |
|
|
|
|
|
'timestamp': timestamp_gmt, |
|
|
|
|
|
'uploader_id': flv_config['author_id'], |
|
|
|
|
|
'duration': parse_duration(flv_config['duration']), |
|
|
|
|
|
'categories': [flv_config['category']] |
|
|
|
|
|
|
|
|
'description': flv_config.get('description'), |
|
|
|
|
|
'thumbnail': flv_config.get('thumb'), |
|
|
|
|
|
'timestamp': timestamp, |
|
|
|
|
|
'uploader': flv_config.get('author_name'), |
|
|
|
|
|
'uploader_id': flv_config.get('author_id'), |
|
|
|
|
|
'duration': parse_duration(flv_config.get('duration')), |
|
|
|
|
|
'categories': categories, |
|
|
|
|
|
'formats': formats, |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if 'hq_src' in flv_config: |
|
|
|
|
|
urls = [flv_config['src'], flv_config['hq_src']] |
|
|
|
|
|
|
|
|
|
|
|
ret_attrs['formats'] = [] |
|
|
|
|
|
|
|
|
|
|
|
for url in urls: |
|
|
|
|
|
ret_attrs['formats'].append({ |
|
|
|
|
|
'url': url, |
|
|
|
|
|
'ext': self._guess_ext(url), |
|
|
|
|
|
'format_id': self._type_string(url), |
|
|
|
|
|
'height': int(self._type_string(url)) |
|
|
|
|
|
}) |
|
|
|
|
|
else: |
|
|
|
|
|
ret_attrs['url'] = flv_config['src'] |
|
|
|
|
|
ret_attrs['ext'] = self._guess_ext(flv_config['src']) |
|
|
|
|
|
|
|
|
|
|
|
return ret_attrs |
|
|
|