|
@ -18,7 +18,7 @@ __author__ = ( |
|
|
) |
|
|
) |
|
|
|
|
|
|
|
|
__license__ = 'Public Domain' |
|
|
__license__ = 'Public Domain' |
|
|
__version__ = '2011.11.23' |
|
|
|
|
|
|
|
|
__version__ = '2011.12.08' |
|
|
|
|
|
|
|
|
UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl' |
|
|
UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl' |
|
|
|
|
|
|
|
@ -282,6 +282,14 @@ def _simplify_title(title): |
|
|
expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE) |
|
|
expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE) |
|
|
return expr.sub(u'_', title).strip(u'_') |
|
|
return expr.sub(u'_', title).strip(u'_') |
|
|
|
|
|
|
|
|
|
|
|
def _orderedSet(iterable): |
|
|
|
|
|
""" Remove all duplicates from the input iterable """ |
|
|
|
|
|
res = [] |
|
|
|
|
|
for el in iterable: |
|
|
|
|
|
if el not in res: |
|
|
|
|
|
res.append(el) |
|
|
|
|
|
return res |
|
|
|
|
|
|
|
|
class DownloadError(Exception): |
|
|
class DownloadError(Exception): |
|
|
"""Download Error exception. |
|
|
"""Download Error exception. |
|
|
|
|
|
|
|
@ -309,6 +317,10 @@ class PostProcessingError(Exception): |
|
|
""" |
|
|
""" |
|
|
pass |
|
|
pass |
|
|
|
|
|
|
|
|
|
|
|
class MaxDownloadsReached(Exception): |
|
|
|
|
|
""" --max-downloads limit has been reached. """ |
|
|
|
|
|
pass |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class UnavailableVideoError(Exception): |
|
|
class UnavailableVideoError(Exception): |
|
|
"""Unavailable Format exception. |
|
|
"""Unavailable Format exception. |
|
@ -722,8 +734,7 @@ class FileDownloader(object): |
|
|
max_downloads = self.params.get('max_downloads') |
|
|
max_downloads = self.params.get('max_downloads') |
|
|
if max_downloads is not None: |
|
|
if max_downloads is not None: |
|
|
if self._num_downloads > int(max_downloads): |
|
|
if self._num_downloads > int(max_downloads): |
|
|
self.to_screen(u'[download] Maximum number of downloads reached. Skipping ' + info_dict['title']) |
|
|
|
|
|
return |
|
|
|
|
|
|
|
|
raise MaxDownloadsReached() |
|
|
|
|
|
|
|
|
filename = self.prepare_filename(info_dict) |
|
|
filename = self.prepare_filename(info_dict) |
|
|
|
|
|
|
|
@ -1110,6 +1121,7 @@ class YoutubeIE(InfoExtractor): |
|
|
_NETRC_MACHINE = 'youtube' |
|
|
_NETRC_MACHINE = 'youtube' |
|
|
# Listed in order of quality |
|
|
# Listed in order of quality |
|
|
_available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13'] |
|
|
_available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13'] |
|
|
|
|
|
_available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13'] |
|
|
_video_extensions = { |
|
|
_video_extensions = { |
|
|
'13': '3gp', |
|
|
'13': '3gp', |
|
|
'17': 'mp4', |
|
|
'17': 'mp4', |
|
@ -1359,10 +1371,11 @@ class YoutubeIE(InfoExtractor): |
|
|
url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data) |
|
|
url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data) |
|
|
|
|
|
|
|
|
format_limit = self._downloader.params.get('format_limit', None) |
|
|
format_limit = self._downloader.params.get('format_limit', None) |
|
|
if format_limit is not None and format_limit in self._available_formats: |
|
|
|
|
|
format_list = self._available_formats[self._available_formats.index(format_limit):] |
|
|
|
|
|
|
|
|
available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats |
|
|
|
|
|
if format_limit is not None and format_limit in available_formats: |
|
|
|
|
|
format_list = available_formats[available_formats.index(format_limit):] |
|
|
else: |
|
|
else: |
|
|
format_list = self._available_formats |
|
|
|
|
|
|
|
|
format_list = available_formats |
|
|
existing_formats = [x for x in format_list if x in url_map] |
|
|
existing_formats = [x for x in format_list if x in url_map] |
|
|
if len(existing_formats) == 0: |
|
|
if len(existing_formats) == 0: |
|
|
self._downloader.trouble(u'ERROR: no known formats available for video') |
|
|
self._downloader.trouble(u'ERROR: no known formats available for video') |
|
@ -3744,6 +3757,124 @@ class MixcloudIE(InfoExtractor): |
|
|
except UnavailableVideoError, err: |
|
|
except UnavailableVideoError, err: |
|
|
self._downloader.trouble(u'ERROR: unable to download file') |
|
|
self._downloader.trouble(u'ERROR: unable to download file') |
|
|
|
|
|
|
|
|
|
|
|
class StanfordOpenClassroomIE(InfoExtractor): |
|
|
|
|
|
"""Information extractor for Stanford's Open ClassRoom""" |
|
|
|
|
|
|
|
|
|
|
|
_VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$' |
|
|
|
|
|
IE_NAME = u'stanfordoc' |
|
|
|
|
|
|
|
|
|
|
|
def report_download_webpage(self, objid): |
|
|
|
|
|
"""Report information extraction.""" |
|
|
|
|
|
self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid)) |
|
|
|
|
|
|
|
|
|
|
|
def report_extraction(self, video_id): |
|
|
|
|
|
"""Report information extraction.""" |
|
|
|
|
|
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) |
|
|
|
|
|
|
|
|
|
|
|
def _real_extract(self, url): |
|
|
|
|
|
mobj = re.match(self._VALID_URL, url) |
|
|
|
|
|
if mobj is None: |
|
|
|
|
|
self._downloader.trouble(u'ERROR: invalid URL: %s' % url) |
|
|
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
if mobj.group('course') and mobj.group('video'): # A specific video |
|
|
|
|
|
course = mobj.group('course') |
|
|
|
|
|
video = mobj.group('video') |
|
|
|
|
|
info = { |
|
|
|
|
|
'id': _simplify_title(course + '_' + video), |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
self.report_extraction(info['id']) |
|
|
|
|
|
baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/' |
|
|
|
|
|
xmlUrl = baseUrl + video + '.xml' |
|
|
|
|
|
try: |
|
|
|
|
|
metaXml = urllib2.urlopen(xmlUrl).read() |
|
|
|
|
|
except (urllib2.URLError, httplib.HTTPException, socket.error), err: |
|
|
|
|
|
self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err)) |
|
|
|
|
|
return |
|
|
|
|
|
mdoc = xml.etree.ElementTree.fromstring(metaXml) |
|
|
|
|
|
try: |
|
|
|
|
|
info['title'] = mdoc.findall('./title')[0].text |
|
|
|
|
|
info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text |
|
|
|
|
|
except IndexError: |
|
|
|
|
|
self._downloader.trouble(u'\nERROR: Invalid metadata XML file') |
|
|
|
|
|
return |
|
|
|
|
|
info['stitle'] = _simplify_title(info['title']) |
|
|
|
|
|
info['ext'] = info['url'].rpartition('.')[2] |
|
|
|
|
|
info['format'] = info['ext'] |
|
|
|
|
|
self._downloader.increment_downloads() |
|
|
|
|
|
try: |
|
|
|
|
|
self._downloader.process_info(info) |
|
|
|
|
|
except UnavailableVideoError, err: |
|
|
|
|
|
self._downloader.trouble(u'\nERROR: unable to download video') |
|
|
|
|
|
elif mobj.group('course'): # A course page |
|
|
|
|
|
unescapeHTML = HTMLParser.HTMLParser().unescape |
|
|
|
|
|
|
|
|
|
|
|
course = mobj.group('course') |
|
|
|
|
|
info = { |
|
|
|
|
|
'id': _simplify_title(course), |
|
|
|
|
|
'type': 'playlist', |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
self.report_download_webpage(info['id']) |
|
|
|
|
|
try: |
|
|
|
|
|
coursepage = urllib2.urlopen(url).read() |
|
|
|
|
|
except (urllib2.URLError, httplib.HTTPException, socket.error), err: |
|
|
|
|
|
self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err)) |
|
|
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
m = re.search('<h1>([^<]+)</h1>', coursepage) |
|
|
|
|
|
if m: |
|
|
|
|
|
info['title'] = unescapeHTML(m.group(1)) |
|
|
|
|
|
else: |
|
|
|
|
|
info['title'] = info['id'] |
|
|
|
|
|
info['stitle'] = _simplify_title(info['title']) |
|
|
|
|
|
|
|
|
|
|
|
m = re.search('<description>([^<]+)</description>', coursepage) |
|
|
|
|
|
if m: |
|
|
|
|
|
info['description'] = unescapeHTML(m.group(1)) |
|
|
|
|
|
|
|
|
|
|
|
links = _orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage)) |
|
|
|
|
|
info['list'] = [ |
|
|
|
|
|
{ |
|
|
|
|
|
'type': 'reference', |
|
|
|
|
|
'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage), |
|
|
|
|
|
} |
|
|
|
|
|
for vpage in links] |
|
|
|
|
|
|
|
|
|
|
|
for entry in info['list']: |
|
|
|
|
|
assert entry['type'] == 'reference' |
|
|
|
|
|
self.extract(entry['url']) |
|
|
|
|
|
else: # Root page |
|
|
|
|
|
unescapeHTML = HTMLParser.HTMLParser().unescape |
|
|
|
|
|
|
|
|
|
|
|
info = { |
|
|
|
|
|
'id': 'Stanford OpenClassroom', |
|
|
|
|
|
'type': 'playlist', |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
self.report_download_webpage(info['id']) |
|
|
|
|
|
rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' |
|
|
|
|
|
try: |
|
|
|
|
|
rootpage = urllib2.urlopen(rootURL).read() |
|
|
|
|
|
except (urllib2.URLError, httplib.HTTPException, socket.error), err: |
|
|
|
|
|
self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err)) |
|
|
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
info['title'] = info['id'] |
|
|
|
|
|
info['stitle'] = _simplify_title(info['title']) |
|
|
|
|
|
|
|
|
|
|
|
links = _orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage)) |
|
|
|
|
|
info['list'] = [ |
|
|
|
|
|
{ |
|
|
|
|
|
'type': 'reference', |
|
|
|
|
|
'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage), |
|
|
|
|
|
} |
|
|
|
|
|
for cpage in links] |
|
|
|
|
|
|
|
|
|
|
|
for entry in info['list']: |
|
|
|
|
|
assert entry['type'] == 'reference' |
|
|
|
|
|
self.extract(entry['url']) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PostProcessor(object): |
|
|
class PostProcessor(object): |
|
@ -3839,8 +3970,13 @@ class FFmpegExtractAudioPP(PostProcessor): |
|
|
return None |
|
|
return None |
|
|
|
|
|
|
|
|
more_opts = [] |
|
|
more_opts = [] |
|
|
if self._preferredcodec == 'best' or self._preferredcodec == filecodec: |
|
|
|
|
|
if filecodec in ['aac', 'mp3', 'vorbis']: |
|
|
|
|
|
|
|
|
if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): |
|
|
|
|
|
if self._preferredcodec == 'm4a' and filecodec == 'aac': |
|
|
|
|
|
# Lossless, but in another container |
|
|
|
|
|
acodec = 'copy' |
|
|
|
|
|
extension = self._preferredcodec |
|
|
|
|
|
more_opts = ['-absf', 'aac_adtstoasc'] |
|
|
|
|
|
elif filecodec in ['aac', 'mp3', 'vorbis']: |
|
|
# Lossless if possible |
|
|
# Lossless if possible |
|
|
acodec = 'copy' |
|
|
acodec = 'copy' |
|
|
extension = filecodec |
|
|
extension = filecodec |
|
@ -3857,13 +3993,15 @@ class FFmpegExtractAudioPP(PostProcessor): |
|
|
more_opts += ['-ab', self._preferredquality] |
|
|
more_opts += ['-ab', self._preferredquality] |
|
|
else: |
|
|
else: |
|
|
# We convert the audio (lossy) |
|
|
# We convert the audio (lossy) |
|
|
acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'vorbis': 'libvorbis'}[self._preferredcodec] |
|
|
|
|
|
|
|
|
acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis'}[self._preferredcodec] |
|
|
extension = self._preferredcodec |
|
|
extension = self._preferredcodec |
|
|
more_opts = [] |
|
|
more_opts = [] |
|
|
if self._preferredquality is not None: |
|
|
if self._preferredquality is not None: |
|
|
more_opts += ['-ab', self._preferredquality] |
|
|
more_opts += ['-ab', self._preferredquality] |
|
|
if self._preferredcodec == 'aac': |
|
|
if self._preferredcodec == 'aac': |
|
|
more_opts += ['-f', 'adts'] |
|
|
more_opts += ['-f', 'adts'] |
|
|
|
|
|
if self._preferredcodec == 'm4a': |
|
|
|
|
|
more_opts += ['-absf', 'aac_adtstoasc'] |
|
|
if self._preferredcodec == 'vorbis': |
|
|
if self._preferredcodec == 'vorbis': |
|
|
extension = 'ogg' |
|
|
extension = 'ogg' |
|
|
|
|
|
|
|
@ -4039,6 +4177,8 @@ def parseOpts(): |
|
|
action='store', dest='format', metavar='FORMAT', help='video format code') |
|
|
action='store', dest='format', metavar='FORMAT', help='video format code') |
|
|
video_format.add_option('--all-formats', |
|
|
video_format.add_option('--all-formats', |
|
|
action='store_const', dest='format', help='download all available video formats', const='all') |
|
|
action='store_const', dest='format', help='download all available video formats', const='all') |
|
|
|
|
|
video_format.add_option('--prefer-free-formats', |
|
|
|
|
|
action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') |
|
|
video_format.add_option('--max-quality', |
|
|
video_format.add_option('--max-quality', |
|
|
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') |
|
|
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') |
|
|
video_format.add_option('-F', '--list-formats', |
|
|
video_format.add_option('-F', '--list-formats', |
|
@ -4110,7 +4250,7 @@ def parseOpts(): |
|
|
postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False, |
|
|
postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False, |
|
|
help='convert video files to audio-only files (requires ffmpeg and ffprobe)') |
|
|
help='convert video files to audio-only files (requires ffmpeg and ffprobe)') |
|
|
postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', |
|
|
postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', |
|
|
help='"best", "aac", "vorbis" or "mp3"; best by default') |
|
|
|
|
|
|
|
|
help='"best", "aac", "vorbis", "mp3", or "m4a"; best by default') |
|
|
postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K', |
|
|
postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K', |
|
|
help='ffmpeg audio bitrate specification, 128k by default') |
|
|
help='ffmpeg audio bitrate specification, 128k by default') |
|
|
postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, |
|
|
postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, |
|
@ -4166,6 +4306,7 @@ def gen_extractors(): |
|
|
SoundcloudIE(), |
|
|
SoundcloudIE(), |
|
|
InfoQIE(), |
|
|
InfoQIE(), |
|
|
MixcloudIE(), |
|
|
MixcloudIE(), |
|
|
|
|
|
StanfordOpenClassroomIE(), |
|
|
|
|
|
|
|
|
GenericIE() |
|
|
GenericIE() |
|
|
] |
|
|
] |
|
@ -4255,7 +4396,7 @@ def _real_main(): |
|
|
except (TypeError, ValueError), err: |
|
|
except (TypeError, ValueError), err: |
|
|
parser.error(u'invalid playlist end number specified') |
|
|
parser.error(u'invalid playlist end number specified') |
|
|
if opts.extractaudio: |
|
|
if opts.extractaudio: |
|
|
if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis']: |
|
|
|
|
|
|
|
|
if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a']: |
|
|
parser.error(u'invalid audio format specified') |
|
|
parser.error(u'invalid audio format specified') |
|
|
|
|
|
|
|
|
# File downloader |
|
|
# File downloader |
|
@ -4302,6 +4443,7 @@ def _real_main(): |
|
|
'matchtitle': opts.matchtitle, |
|
|
'matchtitle': opts.matchtitle, |
|
|
'rejecttitle': opts.rejecttitle, |
|
|
'rejecttitle': opts.rejecttitle, |
|
|
'max_downloads': opts.max_downloads, |
|
|
'max_downloads': opts.max_downloads, |
|
|
|
|
|
'prefer_free_formats': opts.prefer_free_formats, |
|
|
}) |
|
|
}) |
|
|
for extractor in extractors: |
|
|
for extractor in extractors: |
|
|
fd.add_info_extractor(extractor) |
|
|
fd.add_info_extractor(extractor) |
|
@ -4320,7 +4462,12 @@ def _real_main(): |
|
|
parser.error(u'you must provide at least one URL') |
|
|
parser.error(u'you must provide at least one URL') |
|
|
else: |
|
|
else: |
|
|
sys.exit() |
|
|
sys.exit() |
|
|
retcode = fd.download(all_urls) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
retcode = fd.download(all_urls) |
|
|
|
|
|
except MaxDownloadsReached: |
|
|
|
|
|
fd.to_screen(u'--max-download limit reached, aborting.') |
|
|
|
|
|
retcode = 101 |
|
|
|
|
|
|
|
|
# Dump cookie jar if requested |
|
|
# Dump cookie jar if requested |
|
|
if opts.cookiefile is not None: |
|
|
if opts.cookiefile is not None: |
|
|