X-Git-Url: http://git.jankratochvil.net/?a=blobdiff_plain;f=youtube_dl%2F__init__.py;h=77723155d3a65d586af8e6c31d5c3edeedcb3c04;hb=7e45ec57a84c75ee88f7652a4b3bf74da8ef49b1;hp=d7e9c50c0076016d2dcabfec86d49c8cdab1b04e;hpb=c379c181e057491171a43228752fcb7e20c86d5f;p=youtube-dl.git diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index d7e9c50..7772315 100755 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__author__ = ( +__authors__ = ( 'Ricardo Garcia Gonzalez', 'Danny Colligan', 'Benjamin Johnson', @@ -18,12 +18,14 @@ __author__ = ( ) __license__ = 'Public Domain' -__version__ = '2011.11.23' +__version__ = '2012.02.27' UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl' + import cookielib import datetime +import getpass import gzip import htmlentitydefs import HTMLParser @@ -31,9 +33,11 @@ import httplib import locale import math import netrc +import optparse import os import os.path import re +import shlex import socket import string import subprocess @@ -259,14 +263,14 @@ def sanitize_open(filename, open_mode): import msvcrt msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) return (sys.stdout, filename) - stream = open(filename, open_mode) + stream = open(_encodeFilename(filename), open_mode) return (stream, filename) except (IOError, OSError), err: # In case of error, try to remove win32 forbidden chars filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename) # An exception here should be caught in the caller - stream = open(filename, open_mode) + stream = open(_encodeFilename(filename), open_mode) return (stream, filename) @@ -282,6 +286,38 @@ def _simplify_title(title): expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE) return expr.sub(u'_', title).strip(u'_') +def _orderedSet(iterable): + """ Remove all duplicates from the input iterable """ + res = [] + for el in iterable: + if el not in res: + res.append(el) + return res + +def _unescapeHTML(s): + """ + @param s a string (of type unicode) + """ + assert type(s) == type(u'') + + htmlParser = HTMLParser.HTMLParser() + return htmlParser.unescape(s) + +def _encodeFilename(s): + """ + @param s The name of the file (of type unicode) + """ + + assert type(s) == type(u'') + + if sys.platform == 'win32' and sys.getwindowsversion().major >= 5: + # Pass u'' directly to use Unicode APIs on Windows 2000 and up + # (Detecting Windows NT 4 is tricky because 'major >= 4' would + # match Windows 9x series as well. Besides, NT 4 is obsolete.) + return s + else: + return s.encode(sys.getfilesystemencoding(), 'ignore') + class DownloadError(Exception): """Download Error exception. @@ -309,6 +345,10 @@ class PostProcessingError(Exception): """ pass +class MaxDownloadsReached(Exception): + """ --max-downloads limit has been reached. """ + pass + class UnavailableVideoError(Exception): """Unavailable Format exception. @@ -542,16 +582,17 @@ class FileDownloader(object): self._pps.append(pp) pp.set_downloader(self) - def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False): + def to_screen(self, message, skip_eol=False): """Print message to stdout if not in quiet mode.""" - try: - if not self.params.get('quiet', False): - terminator = [u'\n', u''][skip_eol] - print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()), + assert type(message) == type(u'') + if not self.params.get('quiet', False): + terminator = [u'\n', u''][skip_eol] + output = message + terminator + + if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr + output = output.encode(preferredencoding(), 'ignore') + self._screen_file.write(output) self._screen_file.flush() - except (UnicodeEncodeError), err: - if not ignore_encoding_errors: - raise def to_stderr(self, message): """Print message to stderr.""" @@ -601,7 +642,7 @@ class FileDownloader(object): def temp_name(self, filename): """Returns a temporary filename for the given filename.""" if self.params.get('nopart', False) or filename == u'-' or \ - (os.path.exists(filename) and not os.path.isfile(filename)): + (os.path.exists(_encodeFilename(filename)) and not os.path.isfile(_encodeFilename(filename))): return filename return filename + u'.part' @@ -614,7 +655,7 @@ class FileDownloader(object): try: if old_filename == new_filename: return - os.rename(old_filename, new_filename) + os.rename(_encodeFilename(old_filename), _encodeFilename(new_filename)) except (IOError, OSError), err: self.trouble(u'ERROR: unable to rename file') @@ -622,7 +663,7 @@ class FileDownloader(object): """Try to set the last-modified time of the given file.""" if last_modified_hdr is None: return - if not os.path.isfile(filename): + if not os.path.isfile(_encodeFilename(filename)): return timestr = last_modified_hdr if timestr is None: @@ -638,15 +679,15 @@ class FileDownloader(object): def report_writedescription(self, descfn): """ Report that the description file is being written """ - self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True) + self.to_screen(u'[info] Writing video description to: ' + descfn) def report_writeinfojson(self, infofn): """ Report that the metadata file has been written """ - self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True) + self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) def report_destination(self, filename): """Report destination filename.""" - self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True) + self.to_screen(u'[download] Destination: ' + filename) def report_progress(self, percent_str, data_len_str, speed_str, eta_str): """Report download progress.""" @@ -699,15 +740,31 @@ class FileDownloader(object): self.trouble(u'ERROR: invalid system charset or erroneous output template') return None + def _match_entry(self, info_dict): + """ Returns None iff the file should be downloaded """ + + title = info_dict['title'] + matchtitle = self.params.get('matchtitle', False) + if matchtitle and not re.search(matchtitle, title, re.IGNORECASE): + return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' + rejecttitle = self.params.get('rejecttitle', False) + if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE): + return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' + return None + def process_info(self, info_dict): """Process a single dictionary returned by an InfoExtractor.""" + reason = self._match_entry(info_dict) + if reason is not None: + self.to_screen(u'[download] ' + reason) + return + max_downloads = self.params.get('max_downloads') if max_downloads is not None: if self._num_downloads > int(max_downloads): - self.to_screen(u'[download] Maximum number of downloads reached. Skipping ' + info_dict['title']) - return - + raise MaxDownloadsReached() + filename = self.prepare_filename(info_dict) # Forced printings @@ -731,23 +788,9 @@ class FileDownloader(object): if filename is None: return - matchtitle=self.params.get('matchtitle',False) - rejecttitle=self.params.get('rejecttitle',False) - title=info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace') - if matchtitle and not re.search(matchtitle, title, re.IGNORECASE): - self.to_screen(u'[download] "%s" title did not match pattern "%s"' % (title, matchtitle)) - return - if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE): - self.to_screen(u'[download] "%s" title matched reject pattern "%s"' % (title, rejecttitle)) - return - - if self.params.get('nooverwrites', False) and os.path.exists(filename): - self.to_stderr(u'WARNING: file exists and will be skipped') - return - try: - dn = os.path.dirname(filename) - if dn != '' and not os.path.exists(dn): + dn = os.path.dirname(_encodeFilename(filename)) + if dn != '' and not os.path.exists(dn): # dn is already encoded os.makedirs(dn) except (OSError, IOError), err: self.trouble(u'ERROR: unable to create directory ' + unicode(err)) @@ -755,9 +798,9 @@ class FileDownloader(object): if self.params.get('writedescription', False): try: - descfn = filename + '.description' + descfn = filename + u'.description' self.report_writedescription(descfn) - descfile = open(descfn, 'wb') + descfile = open(_encodeFilename(descfn), 'wb') try: descfile.write(info_dict['description'].encode('utf-8')) finally: @@ -767,7 +810,7 @@ class FileDownloader(object): return if self.params.get('writeinfojson', False): - infofn = filename + '.info.json' + infofn = filename + u'.info.json' self.report_writeinfojson(infofn) try: json.dump @@ -775,7 +818,7 @@ class FileDownloader(object): self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') return try: - infof = open(infofn, 'wb') + infof = open(_encodeFilename(infofn), 'wb') try: json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',)) json.dump(json_info_dict, infof) @@ -786,16 +829,19 @@ class FileDownloader(object): return if not self.params.get('skip_download', False): - try: - success = self._do_download(filename, info_dict) - except (OSError, IOError), err: - raise UnavailableVideoError - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self.trouble(u'ERROR: unable to download video data: %s' % str(err)) - return - except (ContentTooShortError, ), err: - self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) - return + if self.params.get('nooverwrites', False) and os.path.exists(_encodeFilename(filename)): + success = True + else: + try: + success = self._do_download(filename, info_dict) + except (OSError, IOError), err: + raise UnavailableVideoError + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self.trouble(u'ERROR: unable to download video data: %s' % str(err)) + return + except (ContentTooShortError, ), err: + self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) + return if success: try: @@ -854,13 +900,21 @@ class FileDownloader(object): # the connection was interrumpted and resuming appears to be # possible. This is part of rtmpdump's normal usage, AFAIK. basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] - retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]) + args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)] + if self.params.get('verbose', False): + try: + import pipes + shell_quote = lambda args: ' '.join(map(pipes.quote, args)) + except ImportError: + shell_quote = repr + self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) + retval = subprocess.call(args) while retval == 2 or retval == 1: - prevsize = os.path.getsize(tmpfilename) + prevsize = os.path.getsize(_encodeFilename(tmpfilename)) self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) time.sleep(5.0) # This seems to be needed retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) - cursize = os.path.getsize(tmpfilename) + cursize = os.path.getsize(_encodeFilename(tmpfilename)) if prevsize == cursize and retval == 1: break # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those @@ -869,7 +923,7 @@ class FileDownloader(object): retval = 0 break if retval == 0: - self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename)) + self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(_encodeFilename(tmpfilename))) self.try_rename(tmpfilename, filename) return True else: @@ -881,7 +935,7 @@ class FileDownloader(object): player_url = info_dict.get('player_url', None) # Check file already present - if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False): + if self.params.get('continuedl', False) and os.path.isfile(_encodeFilename(filename)) and not self.params.get('nopart', False): self.report_file_already_downloaded(filename) return True @@ -898,8 +952,8 @@ class FileDownloader(object): request = urllib2.Request(url, None, headers) # Establish possible resume length - if os.path.isfile(tmpfilename): - resume_len = os.path.getsize(tmpfilename) + if os.path.isfile(_encodeFilename(tmpfilename)): + resume_len = os.path.getsize(_encodeFilename(tmpfilename)) else: resume_len = 0 @@ -1103,6 +1157,7 @@ class YoutubeIE(InfoExtractor): _NETRC_MACHINE = 'youtube' # Listed in order of quality _available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13'] + _available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13'] _video_extensions = { '13': '3gp', '17': 'mp4', @@ -1326,10 +1381,9 @@ class YoutubeIE(InfoExtractor): lxml.etree except NameError: video_description = u'No description available.' - if self._downloader.params.get('forcedescription', False) or self._downloader.params.get('writedescription', False): - mobj = re.search(r'', video_webpage) - if mobj is not None: - video_description = mobj.group(1).decode('utf-8') + mobj = re.search(r'', video_webpage) + if mobj is not None: + video_description = mobj.group(1).decode('utf-8') else: html_parser = lxml.etree.HTMLParser(encoding='utf-8') vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser) @@ -1352,10 +1406,11 @@ class YoutubeIE(InfoExtractor): url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data) format_limit = self._downloader.params.get('format_limit', None) - if format_limit is not None and format_limit in self._available_formats: - format_list = self._available_formats[self._available_formats.index(format_limit):] + available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats + if format_limit is not None and format_limit in available_formats: + format_list = available_formats[available_formats.index(format_limit):] else: - format_list = self._available_formats + format_list = available_formats existing_formats = [x for x in format_list if x in url_map] if len(existing_formats) == 0: self._downloader.trouble(u'ERROR: no known formats available for video') @@ -1581,7 +1636,6 @@ class DailymotionIE(InfoExtractor): self._downloader.increment_downloads() video_id = mobj.group(1) - simple_title = mobj.group(2).decode('utf-8') video_extension = 'flv' # Retrieve video webpage to extract further information @@ -1611,12 +1665,13 @@ class DailymotionIE(InfoExtractor): video_url = mediaURL - mobj = re.search(r'(?im)Dailymotion\s*-\s*(.+)\s*-\s*[^<]+?', webpage) + mobj = re.search(r'', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract title') return - video_title = mobj.group(1).decode('utf-8') + video_title = _unescapeHTML(mobj.group('title').decode('utf-8')) video_title = sanitize_title(video_title) + simple_title = _simplify_title(video_title) mobj = re.search(r'(?im)[^<]+?]+?>([^<]+?)', webpage) if mobj is None: @@ -2285,8 +2340,8 @@ class GoogleSearchIE(InfoExtractor): """Information Extractor for Google Video search queries.""" _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+' _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en' - _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&' - _MORE_PAGES_INDICATOR = r'Next' + _VIDEO_INDICATOR = r'', html) + mMovieParams = re.findall('(?:/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P[^&]+)(&video=(?P', coursepage)) + info['list'] = [ + { + 'type': 'reference', + 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage), + } + for vpage in links] + + for entry in info['list']: + assert entry['type'] == 'reference' + self.extract(entry['url']) + else: # Root page + unescapeHTML = HTMLParser.HTMLParser().unescape + + info = { + 'id': 'Stanford OpenClassroom', + 'type': 'playlist', + } + + self.report_download_webpage(info['id']) + rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' + try: + rootpage = urllib2.urlopen(rootURL).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err)) + return + + info['title'] = info['id'] + info['stitle'] = _simplify_title(info['title']) + + links = _orderedSet(re.findall('', rootpage)) + info['list'] = [ + { + 'type': 'reference', + 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage), + } + for cpage in links] + + for entry in info['list']: + assert entry['type'] == 'reference' + self.extract(entry['url']) + +class MTVIE(InfoExtractor): + """Information extractor for MTV.com""" + + _VALID_URL = r'^(?Phttps?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P[0-9]+)/[^/]+$' + IE_NAME = u'mtv' + + def report_webpage(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + if not mobj.group('proto'): + url = 'http://' + url + video_id = mobj.group('videoid') + self.report_webpage(video_id) + + request = urllib2.Request(url) + try: + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) + return + + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract song name') + return + song_name = _unescapeHTML(mobj.group(1).decode('iso-8859-1')) + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract performer') + return + performer = _unescapeHTML(mobj.group(1).decode('iso-8859-1')) + video_title = performer + ' - ' + song_name + + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to mtvn_uri') + return + mtvn_uri = mobj.group(1) + + mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract content id') + return + content_id = mobj.group(1) + + videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri + self.report_extraction(video_id) + request = urllib2.Request(videogen_url) + try: + metadataXml = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err)) + return + + mdoc = xml.etree.ElementTree.fromstring(metadataXml) + renditions = mdoc.findall('.//rendition') + + # For now, always pick the highest quality. + rendition = renditions[-1] + + try: + _,_,ext = rendition.attrib['type'].partition('/') + format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate'] + video_url = rendition.find('./src').text + except KeyError: + self._downloader.trouble('Invalid rendition field.') + return + + self._downloader.increment_downloads() + info = { + 'id': video_id, + 'url': video_url, + 'uploader': performer, + 'title': video_title, + 'stitle': _simplify_title(video_title), + 'ext': ext, + 'format': format, + } + + try: + self._downloader.process_info(info) + except UnavailableVideoError, err: + self._downloader.trouble(u'\nERROR: unable to download ' + video_id) class PostProcessor(object): @@ -3785,6 +4050,9 @@ class PostProcessor(object): """ return information # by default, do nothing +class AudioConversionError(BaseException): + def __init__(self, message): + self.message = message class FFmpegExtractAudioPP(PostProcessor): @@ -3799,7 +4067,7 @@ class FFmpegExtractAudioPP(PostProcessor): @staticmethod def get_audio_codec(path): try: - cmd = ['ffprobe', '-show_streams', '--', path] + cmd = ['ffprobe', '-show_streams', '--', _encodeFilename(path)] handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE) output = handle.communicate()[0] if handle.wait() != 0: @@ -3816,12 +4084,23 @@ class FFmpegExtractAudioPP(PostProcessor): @staticmethod def run_ffmpeg(path, out_path, codec, more_opts): + if codec is None: + acodec_opts = [] + else: + acodec_opts = ['-acodec', codec] + cmd = ['ffmpeg', '-y', '-i', _encodeFilename(path), '-vn'] + acodec_opts + more_opts + ['--', _encodeFilename(out_path)] try: - cmd = ['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + ['--', out_path] - ret = subprocess.call(cmd, stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT) - return (ret == 0) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = p.communicate() except (IOError, OSError): - return False + e = sys.exc_info()[1] + if isinstance(e, OSError) and e.errno == 2: + raise AudioConversionError('ffmpeg not found. Please install ffmpeg.') + else: + raise e + if p.returncode != 0: + msg = stderr.strip().split('\n')[-1] + raise AudioConversionError(msg) def run(self, information): path = information['filepath'] @@ -3832,8 +4111,13 @@ class FFmpegExtractAudioPP(PostProcessor): return None more_opts = [] - if self._preferredcodec == 'best' or self._preferredcodec == filecodec: - if filecodec in ['aac', 'mp3', 'vorbis']: + if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): + if self._preferredcodec == 'm4a' and filecodec == 'aac': + # Lossless, but in another container + acodec = 'copy' + extension = self._preferredcodec + more_opts = ['-absf', 'aac_adtstoasc'] + elif filecodec in ['aac', 'mp3', 'vorbis']: # Lossless if possible acodec = 'copy' extension = filecodec @@ -3850,35 +4134,44 @@ class FFmpegExtractAudioPP(PostProcessor): more_opts += ['-ab', self._preferredquality] else: # We convert the audio (lossy) - acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'vorbis': 'libvorbis'}[self._preferredcodec] + acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec] extension = self._preferredcodec more_opts = [] if self._preferredquality is not None: more_opts += ['-ab', self._preferredquality] if self._preferredcodec == 'aac': more_opts += ['-f', 'adts'] + if self._preferredcodec == 'm4a': + more_opts += ['-absf', 'aac_adtstoasc'] if self._preferredcodec == 'vorbis': extension = 'ogg' + if self._preferredcodec == 'wav': + extension = 'wav' + more_opts += ['-f', 'wav'] - (prefix, ext) = os.path.splitext(path) - new_path = prefix + '.' + extension - self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path) - status = self.run_ffmpeg(path, new_path, acodec, more_opts) - - if not status: - self._downloader.to_stderr(u'WARNING: error running ffmpeg') + prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups + new_path = prefix + sep + extension + self._downloader.to_screen(u'[ffmpeg] Destination: ' + new_path) + try: + self.run_ffmpeg(path, new_path, acodec, more_opts) + except: + etype,e,tb = sys.exc_info() + if isinstance(e, AudioConversionError): + self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message) + else: + self._downloader.to_stderr(u'ERROR: error running ffmpeg') return None # Try to update the date time for extracted audio file. if information.get('filetime') is not None: try: - os.utime(new_path, (time.time(), information['filetime'])) + os.utime(_encodeFilename(new_path), (time.time(), information['filetime'])) except: self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') if not self._keepvideo: try: - os.remove(path) + os.remove(_encodeFilename(path)) except (IOError, OSError): self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') return None @@ -3893,7 +4186,7 @@ def updateSelf(downloader, filename): if not os.access(filename, os.W_OK): sys.exit('ERROR: no write permissions on %s' % filename) - downloader.to_screen('Updating to latest version...') + downloader.to_screen(u'Updating to latest version...') try: try: @@ -3902,7 +4195,7 @@ def updateSelf(downloader, filename): vmatch = re.search("__version__ = '([^']+)'", newcontent) if vmatch is not None and vmatch.group(1) == __version__: - downloader.to_screen('youtube-dl is up-to-date (' + __version__ + ')') + downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') return finally: urlh.close() @@ -3918,17 +4211,12 @@ def updateSelf(downloader, filename): except (IOError, OSError), err: sys.exit('ERROR: unable to overwrite current version') - downloader.to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.') + downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') def parseOpts(): - # Deferred imports - import getpass - import optparse - import shlex - - def _readOptions(filename): + def _readOptions(filename_bytes): try: - optionf = open(filename) + optionf = open(filename_bytes) except IOError: return [] # silently skip if file is not present try: @@ -4032,6 +4320,8 @@ def parseOpts(): action='store', dest='format', metavar='FORMAT', help='video format code') video_format.add_option('--all-formats', action='store_const', dest='format', help='download all available video formats', const='all') + video_format.add_option('--prefer-free-formats', + action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') video_format.add_option('--max-quality', action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') video_format.add_option('-F', '--list-formats', @@ -4065,6 +4355,8 @@ def parseOpts(): verbosity.add_option('--console-title', action='store_true', dest='consoletitle', help='display progress in console titlebar', default=False) + verbosity.add_option('-v', '--verbose', + action='store_true', dest='verbose', help='print various debugging information', default=False) filesystem.add_option('-t', '--title', @@ -4075,13 +4367,13 @@ def parseOpts(): action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False) filesystem.add_option('-o', '--output', - dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), and %% for a literal percent') + dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), and %% for a literal percent. Use - to output to stdout.') filesystem.add_option('-a', '--batch-file', dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') filesystem.add_option('-w', '--no-overwrites', action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) filesystem.add_option('-c', '--continue', - action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False) + action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True) filesystem.add_option('--no-continue', action='store_false', dest='continue_dl', help='do not resume partially downloaded files (restart from beginning)') @@ -4103,7 +4395,7 @@ def parseOpts(): postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False, help='convert video files to audio-only files (requires ffmpeg and ffprobe)') postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', - help='"best", "aac", "vorbis" or "mp3"; best by default') + help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default') postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K', help='ffmpeg audio bitrate specification, 128k by default') postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, @@ -4118,7 +4410,12 @@ def parseOpts(): parser.add_option_group(authentication) parser.add_option_group(postproc) - argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(os.path.expanduser('~/.youtube-dl.conf')) + sys.argv[1:] + xdg_config_home = os.environ.get('XDG_CONFIG_HOME') + if xdg_config_home: + userConf = os.path.join(xdg_config_home, 'youtube-dl.conf') + else: + userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') + argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:] opts, args = parser.parse_args(argv) return parser, opts, args @@ -4154,6 +4451,8 @@ def gen_extractors(): SoundcloudIE(), InfoQIE(), MixcloudIE(), + StanfordOpenClassroomIE(), + MTVIE(), GenericIE() ] @@ -4194,10 +4493,14 @@ def _real_main(): # General configuration cookie_processor = urllib2.HTTPCookieProcessor(jar) - opener = urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler()) + proxy_handler = urllib2.ProxyHandler() + opener = urllib2.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler()) urllib2.install_opener(opener) socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) + if opts.verbose: + print(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) + extractors = gen_extractors() if opts.list_extractors: @@ -4243,7 +4546,7 @@ def _real_main(): except (TypeError, ValueError), err: parser.error(u'invalid playlist end number specified') if opts.extractaudio: - if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis']: + if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']: parser.error(u'invalid audio format specified') # File downloader @@ -4290,6 +4593,8 @@ def _real_main(): 'matchtitle': opts.matchtitle, 'rejecttitle': opts.rejecttitle, 'max_downloads': opts.max_downloads, + 'prefer_free_formats': opts.prefer_free_formats, + 'verbose': opts.verbose, }) for extractor in extractors: fd.add_info_extractor(extractor) @@ -4308,7 +4613,12 @@ def _real_main(): parser.error(u'you must provide at least one URL') else: sys.exit() - retcode = fd.download(all_urls) + + try: + retcode = fd.download(all_urls) + except MaxDownloadsReached: + fd.to_screen(u'--max-download limit reached, aborting.') + retcode = 101 # Dump cookie jar if requested if opts.cookiefile is not None: