#!/usr/bin/env python
# -*- coding: utf-8 -*-
-# Author: Ricardo Garcia Gonzalez
-# Author: Danny Colligan
-# Author: Benjamin Johnson
-# Author: Vasyl' Vavrychuk
-# Author: Witold Baryluk
-# Author: Paweł Paprota
-# Author: Gergely Imreh
-# License: Public domain code
+
+__author__ = (
+ "Ricardo Garcia Gonzalez",
+ "Danny Colligan",
+ "Benjamin Johnson",
+ "Vasyl' Vavrychuk",
+ "Witold Baryluk",
+ "Paweł Paprota",
+ "Gergely Imreh",
+ )
+
+__license__ = "Public Domain"
+__version__ = '2011.08.04'
+
import cookielib
import ctypes
import datetime
from cgi import parse_qs
std_headers = {
- 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:2.0b11) Gecko/20100101 Firefox/4.0b11',
+ 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
else:
# Examine the reported length
if (content_length is not None and
- (resume_len - 100 < long(content_length) < resume_len + 100)):
+ (resume_len - 100 < long(content_length) < resume_len + 100)):
# The file had already been fully downloaded.
# Explanation to the above condition: in issue #175 it was revealed that
# YouTube sometimes adds or removes a few bytes from the end of the file,
class YoutubeIE(InfoExtractor):
"""Information extractor for youtube.com."""
- _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
+ _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
# upload date
upload_date = u'NA'
- mobj = re.search(r'id="eow-date".*?>(.*?)</span>', video_webpage, re.DOTALL)
+ mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
if mobj is not None:
upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
# Decide which formats to download
req_format = self._downloader.params.get('format', None)
- if 'fmt_url_map' in video_info:
- url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(','))
+ if 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
+ url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
+ url_data = [dict(pairStr.split('=') for pairStr in uds.split('&')) for uds in url_data_strs]
+ url_map = dict((ud['itag'], urllib.unquote(ud['url'])) for ud in url_data)
format_limit = self._downloader.params.get('format_limit', None)
if format_limit is not None and format_limit in self._available_formats:
format_list = self._available_formats[self._available_formats.index(format_limit):]
yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
yv_bitrate = '700' # according to Wikipedia this is hard-coded
request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
- '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
- '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
+ '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
+ '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
try:
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
video_ids = video_ids[playliststart:]
else:
video_ids = video_ids[playliststart:playlistend]
-
+
self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" %
- (username, all_ids_count, len(video_ids)))
+ (username, all_ids_count, len(video_ids)))
for video_id in video_ids:
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id)
# description
video_description = 'No description available.'
if (self._downloader.params.get('forcedescription', False) and
- 'description' in video_info):
+ 'description' in video_info):
video_description = video_info['description']
url_map = video_info['video_urls']
@staticmethod
def get_audio_codec(path):
- handle = subprocess.Popen(['ffprobe', '-show_streams', path],
- stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
- output = handle.communicate()[0]
- if handle.wait() != 0:
+ try:
+ cmd = ['ffprobe', '-show_streams', '--', path]
+ handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
+ output = handle.communicate()[0]
+ if handle.wait() != 0:
+ return None
+ except (IOError, OSError):
return None
audio_codec = None
for line in output.split('\n'):
@staticmethod
def run_ffmpeg(path, out_path, codec, more_opts):
try:
- ret = subprocess.call(['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + [out_path],
- stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
+ cmd = ['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + ['--', out_path]
+ ret = subprocess.call(cmd, stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
return (ret == 0)
except (IOError, OSError):
return False
filecodec = self.get_audio_codec(path)
if filecodec is None:
- self._downloader.to_stderr(u'WARNING: no audio codec found in file')
+ self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
return None
more_opts = []
status = self.run_ffmpeg(path, new_path, acodec, more_opts)
if not status:
- self._downloader.to_stderr(u'WARNING: error running ffmpeg' % ret)
+ self._downloader.to_stderr(u'WARNING: error running ffmpeg')
return None
try:
information['filepath'] = new_path
return information
-### MAIN PROGRAM ###
-if __name__ == '__main__':
+
+def updateSelf(downloader, filename):
+ ''' Update the program file with the latest version from the repository '''
+ # Note: downloader only used for options
+ if not os.access(filename, os.W_OK):
+ sys.exit('ERROR: no write permissions on %s' % filename)
+
+ downloader.to_screen('Updating to latest stable version...')
+
try:
- # Modules needed only when running the main program
- import getpass
- import optparse
+ latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
+ latest_version = urllib.urlopen(latest_url).read().strip()
+ prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
+ newcontent = urllib.urlopen(prog_url).read()
+ except (IOError, OSError), err:
+ sys.exit('ERROR: unable to download latest version')
- # Function to update the program file with the latest version from the repository.
- def update_self(downloader, filename):
- # Note: downloader only used for options
- if not os.access(filename, os.W_OK):
- sys.exit('ERROR: no write permissions on %s' % filename)
+ try:
+ stream = open(filename, 'w')
+ stream.write(newcontent)
+ stream.close()
+ except (IOError, OSError), err:
+ sys.exit('ERROR: unable to overwrite current version')
- downloader.to_screen('Updating to latest stable version...')
- try:
- latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
- latest_version = urllib.urlopen(latest_url).read().strip()
- prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
- newcontent = urllib.urlopen(prog_url).read()
- except (IOError, OSError), err:
- sys.exit('ERROR: unable to download latest version')
- try:
- stream = open(filename, 'w')
- stream.write(newcontent)
- stream.close()
- except (IOError, OSError), err:
- sys.exit('ERROR: unable to overwrite current version')
- downloader.to_screen('Updated to version %s' % latest_version)
-
- # Parse command line
- parser = optparse.OptionParser(
- usage='Usage: %prog [options] url...',
- version='2011.01.30',
- conflict_handler='resolve',
- )
-
- parser.add_option('-h', '--help',
- action='help', help='print this help text and exit')
- parser.add_option('-v', '--version',
- action='version', help='print program version and exit')
- parser.add_option('-U', '--update',
- action='store_true', dest='update_self', help='update this program to latest stable version')
- parser.add_option('-i', '--ignore-errors',
- action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
- parser.add_option('-r', '--rate-limit',
- dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
- parser.add_option('-R', '--retries',
- dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
- parser.add_option('--playlist-start',
- dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
- parser.add_option('--playlist-end',
- dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
- parser.add_option('--dump-user-agent',
- action='store_true', dest='dump_user_agent',
- help='display the current browser identification', default=False)
-
- authentication = optparse.OptionGroup(parser, 'Authentication Options')
- authentication.add_option('-u', '--username',
- dest='username', metavar='USERNAME', help='account username')
- authentication.add_option('-p', '--password',
- dest='password', metavar='PASSWORD', help='account password')
- authentication.add_option('-n', '--netrc',
- action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
- parser.add_option_group(authentication)
-
- video_format = optparse.OptionGroup(parser, 'Video Format Options')
- video_format.add_option('-f', '--format',
- action='store', dest='format', metavar='FORMAT', help='video format code')
- video_format.add_option('--all-formats',
- action='store_const', dest='format', help='download all available video formats', const='-1')
- video_format.add_option('--max-quality',
- action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
- parser.add_option_group(video_format)
-
- verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
- verbosity.add_option('-q', '--quiet',
- action='store_true', dest='quiet', help='activates quiet mode', default=False)
- verbosity.add_option('-s', '--simulate',
- action='store_true', dest='simulate', help='do not download video', default=False)
- verbosity.add_option('-g', '--get-url',
- action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
- verbosity.add_option('-e', '--get-title',
- action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
- verbosity.add_option('--get-thumbnail',
- action='store_true', dest='getthumbnail',
- help='simulate, quiet but print thumbnail URL', default=False)
- verbosity.add_option('--get-description',
- action='store_true', dest='getdescription',
- help='simulate, quiet but print video description', default=False)
- verbosity.add_option('--get-filename',
- action='store_true', dest='getfilename',
- help='simulate, quiet but print output filename', default=False)
- verbosity.add_option('--no-progress',
- action='store_true', dest='noprogress', help='do not print progress bar', default=False)
- verbosity.add_option('--console-title',
- action='store_true', dest='consoletitle',
- help='display progress in console titlebar', default=False)
- parser.add_option_group(verbosity)
-
- filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
- filesystem.add_option('-t', '--title',
- action='store_true', dest='usetitle', help='use title in file name', default=False)
- filesystem.add_option('-l', '--literal',
- action='store_true', dest='useliteral', help='use literal title in file name', default=False)
- filesystem.add_option('-A', '--auto-number',
- action='store_true', dest='autonumber',
- help='number downloaded files starting from 00000', default=False)
- filesystem.add_option('-o', '--output',
- dest='outtmpl', metavar='TEMPLATE', help='output filename template')
- filesystem.add_option('-a', '--batch-file',
- dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
- filesystem.add_option('-w', '--no-overwrites',
- action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
- filesystem.add_option('-c', '--continue',
- action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
- filesystem.add_option('--cookies',
- dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
- filesystem.add_option('--no-part',
- action='store_true', dest='nopart', help='do not use .part files', default=False)
- filesystem.add_option('--no-mtime',
- action='store_false', dest='updatetime',
- help='do not use the Last-modified header to set the file modification time', default=True)
- parser.add_option_group(filesystem)
-
- postproc = optparse.OptionGroup(parser, 'Post-processing Options')
- postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
- help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
- postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
- help='"best", "aac" or "mp3"; best by default')
- parser.add_option_group(postproc)
-
- (opts, args) = parser.parse_args()
+ downloader.to_screen('Updated to version %s' % latest_version)
+
+
+def parseOpts():
+ # Deferred imports
+ import getpass
+ import optparse
+
+ def _format_option_string(option):
+ ''' ('-o', '--option') -> -o, --format METAVAR'''
+
+ opts = []
+
+ if option._short_opts: opts.append(option._short_opts[0])
+ if option._long_opts: opts.append(option._long_opts[0])
+ if len(opts) > 1: opts.insert(1, ', ')
+
+ if option.takes_value(): opts.append(' %s' % option.metavar)
+
+ return "".join(opts)
+
+ max_width = 80
+ max_help_position = 80
+
+ # No need to wrap help messages if we're on a wide console
+ columns = os.environ.get('COLUMNS', None)
+ if columns: max_width = columns
+
+ fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
+ fmt.format_option_strings = _format_option_string
+
+ kw = {
+ 'version' : __version__,
+ 'formatter' : fmt,
+ 'usage' : 'Usage : %prog [options] url...',
+ 'conflict_handler' : 'resolve',
+ }
+
+ parser = optparse.OptionParser(**kw)
+
+ # option groups
+ general = optparse.OptionGroup(parser, 'General Options')
+ authentication = optparse.OptionGroup(parser, 'Authentication Options')
+ video_format = optparse.OptionGroup(parser, 'Video Format Options')
+ postproc = optparse.OptionGroup(parser, 'Post-processing Options')
+ filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
+ verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
+
+ general.add_option('-h', '--help',
+ action='help', help='print this help text and exit')
+ general.add_option('-v', '--version',
+ action='version', help='print program version and exit')
+ general.add_option('-U', '--update',
+ action='store_true', dest='update_self', help='update this program to latest stable version')
+ general.add_option('-i', '--ignore-errors',
+ action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
+ general.add_option('-r', '--rate-limit',
+ dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
+ general.add_option('-R', '--retries',
+ dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
+ general.add_option('--playlist-start',
+ dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
+ general.add_option('--playlist-end',
+ dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
+ general.add_option('--dump-user-agent',
+ action='store_true', dest='dump_user_agent',
+ help='display the current browser identification', default=False)
+
+ authentication.add_option('-u', '--username',
+ dest='username', metavar='USERNAME', help='account username')
+ authentication.add_option('-p', '--password',
+ dest='password', metavar='PASSWORD', help='account password')
+ authentication.add_option('-n', '--netrc',
+ action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
+
+
+ video_format.add_option('-f', '--format',
+ action='store', dest='format', metavar='FORMAT', help='video format code')
+ video_format.add_option('--all-formats',
+ action='store_const', dest='format', help='download all available video formats', const='-1')
+ video_format.add_option('--max-quality',
+ action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
+
+
+ verbosity.add_option('-q', '--quiet',
+ action='store_true', dest='quiet', help='activates quiet mode', default=False)
+ verbosity.add_option('-s', '--simulate',
+ action='store_true', dest='simulate', help='do not download video', default=False)
+ verbosity.add_option('-g', '--get-url',
+ action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
+ verbosity.add_option('-e', '--get-title',
+ action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
+ verbosity.add_option('--get-thumbnail',
+ action='store_true', dest='getthumbnail',
+ help='simulate, quiet but print thumbnail URL', default=False)
+ verbosity.add_option('--get-description',
+ action='store_true', dest='getdescription',
+ help='simulate, quiet but print video description', default=False)
+ verbosity.add_option('--get-filename',
+ action='store_true', dest='getfilename',
+ help='simulate, quiet but print output filename', default=False)
+ verbosity.add_option('--no-progress',
+ action='store_true', dest='noprogress', help='do not print progress bar', default=False)
+ verbosity.add_option('--console-title',
+ action='store_true', dest='consoletitle',
+ help='display progress in console titlebar', default=False)
+
+
+ filesystem.add_option('-t', '--title',
+ action='store_true', dest='usetitle', help='use title in file name', default=False)
+ filesystem.add_option('-l', '--literal',
+ action='store_true', dest='useliteral', help='use literal title in file name', default=False)
+ filesystem.add_option('-A', '--auto-number',
+ action='store_true', dest='autonumber',
+ help='number downloaded files starting from 00000', default=False)
+ filesystem.add_option('-o', '--output',
+ dest='outtmpl', metavar='TEMPLATE', help='output filename template')
+ filesystem.add_option('-a', '--batch-file',
+ dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
+ filesystem.add_option('-w', '--no-overwrites',
+ action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
+ filesystem.add_option('-c', '--continue',
+ action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
+ filesystem.add_option('--cookies',
+ dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
+ filesystem.add_option('--no-part',
+ action='store_true', dest='nopart', help='do not use .part files', default=False)
+ filesystem.add_option('--no-mtime',
+ action='store_false', dest='updatetime',
+ help='do not use the Last-modified header to set the file modification time', default=True)
+
+
+ postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
+ help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
+ postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
+ help='"best", "aac" or "mp3"; best by default')
+
+
+ parser.add_option_group(general)
+ parser.add_option_group(filesystem)
+ parser.add_option_group(verbosity)
+ parser.add_option_group(video_format)
+ parser.add_option_group(authentication)
+ parser.add_option_group(postproc)
+
+ opts, args = parser.parse_args()
+
+ return parser, opts, args
+
+
+if __name__ == '__main__':
+ try:
+ parser, opts, args = parseOpts()
# Open appropriate CookieJar
if opts.cookiefile is None:
# Update version
if opts.update_self:
- update_self(fd, sys.argv[0])
+ updateSelf(fd, sys.argv[0])
# Maybe do nothing
if len(all_urls) < 1:
sys.exit(u'ERROR: fixed output name but more than one file to download')
except KeyboardInterrupt:
sys.exit(u'\nERROR: Interrupted by user')
+
+# vim: set ts=4 sw=4 sts=4 noet ai si filetype=python:
git://git.jankratochvil.net / youtube-dl.git / blobdiff