youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # License: Public domain code
   6 import htmlentitydefs
   7 import httplib
   8 import locale
   9 import math
  10 import netrc
  11 import os
  12 import os.path
  13 import re
  14 import socket
  15 import string
  16 import sys
  17 import time
  18 import urllib
  19 import urllib2
  20
  21 std_headers = {
  22         'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8',
  23         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  24         'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
  25         'Accept-Language': 'en-us,en;q=0.5',
  26 }
  27
  28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  29
  30 class DownloadError(Exception):
  31         """Download Error exception.
  32
  33         This exception may be thrown by FileDownloader objects if they are not
  34         configured to continue on errors. They will contain the appropriate
  35         error message.
  36         """
  37         pass
  38
  39 class SameFileError(Exception):
  40         """Same File exception.
  41
  42         This exception will be thrown by FileDownloader objects if they detect
  43         multiple files would have to be downloaded to the same file on disk.
  44         """
  45         pass
  46
  47 class PostProcessingError(Exception):
  48         """Post Processing exception.
  49
  50         This exception may be raised by PostProcessor's .run() method to
  51         indicate an error in the postprocessing task.
  52         """
  53         pass
  54
  55 class UnavailableFormatError(Exception):
  56         """Unavailable Format exception.
  57
  58         This exception will be thrown when a video is requested
  59         in a format that is not available for that video.
  60         """
  61         pass
  62
  63 class ContentTooShortError(Exception):
  64         """Content Too Short exception.
  65
  66         This exception may be raised by FileDownloader objects when a file they
  67         download is too small for what the server announced first, indicating
  68         the connection was probably interrupted.
  69         """
  70         # Both in bytes
  71         downloaded = None
  72         expected = None
  73
  74         def __init__(self, downloaded, expected):
  75                 self.downloaded = downloaded
  76                 self.expected = expected
  77
  78 class FileDownloader(object):
  79         """File Downloader class.
  80
  81         File downloader objects are the ones responsible of downloading the
  82         actual video file and writing it to disk if the user has requested
  83         it, among some other tasks. In most cases there should be one per
  84         program. As, given a video URL, the downloader doesn't know how to
  85         extract all the needed information, task that InfoExtractors do, it
  86         has to pass the URL to one of them.
  87
  88         For this, file downloader objects have a method that allows
  89         InfoExtractors to be registered in a given order. When it is passed
  90         a URL, the file downloader handles it to the first InfoExtractor it
  91         finds that reports being able to handle it. The InfoExtractor extracts
  92         all the information about the video or videos the URL refers to, and
  93         asks the FileDownloader to process the video information, possibly
  94         downloading the video.
  95
  96         File downloaders accept a lot of parameters. In order not to saturate
  97         the object constructor with arguments, it receives a dictionary of
  98         options instead. These options are available through the params
  99         attribute for the InfoExtractors to use. The FileDownloader also
 100         registers itself as the downloader in charge for the InfoExtractors
 101         that are added to it, so this is a "mutual registration".
 102
 103         Available options:
 104
 105         username:       Username for authentication purposes.
 106         password:       Password for authentication purposes.
 107         usenetrc:       Use netrc for authentication instead.
 108         quiet:          Do not print messages to stdout.
 109         forceurl:       Force printing final URL.
 110         forcetitle:     Force printing title.
 111         simulate:       Do not download the video files.
 112         format:         Video format code.
 113         outtmpl:        Template for output names.
 114         ignoreerrors:   Do not stop on download errors.
 115         ratelimit:      Download speed limit, in bytes/sec.
 116         nooverwrites:   Prevent overwriting files.
 117         """
 118
 119         params = None
 120         _ies = []
 121         _pps = []
 122         _download_retcode = None
 123
 124         def __init__(self, params):
 125                 """Create a FileDownloader object with the given options."""
 126                 self._ies = []
 127                 self._pps = []
 128                 self._download_retcode = 0
 129                 self.params = params
 130
 131         @staticmethod
 132         def pmkdir(filename):
 133                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 134                 components = filename.split(os.sep)
 135                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 136                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 137                 for dir in aggregate:
 138                         if not os.path.exists(dir):
 139                                 os.mkdir(dir)
 140
 141         @staticmethod
 142         def format_bytes(bytes):
 143                 if bytes is None:
 144                         return 'N/A'
 145                 if bytes == 0:
 146                         exponent = 0
 147                 else:
 148                         exponent = long(math.log(float(bytes), 1024.0))
 149                 suffix = 'bkMGTPEZY'[exponent]
 150                 converted = float(bytes) / float(1024**exponent)
 151                 return '%.2f%s' % (converted, suffix)
 152
 153         @staticmethod
 154         def calc_percent(byte_counter, data_len):
 155                 if data_len is None:
 156                         return '---.-%'
 157                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 158
 159         @staticmethod
 160         def calc_eta(start, now, total, current):
 161                 if total is None:
 162                         return '--:--'
 163                 dif = now - start
 164                 if current == 0 or dif < 0.001: # One millisecond
 165                         return '--:--'
 166                 rate = float(current) / dif
 167                 eta = long((float(total) - float(current)) / rate)
 168                 (eta_mins, eta_secs) = divmod(eta, 60)
 169                 if eta_mins > 99:
 170                         return '--:--'
 171                 return '%02d:%02d' % (eta_mins, eta_secs)
 172
 173         @staticmethod
 174         def calc_speed(start, now, bytes):
 175                 dif = now - start
 176                 if bytes == 0 or dif < 0.001: # One millisecond
 177                         return '%10s' % '---b/s'
 178                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 179
 180         @staticmethod
 181         def best_block_size(elapsed_time, bytes):
 182                 new_min = max(bytes / 2.0, 1.0)
 183                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 184                 if elapsed_time < 0.001:
 185                         return long(new_max)
 186                 rate = bytes / elapsed_time
 187                 if rate > new_max:
 188                         return long(new_max)
 189                 if rate < new_min:
 190                         return long(new_min)
 191                 return long(rate)
 192
 193         @staticmethod
 194         def parse_bytes(bytestr):
 195                 """Parse a string indicating a byte quantity into a long integer."""
 196                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 197                 if matchobj is None:
 198                         return None
 199                 number = float(matchobj.group(1))
 200                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 201                 return long(round(number * multiplier))
 202
 203         @staticmethod
 204         def verify_url(url):
 205                 """Verify a URL is valid and data could be downloaded."""
 206                 request = urllib2.Request(url, None, std_headers)
 207                 data = urllib2.urlopen(request)
 208                 data.read(1)
 209                 data.close()
 210
 211         def add_info_extractor(self, ie):
 212                 """Add an InfoExtractor object to the end of the list."""
 213                 self._ies.append(ie)
 214                 ie.set_downloader(self)
 215
 216         def add_post_processor(self, pp):
 217                 """Add a PostProcessor object to the end of the chain."""
 218                 self._pps.append(pp)
 219                 pp.set_downloader(self)
 220
 221         def to_stdout(self, message, skip_eol=False):
 222                 """Print message to stdout if not in quiet mode."""
 223                 if not self.params.get('quiet', False):
 224                         print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(locale.getpreferredencoding()),
 225                         sys.stdout.flush()
 226
 227         def to_stderr(self, message):
 228                 """Print message to stderr."""
 229                 print >>sys.stderr, message
 230
 231         def fixed_template(self):
 232                 """Checks if the output template is fixed."""
 233                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 234
 235         def trouble(self, message=None):
 236                 """Determine action to take when a download problem appears.
 237
 238                 Depending on if the downloader has been configured to ignore
 239                 download errors or not, this method may throw an exception or
 240                 not when errors are found, after printing the message.
 241                 """
 242                 if message is not None:
 243                         self.to_stderr(message)
 244                 if not self.params.get('ignoreerrors', False):
 245                         raise DownloadError(message)
 246                 self._download_retcode = 1
 247
 248         def slow_down(self, start_time, byte_counter):
 249                 """Sleep if the download speed is over the rate limit."""
 250                 rate_limit = self.params.get('ratelimit', None)
 251                 if rate_limit is None or byte_counter == 0:
 252                         return
 253                 now = time.time()
 254                 elapsed = now - start_time
 255                 if elapsed <= 0.0:
 256                         return
 257                 speed = float(byte_counter) / elapsed
 258                 if speed > rate_limit:
 259                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 260
 261         def report_destination(self, filename):
 262                 """Report destination filename."""
 263                 self.to_stdout(u'[download] Destination: %s' % filename)
 264
 265         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 266                 """Report download progress."""
 267                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 268                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 269
 270         def report_resuming_byte(self, resume_len):
 271                 """Report attemtp to resume at given byte."""
 272                 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
 273
 274         def report_file_already_downloaded(self, file_name):
 275                 """Report file has already been fully downloaded."""
 276                 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
 277
 278         def report_unable_to_resume(self):
 279                 """Report it was impossible to resume download."""
 280                 self.to_stdout(u'[download] Unable to resume')
 281
 282         def report_finish(self):
 283                 """Report download finished."""
 284                 self.to_stdout(u'')
 285
 286         def process_info(self, info_dict):
 287                 """Process a single dictionary returned by an InfoExtractor."""
 288                 # Do nothing else if in simulate mode
 289                 if self.params.get('simulate', False):
 290                         try:
 291                                 self.verify_url(info_dict['url'])
 292                         except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
 293                                 raise UnavailableFormatError
 294
 295                         # Forced printings
 296                         if self.params.get('forcetitle', False):
 297                                 print info_dict['title'].encode(locale.getpreferredencoding())
 298                         if self.params.get('forceurl', False):
 299                                 print info_dict['url'].encode(locale.getpreferredencoding())
 300
 301                         return
 302
 303                 try:
 304                         template_dict = dict(info_dict)
 305                         template_dict['epoch'] = unicode(long(time.time()))
 306                         filename = self.params['outtmpl'] % template_dict
 307                         self.report_destination(filename)
 308                 except (ValueError, KeyError), err:
 309                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 310                 if self.params['nooverwrites'] and os.path.exists(filename):
 311                         self.to_stderr('WARNING: file exists: %s; skipping' % filename)
 312                         return
 313
 314                 try:
 315                         self.pmkdir(filename)
 316                 except (OSError, IOError), err:
 317                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 318                         return
 319
 320                 try:
 321                         outstream = open(filename, 'ab')
 322                 except (OSError, IOError), err:
 323                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 324                         return
 325
 326                 try:
 327                         self._do_download(outstream, info_dict['url'])
 328                         outstream.close()
 329                 except (OSError, IOError), err:
 330                         outstream.close()
 331                         os.remove(filename)
 332                         raise UnavailableFormatError
 333                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 334                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 335                         return
 336                 except (ContentTooShortError, ), err:
 337                         self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 338                         return
 339
 340                 try:
 341                         self.post_process(filename, info_dict)
 342                 except (PostProcessingError), err:
 343                         self.trouble('ERROR: postprocessing: %s' % str(err))
 344                         return
 345
 346         def download(self, url_list):
 347                 """Download a given list of URLs."""
 348                 if len(url_list) > 1 and self.fixed_template():
 349                         raise SameFileError(self.params['outtmpl'])
 350
 351                 for url in url_list:
 352                         suitable_found = False
 353                         for ie in self._ies:
 354                                 # Go to next InfoExtractor if not suitable
 355                                 if not ie.suitable(url):
 356                                         continue
 357
 358                                 # Suitable InfoExtractor found
 359                                 suitable_found = True
 360
 361                                 # Extract information from URL and process it
 362                                 ie.extract(url)
 363
 364                                 # Suitable InfoExtractor had been found; go to next URL
 365                                 break
 366
 367                         if not suitable_found:
 368                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 369
 370                 return self._download_retcode
 371
 372         def post_process(self, filename, ie_info):
 373                 """Run the postprocessing chain on the given file."""
 374                 info = dict(ie_info)
 375                 info['filepath'] = filename
 376                 for pp in self._pps:
 377                         info = pp.run(info)
 378                         if info is None:
 379                                 break
 380
 381         def _do_download(self, stream, url):
 382                 basic_request = urllib2.Request(url, None, std_headers)
 383                 request = urllib2.Request(url, None, std_headers)
 384
 385                 # Resume transfer if filesize is non-zero
 386                 resume_len = stream.tell()
 387                 if self.params['continuedl'] and resume_len != 0:
 388                         self.report_resuming_byte(resume_len)
 389                         request.add_header('Range','bytes=%d-' % resume_len)
 390                 else:
 391                         stream.close()
 392                         stream = open(stream.name,'wb')
 393                 try:
 394                         data = urllib2.urlopen(request)
 395                 except urllib2.HTTPError, e:
 396                         if not e.code == 416: #  416 is 'Requested range not satisfiable'
 397                                 raise
 398                         data = urllib2.urlopen(basic_request)
 399                         content_length = data.info()['Content-Length']
 400                         if content_length is not None and long(content_length) == resume_len:
 401                                 self.report_file_already_downloaded(stream.name)
 402                                 return
 403                         else:
 404                                 self.report_unable_to_resume()
 405                                 stream.close()
 406                                 stream = open(stream.name,'wb')
 407
 408                 data_len = data.info().get('Content-length', None)
 409                 data_len_str = self.format_bytes(data_len)
 410                 byte_counter = 0
 411                 block_size = 1024
 412                 start = time.time()
 413                 while True:
 414                         # Progress message
 415                         percent_str = self.calc_percent(byte_counter, data_len)
 416                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 417                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 418                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 419
 420                         # Download and write
 421                         before = time.time()
 422                         data_block = data.read(block_size)
 423                         after = time.time()
 424                         data_block_len = len(data_block)
 425                         if data_block_len == 0:
 426                                 break
 427                         byte_counter += data_block_len
 428                         stream.write(data_block)
 429                         block_size = self.best_block_size(after - before, data_block_len)
 430
 431                         # Apply rate limit
 432                         self.slow_down(start, byte_counter)
 433
 434                 self.report_finish()
 435                 if data_len is not None and str(byte_counter) != data_len:
 436                         raise ContentTooShortError(byte_counter, long(data_len))
 437
 438 class InfoExtractor(object):
 439         """Information Extractor class.
 440
 441         Information extractors are the classes that, given a URL, extract
 442         information from the video (or videos) the URL refers to. This
 443         information includes the real video URL, the video title and simplified
 444         title, author and others. The information is stored in a dictionary
 445         which is then passed to the FileDownloader. The FileDownloader
 446         processes this information possibly downloading the video to the file
 447         system, among other possible outcomes. The dictionaries must include
 448         the following fields:
 449
 450         id:             Video identifier.
 451         url:            Final video URL.
 452         uploader:       Nickname of the video uploader.
 453         title:          Literal title.
 454         stitle:         Simplified title.
 455         ext:            Video filename extension.
 456
 457         Subclasses of this one should re-define the _real_initialize() and
 458         _real_extract() methods, as well as the suitable() static method.
 459         Probably, they should also be instantiated and added to the main
 460         downloader.
 461         """
 462
 463         _ready = False
 464         _downloader = None
 465
 466         def __init__(self, downloader=None):
 467                 """Constructor. Receives an optional downloader."""
 468                 self._ready = False
 469                 self.set_downloader(downloader)
 470
 471         @staticmethod
 472         def suitable(url):
 473                 """Receives a URL and returns True if suitable for this IE."""
 474                 return False
 475
 476         def initialize(self):
 477                 """Initializes an instance (authentication, etc)."""
 478                 if not self._ready:
 479                         self._real_initialize()
 480                         self._ready = True
 481
 482         def extract(self, url):
 483                 """Extracts URL information and returns it in list of dicts."""
 484                 self.initialize()
 485                 return self._real_extract(url)
 486
 487         def set_downloader(self, downloader):
 488                 """Sets the downloader for this IE."""
 489                 self._downloader = downloader
 490
 491         def _real_initialize(self):
 492                 """Real initialization process. Redefine in subclasses."""
 493                 pass
 494
 495         def _real_extract(self, url):
 496                 """Real extraction process. Redefine in subclasses."""
 497                 pass
 498
 499 class YoutubeIE(InfoExtractor):
 500         """Information extractor for youtube.com."""
 501
 502         _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
 503         _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 504         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 505         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 506         _NETRC_MACHINE = 'youtube'
 507         _available_formats = ['22', '35', '18', '17', '13'] # listed in order of priority for -b flag
 508         _video_extensions = {
 509                 '13': '3gp',
 510                 '17': 'mp4',
 511                 '18': 'mp4',
 512                 '22': 'mp4',
 513         }
 514
 515         @staticmethod
 516         def suitable(url):
 517                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 518
 519         @staticmethod
 520         def htmlentity_transform(matchobj):
 521                 """Transforms an HTML entity to a Unicode character."""
 522                 entity = matchobj.group(1)
 523
 524                 # Known non-numeric HTML entity
 525                 if entity in htmlentitydefs.name2codepoint:
 526                         return unichr(htmlentitydefs.name2codepoint[entity])
 527
 528                 # Unicode character
 529                 mobj = re.match(ur'(?u)#(x?\d+)', entity)
 530                 if mobj is not None:
 531                         numstr = mobj.group(1)
 532                         if numstr.startswith(u'x'):
 533                                 base = 16
 534                                 numstr = u'0%s' % numstr
 535                         else:
 536                                 base = 10
 537                         return unichr(long(numstr, base))
 538
 539                 # Unknown entity in name, return its literal representation
 540                 return (u'&%s;' % entity)
 541
 542         def report_lang(self):
 543                 """Report attempt to set language."""
 544                 self._downloader.to_stdout(u'[youtube] Setting language')
 545
 546         def report_login(self):
 547                 """Report attempt to log in."""
 548                 self._downloader.to_stdout(u'[youtube] Logging in')
 549
 550         def report_age_confirmation(self):
 551                 """Report attempt to confirm age."""
 552                 self._downloader.to_stdout(u'[youtube] Confirming age')
 553
 554         def report_webpage_download(self, video_id):
 555                 """Report attempt to download webpage."""
 556                 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
 557
 558         def report_information_extraction(self, video_id):
 559                 """Report attempt to extract video information."""
 560                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 561
 562         def report_video_url(self, video_id, video_real_url):
 563                 """Report extracted video URL."""
 564                 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
 565
 566         def report_unavailable_format(self, video_id, format):
 567                 """Report extracted video URL."""
 568                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 569
 570         def _real_initialize(self):
 571                 if self._downloader is None:
 572                         return
 573
 574                 username = None
 575                 password = None
 576                 downloader_params = self._downloader.params
 577
 578                 # Attempt to use provided username and password or .netrc data
 579                 if downloader_params.get('username', None) is not None:
 580                         username = downloader_params['username']
 581                         password = downloader_params['password']
 582                 elif downloader_params.get('usenetrc', False):
 583                         try:
 584                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 585                                 if info is not None:
 586                                         username = info[0]
 587                                         password = info[2]
 588                                 else:
 589                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 590                         except (IOError, netrc.NetrcParseError), err:
 591                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 592                                 return
 593
 594                 # Set language
 595                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 596                 try:
 597                         self.report_lang()
 598                         urllib2.urlopen(request).read()
 599                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 600                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 601                         return
 602
 603                 # No authentication to be performed
 604                 if username is None:
 605                         return
 606
 607                 # Log in
 608                 login_form = {
 609                                 'current_form': 'loginForm',
 610                                 'next':         '/',
 611                                 'action_login': 'Log In',
 612                                 'username':     username,
 613                                 'password':     password,
 614                                 }
 615                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 616                 try:
 617                         self.report_login()
 618                         login_results = urllib2.urlopen(request).read()
 619                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 620                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 621                                 return
 622                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 623                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 624                         return
 625
 626                 # Confirm age
 627                 age_form = {
 628                                 'next_url':             '/',
 629                                 'action_confirm':       'Confirm',
 630                                 }
 631                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 632                 try:
 633                         self.report_age_confirmation()
 634                         age_results = urllib2.urlopen(request).read()
 635                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 636                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 637                         return
 638
 639         def _real_extract(self, url):
 640                 # Extract video id from URL
 641                 mobj = re.match(self._VALID_URL, url)
 642                 if mobj is None:
 643                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 644                         return
 645                 video_id = mobj.group(2)
 646
 647                 # Downloader parameters
 648                 best_quality = False
 649                 format_param = None
 650                 quality_index = 0
 651                 if self._downloader is not None:
 652                         params = self._downloader.params
 653                         format_param = params.get('format', None)
 654                         if format_param == '0':
 655                                 format_param = self._available_formats[quality_index]
 656                                 best_quality = True
 657
 658                 while True:
 659                         # Extension
 660                         video_extension = self._video_extensions.get(format_param, 'flv')
 661
 662                         # Normalize URL, including format
 663                         normalized_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id
 664                         if format_param is not None:
 665                                 normalized_url = '%s&fmt=%s' % (normalized_url, format_param)
 666                         request = urllib2.Request(normalized_url, None, std_headers)
 667                         try:
 668                                 self.report_webpage_download(video_id)
 669                                 video_webpage = urllib2.urlopen(request).read()
 670                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 671                                 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 672                                 return
 673                         self.report_information_extraction(video_id)
 674
 675                         # "t" param
 676                         mobj = re.search(r', "t": "([^"]+)"', video_webpage)
 677                         if mobj is None:
 678                                 self._downloader.trouble(u'ERROR: unable to extract "t" parameter')
 679                                 return
 680                         video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&el=detailpage&ps=' % (video_id, mobj.group(1))
 681                         if format_param is not None:
 682                                 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 683                         self.report_video_url(video_id, video_real_url)
 684
 685                         # uploader
 686                         mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
 687                         if mobj is None:
 688                                 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 689                                 return
 690                         video_uploader = mobj.group(1)
 691
 692                         # title
 693                         mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>', video_webpage)
 694                         if mobj is None:
 695                                 self._downloader.trouble(u'ERROR: unable to extract video title')
 696                                 return
 697                         video_title = mobj.group(1).decode('utf-8')
 698                         video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
 699                         video_title = video_title.replace(os.sep, u'%')
 700
 701                         # simplified title
 702                         simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 703                         simple_title = simple_title.strip(ur'_')
 704
 705                         try:
 706                                 # Process video information
 707                                 self._downloader.process_info({
 708                                         'id':           video_id.decode('utf-8'),
 709                                         'url':          video_real_url.decode('utf-8'),
 710                                         'uploader':     video_uploader.decode('utf-8'),
 711                                         'title':        video_title,
 712                                         'stitle':       simple_title,
 713                                         'ext':          video_extension.decode('utf-8'),
 714                                 })
 715
 716                                 return
 717
 718                         except UnavailableFormatError, err:
 719                                 if best_quality:
 720                                         if quality_index == len(self._available_formats) - 1:
 721                                                 # I don't ever expect this to happen
 722                                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 723                                                 return
 724                                         else:
 725                                                 self.report_unavailable_format(video_id, format_param)
 726                                                 quality_index += 1
 727                                                 format_param = self._available_formats[quality_index]
 728                                                 continue
 729                                 else:
 730                                         self._downloader.trouble('ERROR: format not available for video')
 731                                         return
 732
 733
 734 class MetacafeIE(InfoExtractor):
 735         """Information Extractor for metacafe.com."""
 736
 737         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 738         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 739         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 740         _youtube_ie = None
 741
 742         def __init__(self, youtube_ie, downloader=None):
 743                 InfoExtractor.__init__(self, downloader)
 744                 self._youtube_ie = youtube_ie
 745
 746         @staticmethod
 747         def suitable(url):
 748                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 749
 750         def report_disclaimer(self):
 751                 """Report disclaimer retrieval."""
 752                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 753
 754         def report_age_confirmation(self):
 755                 """Report attempt to confirm age."""
 756                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 757
 758         def report_download_webpage(self, video_id):
 759                 """Report webpage download."""
 760                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
 761
 762         def report_extraction(self, video_id):
 763                 """Report information extraction."""
 764                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
 765
 766         def _real_initialize(self):
 767                 # Retrieve disclaimer
 768                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
 769                 try:
 770                         self.report_disclaimer()
 771                         disclaimer = urllib2.urlopen(request).read()
 772                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 773                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 774                         return
 775
 776                 # Confirm age
 777                 disclaimer_form = {
 778                         'filters': '0',
 779                         'submit': "Continue - I'm over 18",
 780                         }
 781                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
 782                 try:
 783                         self.report_age_confirmation()
 784                         disclaimer = urllib2.urlopen(request).read()
 785                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 786                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 787                         return
 788
 789         def _real_extract(self, url):
 790                 # Extract id and simplified title from URL
 791                 mobj = re.match(self._VALID_URL, url)
 792                 if mobj is None:
 793                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 794                         return
 795
 796                 video_id = mobj.group(1)
 797
 798                 # Check if video comes from YouTube
 799                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 800                 if mobj2 is not None:
 801                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
 802                         return
 803
 804                 simple_title = mobj.group(2).decode('utf-8')
 805                 video_extension = 'flv'
 806
 807                 # Retrieve video webpage to extract further information
 808                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 809                 try:
 810                         self.report_download_webpage(video_id)
 811                         webpage = urllib2.urlopen(request).read()
 812                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 813                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 814                         return
 815
 816                 # Extract URL, uploader and title from webpage
 817                 self.report_extraction(video_id)
 818                 mobj = re.search(r'(?m)&mediaURL=(http.*?\.flv)', webpage)
 819                 if mobj is None:
 820                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 821                         return
 822                 mediaURL = urllib.unquote(mobj.group(1))
 823
 824                 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 825                 if mobj is None:
 826                         self._downloader.trouble(u'ERROR: unable to extract gdaKey')
 827                         return
 828                 gdaKey = mobj.group(1)
 829
 830                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 831
 832                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 833                 if mobj is None:
 834                         self._downloader.trouble(u'ERROR: unable to extract title')
 835                         return
 836                 video_title = mobj.group(1).decode('utf-8')
 837
 838                 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
 839                 if mobj is None:
 840                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 841                         return
 842                 video_uploader = mobj.group(1)
 843
 844                 try:
 845                         # Process video information
 846                         self._downloader.process_info({
 847                                 'id':           video_id.decode('utf-8'),
 848                                 'url':          video_url.decode('utf-8'),
 849                                 'uploader':     video_uploader.decode('utf-8'),
 850                                 'title':        video_title,
 851                                 'stitle':       simple_title,
 852                                 'ext':          video_extension.decode('utf-8'),
 853                         })
 854                 except UnavailableFormatError:
 855                         self._downloader.trouble(u'ERROR: format not available for video')
 856
 857
 858 class YoutubeSearchIE(InfoExtractor):
 859         """Information Extractor for YouTube search queries."""
 860         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
 861         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
 862         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
 863         _MORE_PAGES_INDICATOR = r'>Next</a>'
 864         _youtube_ie = None
 865         _max_youtube_results = 1000
 866
 867         def __init__(self, youtube_ie, downloader=None):
 868                 InfoExtractor.__init__(self, downloader)
 869                 self._youtube_ie = youtube_ie
 870
 871         @staticmethod
 872         def suitable(url):
 873                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
 874
 875         def report_download_page(self, query, pagenum):
 876                 """Report attempt to download playlist page with given number."""
 877                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
 878
 879         def _real_initialize(self):
 880                 self._youtube_ie.initialize()
 881
 882         def _real_extract(self, query):
 883                 mobj = re.match(self._VALID_QUERY, query)
 884                 if mobj is None:
 885                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
 886                         return
 887
 888                 prefix, query = query.split(':')
 889                 prefix = prefix[8:]
 890                 if prefix == '':
 891                         self._download_n_results(query, 1)
 892                         return
 893                 elif prefix == 'all':
 894                         self._download_n_results(query, self._max_youtube_results)
 895                         return
 896                 else:
 897                         try:
 898                                 n = long(prefix)
 899                                 if n <= 0:
 900                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
 901                                         return
 902                                 elif n > self._max_youtube_results:
 903                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
 904                                         n = self._max_youtube_results
 905                                 self._download_n_results(query, n)
 906                                 return
 907                         except ValueError: # parsing prefix as integer fails
 908                                 self._download_n_results(query, 1)
 909                                 return
 910
 911         def _download_n_results(self, query, n):
 912                 """Downloads a specified number of results for a query"""
 913
 914                 video_ids = []
 915                 already_seen = set()
 916                 pagenum = 1
 917
 918                 while True:
 919                         self.report_download_page(query, pagenum)
 920                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
 921                         request = urllib2.Request(result_url, None, std_headers)
 922                         try:
 923                                 page = urllib2.urlopen(request).read()
 924                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 925                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
 926                                 return
 927
 928                         # Extract video identifiers
 929                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
 930                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
 931                                 if video_id not in already_seen:
 932                                         video_ids.append(video_id)
 933                                         already_seen.add(video_id)
 934                                         if len(video_ids) == n:
 935                                                 # Specified n videos reached
 936                                                 for id in video_ids:
 937                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 938                                                 return
 939
 940                         if self._MORE_PAGES_INDICATOR not in page:
 941                                 for id in video_ids:
 942                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 943                                 return
 944
 945                         pagenum = pagenum + 1
 946
 947 class YoutubePlaylistIE(InfoExtractor):
 948         """Information Extractor for YouTube playlists."""
 949
 950         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/view_play_list\?p=(.+)'
 951         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
 952         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
 953         _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&amp;page=%s'
 954         _youtube_ie = None
 955
 956         def __init__(self, youtube_ie, downloader=None):
 957                 InfoExtractor.__init__(self, downloader)
 958                 self._youtube_ie = youtube_ie
 959
 960         @staticmethod
 961         def suitable(url):
 962                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
 963
 964         def report_download_page(self, playlist_id, pagenum):
 965                 """Report attempt to download playlist page with given number."""
 966                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
 967
 968         def _real_initialize(self):
 969                 self._youtube_ie.initialize()
 970
 971         def _real_extract(self, url):
 972                 # Extract playlist id
 973                 mobj = re.match(self._VALID_URL, url)
 974                 if mobj is None:
 975                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
 976                         return
 977
 978                 # Download playlist pages
 979                 playlist_id = mobj.group(1)
 980                 video_ids = []
 981                 pagenum = 1
 982
 983                 while True:
 984                         self.report_download_page(playlist_id, pagenum)
 985                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
 986                         try:
 987                                 page = urllib2.urlopen(request).read()
 988                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 989                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
 990                                 return
 991
 992                         # Extract video identifiers
 993                         ids_in_page = []
 994                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
 995                                 if mobj.group(1) not in ids_in_page:
 996                                         ids_in_page.append(mobj.group(1))
 997                         video_ids.extend(ids_in_page)
 998
 999                         if (self._MORE_PAGES_INDICATOR % (playlist_id, pagenum + 1)) not in page:
1000                                 break
1001                         pagenum = pagenum + 1
1002
1003                 for id in video_ids:
1004                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1005                 return
1006
1007 class PostProcessor(object):
1008         """Post Processor class.
1009
1010         PostProcessor objects can be added to downloaders with their
1011         add_post_processor() method. When the downloader has finished a
1012         successful download, it will take its internal chain of PostProcessors
1013         and start calling the run() method on each one of them, first with
1014         an initial argument and then with the returned value of the previous
1015         PostProcessor.
1016
1017         The chain will be stopped if one of them ever returns None or the end
1018         of the chain is reached.
1019
1020         PostProcessor objects follow a "mutual registration" process similar
1021         to InfoExtractor objects.
1022         """
1023
1024         _downloader = None
1025
1026         def __init__(self, downloader=None):
1027                 self._downloader = downloader
1028
1029         def set_downloader(self, downloader):
1030                 """Sets the downloader for this PP."""
1031                 self._downloader = downloader
1032
1033         def run(self, information):
1034                 """Run the PostProcessor.
1035
1036                 The "information" argument is a dictionary like the ones
1037                 composed by InfoExtractors. The only difference is that this
1038                 one has an extra field called "filepath" that points to the
1039                 downloaded file.
1040
1041                 When this method returns None, the postprocessing chain is
1042                 stopped. However, this method may return an information
1043                 dictionary that will be passed to the next postprocessing
1044                 object in the chain. It can be the one it received after
1045                 changing some fields.
1046
1047                 In addition, this method may raise a PostProcessingError
1048                 exception that will be taken into account by the downloader
1049                 it was called from.
1050                 """
1051                 return information # by default, do nothing
1052
1053 ### MAIN PROGRAM ###
1054 if __name__ == '__main__':
1055         try:
1056                 # Modules needed only when running the main program
1057                 import getpass
1058                 import optparse
1059
1060                 # General configuration
1061                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1062                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1063                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1064
1065                 # Parse command line
1066                 parser = optparse.OptionParser(
1067                         usage='Usage: %prog [options] url...',
1068                         version='INTERNAL',
1069                         conflict_handler='resolve',
1070                 )
1071
1072                 parser.add_option('-h', '--help',
1073                                 action='help', help='print this help text and exit')
1074                 parser.add_option('-v', '--version',
1075                                 action='version', help='print program version and exit')
1076                 parser.add_option('-i', '--ignore-errors',
1077                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1078                 parser.add_option('-r', '--rate-limit',
1079                                 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1080
1081                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1082                 authentication.add_option('-u', '--username',
1083                                 dest='username', metavar='UN', help='account username')
1084                 authentication.add_option('-p', '--password',
1085                                 dest='password', metavar='PW', help='account password')
1086                 authentication.add_option('-n', '--netrc',
1087                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1088                 parser.add_option_group(authentication)
1089
1090                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1091                 video_format.add_option('-f', '--format',
1092                                 action='store', dest='format', metavar='FMT', help='video format code')
1093                 video_format.add_option('-b', '--best-quality',
1094                                 action='store_const', dest='format', help='download the best quality video possible', const='0')
1095                 video_format.add_option('-m', '--mobile-version',
1096                                 action='store_const', dest='format', help='alias for -f 17', const='17')
1097                 video_format.add_option('-d', '--high-def',
1098                                 action='store_const', dest='format', help='alias for -f 22', const='22')
1099                 parser.add_option_group(video_format)
1100
1101                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1102                 verbosity.add_option('-q', '--quiet',
1103                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1104                 verbosity.add_option('-s', '--simulate',
1105                                 action='store_true', dest='simulate', help='do not download video', default=False)
1106                 verbosity.add_option('-g', '--get-url',
1107                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1108                 verbosity.add_option('-e', '--get-title',
1109                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1110                 parser.add_option_group(verbosity)
1111
1112                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1113                 filesystem.add_option('-t', '--title',
1114                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
1115                 filesystem.add_option('-l', '--literal',
1116                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1117                 filesystem.add_option('-o', '--output',
1118                                 dest='outtmpl', metavar='TPL', help='output filename template')
1119                 filesystem.add_option('-a', '--batch-file',
1120                                 dest='batchfile', metavar='F', help='file containing URLs to download')
1121                 filesystem.add_option('-w', '--no-overwrites',
1122                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1123                 filesystem.add_option('-c', '--continue',
1124                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1125                 parser.add_option_group(filesystem)
1126
1127                 (opts, args) = parser.parse_args()
1128
1129                 # Batch file verification
1130                 batchurls = []
1131                 if opts.batchfile is not None:
1132                         try:
1133                                 batchurls = open(opts.batchfile, 'r').readlines()
1134                                 batchurls = [x.strip() for x in batchurls]
1135                                 batchurls = [x for x in batchurls if len(x) > 0]
1136                         except IOError:
1137                                 sys.exit(u'ERROR: batch file could not be read')
1138                 all_urls = batchurls + args
1139
1140                 # Conflicting, missing and erroneous options
1141                 if len(all_urls) < 1:
1142                         parser.error(u'you must provide at least one URL')
1143                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1144                         parser.error(u'using .netrc conflicts with giving username/password')
1145                 if opts.password is not None and opts.username is None:
1146                         parser.error(u'account username missing')
1147                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1148                         parser.error(u'using output template conflicts with using title or literal title')
1149                 if opts.usetitle and opts.useliteral:
1150                         parser.error(u'using title conflicts with using literal title')
1151                 if opts.username is not None and opts.password is None:
1152                         opts.password = getpass.getpass(u'Type account password and press return:')
1153                 if opts.ratelimit is not None:
1154                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1155                         if numeric_limit is None:
1156                                 parser.error(u'invalid rate limit specified')
1157                         opts.ratelimit = numeric_limit
1158
1159                 # Information extractors
1160                 youtube_ie = YoutubeIE()
1161                 metacafe_ie = MetacafeIE(youtube_ie)
1162                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1163                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1164
1165                 # File downloader
1166                 fd = FileDownloader({
1167                         'usenetrc': opts.usenetrc,
1168                         'username': opts.username,
1169                         'password': opts.password,
1170                         'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1171                         'forceurl': opts.geturl,
1172                         'forcetitle': opts.gettitle,
1173                         'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1174                         'format': opts.format,
1175                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
1176                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1177                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1178                                 or u'%(id)s.%(ext)s'),
1179                         'ignoreerrors': opts.ignoreerrors,
1180                         'ratelimit': opts.ratelimit,
1181                         'nooverwrites': opts.nooverwrites,
1182                         'continuedl': opts.continue_dl,
1183                         })
1184                 fd.add_info_extractor(youtube_search_ie)
1185                 fd.add_info_extractor(youtube_pl_ie)
1186                 fd.add_info_extractor(metacafe_ie)
1187                 fd.add_info_extractor(youtube_ie)
1188                 retcode = fd.download(all_urls)
1189                 sys.exit(retcode)
1190
1191         except DownloadError:
1192                 sys.exit(1)
1193         except SameFileError:
1194                 sys.exit(u'ERROR: fixed output name but more than one file to download')
1195         except KeyboardInterrupt:
1196                 sys.exit(u'\nERROR: Interrupted by user')