youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # Author: Benjamin Johnson
   6 # Author: Vasyl' Vavrychuk
   7 # License: Public domain code
   8 import cookielib
   9 import datetime
  10 import htmlentitydefs
  11 import httplib
  12 import locale
  13 import math
  14 import netrc
  15 import os
  16 import os.path
  17 import re
  18 import socket
  19 import string
  20 import subprocess
  21 import sys
  22 import time
  23 import urllib
  24 import urllib2
  25
  26 # parse_qs was moved from the cgi module to the urlparse module recently.
  27 try:
  28         from urlparse import parse_qs
  29 except ImportError:
  30         from cgi import parse_qs
  31
  32 std_headers = {
  33         'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12',
  34         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  35         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  36         'Accept-Language': 'en-us,en;q=0.5',
  37 }
  38
  39 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  40
  41 def preferredencoding():
  42         """Get preferred encoding.
  43
  44         Returns the best encoding scheme for the system, based on
  45         locale.getpreferredencoding() and some further tweaks.
  46         """
  47         def yield_preferredencoding():
  48                 try:
  49                         pref = locale.getpreferredencoding()
  50                         u'TEST'.encode(pref)
  51                 except:
  52                         pref = 'UTF-8'
  53                 while True:
  54                         yield pref
  55         return yield_preferredencoding().next()
  56
  57 def htmlentity_transform(matchobj):
  58         """Transforms an HTML entity to a Unicode character.
  59
  60         This function receives a match object and is intended to be used with
  61         the re.sub() function.
  62         """
  63         entity = matchobj.group(1)
  64
  65         # Known non-numeric HTML entity
  66         if entity in htmlentitydefs.name2codepoint:
  67                 return unichr(htmlentitydefs.name2codepoint[entity])
  68
  69         # Unicode character
  70         mobj = re.match(ur'(?u)#(x?\d+)', entity)
  71         if mobj is not None:
  72                 numstr = mobj.group(1)
  73                 if numstr.startswith(u'x'):
  74                         base = 16
  75                         numstr = u'0%s' % numstr
  76                 else:
  77                         base = 10
  78                 return unichr(long(numstr, base))
  79
  80         # Unknown entity in name, return its literal representation
  81         return (u'&%s;' % entity)
  82
  83 def sanitize_title(utitle):
  84         """Sanitizes a video title so it could be used as part of a filename."""
  85         utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
  86         return utitle.replace(unicode(os.sep), u'%')
  87
  88 def sanitize_open(filename, open_mode):
  89         """Try to open the given filename, and slightly tweak it if this fails.
  90
  91         Attempts to open the given filename. If this fails, it tries to change
  92         the filename slightly, step by step, until it's either able to open it
  93         or it fails and raises a final exception, like the standard open()
  94         function.
  95
  96         It returns the tuple (stream, definitive_file_name).
  97         """
  98         try:
  99                 if filename == u'-':
 100                         if sys.platform == 'win32':
 101                                 import msvcrt
 102                                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 103                         return (sys.stdout, filename)
 104                 stream = open(filename, open_mode)
 105                 return (stream, filename)
 106         except (IOError, OSError), err:
 107                 # In case of error, try to remove win32 forbidden chars
 108                 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 109
 110                 # An exception here should be caught in the caller
 111                 stream = open(filename, open_mode)
 112                 return (stream, filename)
 113
 114 class DownloadError(Exception):
 115         """Download Error exception.
 116
 117         This exception may be thrown by FileDownloader objects if they are not
 118         configured to continue on errors. They will contain the appropriate
 119         error message.
 120         """
 121         pass
 122
 123 class SameFileError(Exception):
 124         """Same File exception.
 125
 126         This exception will be thrown by FileDownloader objects if they detect
 127         multiple files would have to be downloaded to the same file on disk.
 128         """
 129         pass
 130
 131 class PostProcessingError(Exception):
 132         """Post Processing exception.
 133
 134         This exception may be raised by PostProcessor's .run() method to
 135         indicate an error in the postprocessing task.
 136         """
 137         pass
 138
 139 class UnavailableVideoError(Exception):
 140         """Unavailable Format exception.
 141
 142         This exception will be thrown when a video is requested
 143         in a format that is not available for that video.
 144         """
 145         pass
 146
 147 class ContentTooShortError(Exception):
 148         """Content Too Short exception.
 149
 150         This exception may be raised by FileDownloader objects when a file they
 151         download is too small for what the server announced first, indicating
 152         the connection was probably interrupted.
 153         """
 154         # Both in bytes
 155         downloaded = None
 156         expected = None
 157
 158         def __init__(self, downloaded, expected):
 159                 self.downloaded = downloaded
 160                 self.expected = expected
 161
 162 class FileDownloader(object):
 163         """File Downloader class.
 164
 165         File downloader objects are the ones responsible of downloading the
 166         actual video file and writing it to disk if the user has requested
 167         it, among some other tasks. In most cases there should be one per
 168         program. As, given a video URL, the downloader doesn't know how to
 169         extract all the needed information, task that InfoExtractors do, it
 170         has to pass the URL to one of them.
 171
 172         For this, file downloader objects have a method that allows
 173         InfoExtractors to be registered in a given order. When it is passed
 174         a URL, the file downloader handles it to the first InfoExtractor it
 175         finds that reports being able to handle it. The InfoExtractor extracts
 176         all the information about the video or videos the URL refers to, and
 177         asks the FileDownloader to process the video information, possibly
 178         downloading the video.
 179
 180         File downloaders accept a lot of parameters. In order not to saturate
 181         the object constructor with arguments, it receives a dictionary of
 182         options instead. These options are available through the params
 183         attribute for the InfoExtractors to use. The FileDownloader also
 184         registers itself as the downloader in charge for the InfoExtractors
 185         that are added to it, so this is a "mutual registration".
 186
 187         Available options:
 188
 189         username:         Username for authentication purposes.
 190         password:         Password for authentication purposes.
 191         usenetrc:         Use netrc for authentication instead.
 192         quiet:            Do not print messages to stdout.
 193         forceurl:         Force printing final URL.
 194         forcetitle:       Force printing title.
 195         forcethumbnail:   Force printing thumbnail URL.
 196         forcedescription: Force printing description.
 197         simulate:         Do not download the video files.
 198         format:           Video format code.
 199         format_limit:     Highest quality format to try.
 200         outtmpl:          Template for output names.
 201         ignoreerrors:     Do not stop on download errors.
 202         ratelimit:        Download speed limit, in bytes/sec.
 203         nooverwrites:     Prevent overwriting files.
 204         retries:          Number of times to retry for HTTP error 5xx
 205         continuedl:       Try to continue downloads if possible.
 206         noprogress:       Do not print the progress bar.
 207         playliststart:    Playlist item to start at.
 208         playlistend:      Playlist item to end at.
 209         logtostderr:      Log messages to stderr instead of stdout.
 210         """
 211
 212         params = None
 213         _ies = []
 214         _pps = []
 215         _download_retcode = None
 216         _num_downloads = None
 217         _screen_file = None
 218
 219         def __init__(self, params):
 220                 """Create a FileDownloader object with the given options."""
 221                 self._ies = []
 222                 self._pps = []
 223                 self._download_retcode = 0
 224                 self._num_downloads = 0
 225                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 226                 self.params = params
 227
 228         @staticmethod
 229         def pmkdir(filename):
 230                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 231                 components = filename.split(os.sep)
 232                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 233                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 234                 for dir in aggregate:
 235                         if not os.path.exists(dir):
 236                                 os.mkdir(dir)
 237
 238         @staticmethod
 239         def temp_name(filename):
 240                 """Returns a temporary filename for the given filename."""
 241                 if filename == u'-' or (os.path.exists(filename) and not os.path.isfile(filename)):
 242                         return filename
 243                 return filename + u'.part'
 244
 245         @staticmethod
 246         def format_bytes(bytes):
 247                 if bytes is None:
 248                         return 'N/A'
 249                 if type(bytes) is str:
 250                         bytes = float(bytes)
 251                 if bytes == 0.0:
 252                         exponent = 0
 253                 else:
 254                         exponent = long(math.log(bytes, 1024.0))
 255                 suffix = 'bkMGTPEZY'[exponent]
 256                 converted = float(bytes) / float(1024**exponent)
 257                 return '%.2f%s' % (converted, suffix)
 258
 259         @staticmethod
 260         def calc_percent(byte_counter, data_len):
 261                 if data_len is None:
 262                         return '---.-%'
 263                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 264
 265         @staticmethod
 266         def calc_eta(start, now, total, current):
 267                 if total is None:
 268                         return '--:--'
 269                 dif = now - start
 270                 if current == 0 or dif < 0.001: # One millisecond
 271                         return '--:--'
 272                 rate = float(current) / dif
 273                 eta = long((float(total) - float(current)) / rate)
 274                 (eta_mins, eta_secs) = divmod(eta, 60)
 275                 if eta_mins > 99:
 276                         return '--:--'
 277                 return '%02d:%02d' % (eta_mins, eta_secs)
 278
 279         @staticmethod
 280         def calc_speed(start, now, bytes):
 281                 dif = now - start
 282                 if bytes == 0 or dif < 0.001: # One millisecond
 283                         return '%10s' % '---b/s'
 284                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 285
 286         @staticmethod
 287         def best_block_size(elapsed_time, bytes):
 288                 new_min = max(bytes / 2.0, 1.0)
 289                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 290                 if elapsed_time < 0.001:
 291                         return long(new_max)
 292                 rate = bytes / elapsed_time
 293                 if rate > new_max:
 294                         return long(new_max)
 295                 if rate < new_min:
 296                         return long(new_min)
 297                 return long(rate)
 298
 299         @staticmethod
 300         def parse_bytes(bytestr):
 301                 """Parse a string indicating a byte quantity into a long integer."""
 302                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 303                 if matchobj is None:
 304                         return None
 305                 number = float(matchobj.group(1))
 306                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 307                 return long(round(number * multiplier))
 308
 309         def add_info_extractor(self, ie):
 310                 """Add an InfoExtractor object to the end of the list."""
 311                 self._ies.append(ie)
 312                 ie.set_downloader(self)
 313
 314         def add_post_processor(self, pp):
 315                 """Add a PostProcessor object to the end of the chain."""
 316                 self._pps.append(pp)
 317                 pp.set_downloader(self)
 318
 319         def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
 320                 """Print message to stdout if not in quiet mode."""
 321                 try:
 322                         if not self.params.get('quiet', False):
 323                                 terminator = [u'\n', u''][skip_eol]
 324                                 print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
 325                         self._screen_file.flush()
 326                 except (UnicodeEncodeError), err:
 327                         if not ignore_encoding_errors:
 328                                 raise
 329
 330         def to_stderr(self, message):
 331                 """Print message to stderr."""
 332                 print >>sys.stderr, message.encode(preferredencoding())
 333
 334         def fixed_template(self):
 335                 """Checks if the output template is fixed."""
 336                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 337
 338         def trouble(self, message=None):
 339                 """Determine action to take when a download problem appears.
 340
 341                 Depending on if the downloader has been configured to ignore
 342                 download errors or not, this method may throw an exception or
 343                 not when errors are found, after printing the message.
 344                 """
 345                 if message is not None:
 346                         self.to_stderr(message)
 347                 if not self.params.get('ignoreerrors', False):
 348                         raise DownloadError(message)
 349                 self._download_retcode = 1
 350
 351         def slow_down(self, start_time, byte_counter):
 352                 """Sleep if the download speed is over the rate limit."""
 353                 rate_limit = self.params.get('ratelimit', None)
 354                 if rate_limit is None or byte_counter == 0:
 355                         return
 356                 now = time.time()
 357                 elapsed = now - start_time
 358                 if elapsed <= 0.0:
 359                         return
 360                 speed = float(byte_counter) / elapsed
 361                 if speed > rate_limit:
 362                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 363
 364         def try_rename(self, old_filename, new_filename):
 365                 try:
 366                         if old_filename == new_filename:
 367                                 return
 368                         os.rename(old_filename, new_filename)
 369                 except (IOError, OSError), err:
 370                         self.trouble(u'ERROR: unable to rename file')
 371
 372         def report_destination(self, filename):
 373                 """Report destination filename."""
 374                 self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
 375
 376         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 377                 """Report download progress."""
 378                 if self.params.get('noprogress', False):
 379                         return
 380                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 381                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 382
 383         def report_resuming_byte(self, resume_len):
 384                 """Report attempt to resume at given byte."""
 385                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 386
 387         def report_retry(self, count, retries):
 388                 """Report retry in case of HTTP error 5xx"""
 389                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 390
 391         def report_file_already_downloaded(self, file_name):
 392                 """Report file has already been fully downloaded."""
 393                 try:
 394                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 395                 except (UnicodeEncodeError), err:
 396                         self.to_screen(u'[download] The file has already been downloaded')
 397
 398         def report_unable_to_resume(self):
 399                 """Report it was impossible to resume download."""
 400                 self.to_screen(u'[download] Unable to resume')
 401
 402         def report_finish(self):
 403                 """Report download finished."""
 404                 if self.params.get('noprogress', False):
 405                         self.to_screen(u'[download] Download completed')
 406                 else:
 407                         self.to_screen(u'')
 408
 409         def increment_downloads(self):
 410                 """Increment the ordinal that assigns a number to each file."""
 411                 self._num_downloads += 1
 412
 413         def process_info(self, info_dict):
 414                 """Process a single dictionary returned by an InfoExtractor."""
 415                 # Do nothing else if in simulate mode
 416                 if self.params.get('simulate', False):
 417                         # Forced printings
 418                         if self.params.get('forcetitle', False):
 419                                 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
 420                         if self.params.get('forceurl', False):
 421                                 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
 422                         if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 423                                 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
 424                         if self.params.get('forcedescription', False) and 'description' in info_dict:
 425                                 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
 426
 427                         return
 428
 429                 try:
 430                         template_dict = dict(info_dict)
 431                         template_dict['epoch'] = unicode(long(time.time()))
 432                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 433                         filename = self.params['outtmpl'] % template_dict
 434                 except (ValueError, KeyError), err:
 435                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 436                         return
 437                 if self.params.get('nooverwrites', False) and os.path.exists(filename):
 438                         self.to_stderr(u'WARNING: file exists and will be skipped')
 439                         return
 440
 441                 try:
 442                         self.pmkdir(filename)
 443                 except (OSError, IOError), err:
 444                         self.trouble(u'ERROR: unable to create directories: %s' % str(err))
 445                         return
 446
 447                 try:
 448                         success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
 449                 except (OSError, IOError), err:
 450                         raise UnavailableVideoError
 451                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 452                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 453                         return
 454                 except (ContentTooShortError, ), err:
 455                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 456                         return
 457
 458                 if success:
 459                         try:
 460                                 self.post_process(filename, info_dict)
 461                         except (PostProcessingError), err:
 462                                 self.trouble(u'ERROR: postprocessing: %s' % str(err))
 463                                 return
 464
 465         def download(self, url_list):
 466                 """Download a given list of URLs."""
 467                 if len(url_list) > 1 and self.fixed_template():
 468                         raise SameFileError(self.params['outtmpl'])
 469
 470                 for url in url_list:
 471                         suitable_found = False
 472                         for ie in self._ies:
 473                                 # Go to next InfoExtractor if not suitable
 474                                 if not ie.suitable(url):
 475                                         continue
 476
 477                                 # Suitable InfoExtractor found
 478                                 suitable_found = True
 479
 480                                 # Extract information from URL and process it
 481                                 ie.extract(url)
 482
 483                                 # Suitable InfoExtractor had been found; go to next URL
 484                                 break
 485
 486                         if not suitable_found:
 487                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 488
 489                 return self._download_retcode
 490
 491         def post_process(self, filename, ie_info):
 492                 """Run the postprocessing chain on the given file."""
 493                 info = dict(ie_info)
 494                 info['filepath'] = filename
 495                 for pp in self._pps:
 496                         info = pp.run(info)
 497                         if info is None:
 498                                 break
 499
 500         def _download_with_rtmpdump(self, filename, url, player_url):
 501                 self.report_destination(filename)
 502                 tmpfilename = self.temp_name(filename)
 503
 504                 # Check for rtmpdump first
 505                 try:
 506                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 507                 except (OSError, IOError):
 508                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 509                         return False
 510
 511                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 512                 # the connection was interrumpted and resuming appears to be
 513                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 514                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 515                 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
 516                 while retval == 2 or retval == 1:
 517                         prevsize = os.path.getsize(tmpfilename)
 518                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 519                         time.sleep(5.0) # This seems to be needed
 520                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 521                         cursize = os.path.getsize(tmpfilename)
 522                         if prevsize == cursize and retval == 1:
 523                                 break
 524                 if retval == 0:
 525                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
 526                         self.try_rename(tmpfilename, filename)
 527                         return True
 528                 else:
 529                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 530                         return False
 531
 532         def _do_download(self, filename, url, player_url):
 533                 # Check file already present
 534                 if self.params.get('continuedl', False) and os.path.isfile(filename):
 535                         self.report_file_already_downloaded(filename)
 536                         return True
 537
 538                 # Attempt to download using rtmpdump
 539                 if url.startswith('rtmp'):
 540                         return self._download_with_rtmpdump(filename, url, player_url)
 541
 542                 tmpfilename = self.temp_name(filename)
 543                 stream = None
 544                 open_mode = 'wb'
 545                 basic_request = urllib2.Request(url, None, std_headers)
 546                 request = urllib2.Request(url, None, std_headers)
 547
 548                 # Establish possible resume length
 549                 if os.path.isfile(tmpfilename):
 550                         resume_len = os.path.getsize(tmpfilename)
 551                 else:
 552                         resume_len = 0
 553
 554                 # Request parameters in case of being able to resume
 555                 if self.params.get('continuedl', False) and resume_len != 0:
 556                         self.report_resuming_byte(resume_len)
 557                         request.add_header('Range','bytes=%d-' % resume_len)
 558                         open_mode = 'ab'
 559
 560                 count = 0
 561                 retries = self.params.get('retries', 0)
 562                 while count <= retries:
 563                         # Establish connection
 564                         try:
 565                                 data = urllib2.urlopen(request)
 566                                 break
 567                         except (urllib2.HTTPError, ), err:
 568                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 569                                         # Unexpected HTTP error
 570                                         raise
 571                                 elif err.code == 416:
 572                                         # Unable to resume (requested range not satisfiable)
 573                                         try:
 574                                                 # Open the connection again without the range header
 575                                                 data = urllib2.urlopen(basic_request)
 576                                                 content_length = data.info()['Content-Length']
 577                                         except (urllib2.HTTPError, ), err:
 578                                                 if err.code < 500 or err.code >= 600:
 579                                                         raise
 580                                         else:
 581                                                 # Examine the reported length
 582                                                 if (content_length is not None and
 583                                                     (resume_len - 100 < long(content_length) < resume_len + 100)):
 584                                                         # The file had already been fully downloaded.
 585                                                         # Explanation to the above condition: in issue #175 it was revealed that
 586                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 587                                                         # changing the file size slightly and causing problems for some users. So
 588                                                         # I decided to implement a suggested change and consider the file
 589                                                         # completely downloaded if the file size differs less than 100 bytes from
 590                                                         # the one in the hard drive.
 591                                                         self.report_file_already_downloaded(filename)
 592                                                         self.try_rename(tmpfilename, filename)
 593                                                         return True
 594                                                 else:
 595                                                         # The length does not match, we start the download over
 596                                                         self.report_unable_to_resume()
 597                                                         open_mode = 'wb'
 598                                                         break
 599                         # Retry
 600                         count += 1
 601                         if count <= retries:
 602                                 self.report_retry(count, retries)
 603
 604                 if count > retries:
 605                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 606                         return False
 607
 608                 data_len = data.info().get('Content-length', None)
 609                 data_len_str = self.format_bytes(data_len)
 610                 byte_counter = 0
 611                 block_size = 1024
 612                 start = time.time()
 613                 while True:
 614                         # Download and write
 615                         before = time.time()
 616                         data_block = data.read(block_size)
 617                         after = time.time()
 618                         data_block_len = len(data_block)
 619                         if data_block_len == 0:
 620                                 break
 621                         byte_counter += data_block_len
 622
 623                         # Open file just in time
 624                         if stream is None:
 625                                 try:
 626                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 627                                         self.report_destination(filename)
 628                                 except (OSError, IOError), err:
 629                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 630                                         return False
 631                         try:
 632                                 stream.write(data_block)
 633                         except (IOError, OSError), err:
 634                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 635                                 return False
 636                         block_size = self.best_block_size(after - before, data_block_len)
 637
 638                         # Progress message
 639                         percent_str = self.calc_percent(byte_counter, data_len)
 640                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 641                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 642                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 643
 644                         # Apply rate limit
 645                         self.slow_down(start, byte_counter)
 646
 647                 stream.close()
 648                 self.report_finish()
 649                 if data_len is not None and str(byte_counter) != data_len:
 650                         raise ContentTooShortError(byte_counter, long(data_len))
 651                 self.try_rename(tmpfilename, filename)
 652                 return True
 653
 654 class InfoExtractor(object):
 655         """Information Extractor class.
 656
 657         Information extractors are the classes that, given a URL, extract
 658         information from the video (or videos) the URL refers to. This
 659         information includes the real video URL, the video title and simplified
 660         title, author and others. The information is stored in a dictionary
 661         which is then passed to the FileDownloader. The FileDownloader
 662         processes this information possibly downloading the video to the file
 663         system, among other possible outcomes. The dictionaries must include
 664         the following fields:
 665
 666         id:             Video identifier.
 667         url:            Final video URL.
 668         uploader:       Nickname of the video uploader.
 669         title:          Literal title.
 670         stitle:         Simplified title.
 671         ext:            Video filename extension.
 672         format:         Video format.
 673         player_url:     SWF Player URL (may be None).
 674
 675         The following fields are optional. Their primary purpose is to allow
 676         youtube-dl to serve as the backend for a video search function, such
 677         as the one in youtube2mp3.  They are only used when their respective
 678         forced printing functions are called:
 679
 680         thumbnail:      Full URL to a video thumbnail image.
 681         description:    One-line video description.
 682
 683         Subclasses of this one should re-define the _real_initialize() and
 684         _real_extract() methods, as well as the suitable() static method.
 685         Probably, they should also be instantiated and added to the main
 686         downloader.
 687         """
 688
 689         _ready = False
 690         _downloader = None
 691
 692         def __init__(self, downloader=None):
 693                 """Constructor. Receives an optional downloader."""
 694                 self._ready = False
 695                 self.set_downloader(downloader)
 696
 697         @staticmethod
 698         def suitable(url):
 699                 """Receives a URL and returns True if suitable for this IE."""
 700                 return False
 701
 702         def initialize(self):
 703                 """Initializes an instance (authentication, etc)."""
 704                 if not self._ready:
 705                         self._real_initialize()
 706                         self._ready = True
 707
 708         def extract(self, url):
 709                 """Extracts URL information and returns it in list of dicts."""
 710                 self.initialize()
 711                 return self._real_extract(url)
 712
 713         def set_downloader(self, downloader):
 714                 """Sets the downloader for this IE."""
 715                 self._downloader = downloader
 716
 717         def _real_initialize(self):
 718                 """Real initialization process. Redefine in subclasses."""
 719                 pass
 720
 721         def _real_extract(self, url):
 722                 """Real extraction process. Redefine in subclasses."""
 723                 pass
 724
 725 class YoutubeIE(InfoExtractor):
 726         """Information extractor for youtube.com."""
 727
 728         _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$'
 729         _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 730         _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
 731         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 732         _NETRC_MACHINE = 'youtube'
 733         # Listed in order of quality
 734         _available_formats = ['38', '37', '22', '45', '35', '34', '43', '18', '6', '5', '17', '13']
 735         _video_extensions = {
 736                 '13': '3gp',
 737                 '17': 'mp4',
 738                 '18': 'mp4',
 739                 '22': 'mp4',
 740                 '37': 'mp4',
 741                 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
 742                 '43': 'webm',
 743                 '45': 'webm',
 744         }
 745
 746         @staticmethod
 747         def suitable(url):
 748                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 749
 750         def report_lang(self):
 751                 """Report attempt to set language."""
 752                 self._downloader.to_screen(u'[youtube] Setting language')
 753
 754         def report_login(self):
 755                 """Report attempt to log in."""
 756                 self._downloader.to_screen(u'[youtube] Logging in')
 757
 758         def report_age_confirmation(self):
 759                 """Report attempt to confirm age."""
 760                 self._downloader.to_screen(u'[youtube] Confirming age')
 761
 762         def report_video_webpage_download(self, video_id):
 763                 """Report attempt to download video webpage."""
 764                 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
 765
 766         def report_video_info_webpage_download(self, video_id):
 767                 """Report attempt to download video info webpage."""
 768                 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
 769
 770         def report_information_extraction(self, video_id):
 771                 """Report attempt to extract video information."""
 772                 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
 773
 774         def report_unavailable_format(self, video_id, format):
 775                 """Report extracted video URL."""
 776                 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
 777
 778         def report_rtmp_download(self):
 779                 """Indicate the download will use the RTMP protocol."""
 780                 self._downloader.to_screen(u'[youtube] RTMP download detected')
 781
 782         def _real_initialize(self):
 783                 if self._downloader is None:
 784                         return
 785
 786                 username = None
 787                 password = None
 788                 downloader_params = self._downloader.params
 789
 790                 # Attempt to use provided username and password or .netrc data
 791                 if downloader_params.get('username', None) is not None:
 792                         username = downloader_params['username']
 793                         password = downloader_params['password']
 794                 elif downloader_params.get('usenetrc', False):
 795                         try:
 796                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 797                                 if info is not None:
 798                                         username = info[0]
 799                                         password = info[2]
 800                                 else:
 801                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 802                         except (IOError, netrc.NetrcParseError), err:
 803                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 804                                 return
 805
 806                 # Set language
 807                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 808                 try:
 809                         self.report_lang()
 810                         urllib2.urlopen(request).read()
 811                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 812                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 813                         return
 814
 815                 # No authentication to be performed
 816                 if username is None:
 817                         return
 818
 819                 # Log in
 820                 login_form = {
 821                                 'current_form': 'loginForm',
 822                                 'next':         '/',
 823                                 'action_login': 'Log In',
 824                                 'username':     username,
 825                                 'password':     password,
 826                                 }
 827                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 828                 try:
 829                         self.report_login()
 830                         login_results = urllib2.urlopen(request).read()
 831                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 832                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 833                                 return
 834                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 835                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 836                         return
 837
 838                 # Confirm age
 839                 age_form = {
 840                                 'next_url':             '/',
 841                                 'action_confirm':       'Confirm',
 842                                 }
 843                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 844                 try:
 845                         self.report_age_confirmation()
 846                         age_results = urllib2.urlopen(request).read()
 847                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 848                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 849                         return
 850
 851         def _real_extract(self, url):
 852                 # Extract video id from URL
 853                 mobj = re.match(self._VALID_URL, url)
 854                 if mobj is None:
 855                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 856                         return
 857                 video_id = mobj.group(2)
 858
 859                 # Get video webpage
 860                 self.report_video_webpage_download(video_id)
 861                 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&amp;has_verified=1' % video_id, None, std_headers)
 862                 try:
 863                         video_webpage = urllib2.urlopen(request).read()
 864                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 865                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 866                         return
 867
 868                 # Attempt to extract SWF player URL
 869                 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
 870                 if mobj is not None:
 871                         player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
 872                 else:
 873                         player_url = None
 874
 875                 # Get video info
 876                 self.report_video_info_webpage_download(video_id)
 877                 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
 878                         video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
 879                                            % (video_id, el_type))
 880                         request = urllib2.Request(video_info_url, None, std_headers)
 881                         try:
 882                                 video_info_webpage = urllib2.urlopen(request).read()
 883                                 video_info = parse_qs(video_info_webpage)
 884                                 if 'token' in video_info:
 885                                         break
 886                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 887                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 888                                 return
 889                 if 'token' not in video_info:
 890                         if 'reason' in video_info:
 891                                 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
 892                         else:
 893                                 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
 894                         return
 895
 896                 # Start extracting information
 897                 self.report_information_extraction(video_id)
 898
 899                 # uploader
 900                 if 'author' not in video_info:
 901                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 902                         return
 903                 video_uploader = urllib.unquote_plus(video_info['author'][0])
 904
 905                 # title
 906                 if 'title' not in video_info:
 907                         self._downloader.trouble(u'ERROR: unable to extract video title')
 908                         return
 909                 video_title = urllib.unquote_plus(video_info['title'][0])
 910                 video_title = video_title.decode('utf-8')
 911                 video_title = sanitize_title(video_title)
 912
 913                 # simplified title
 914                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 915                 simple_title = simple_title.strip(ur'_')
 916
 917                 # thumbnail image
 918                 if 'thumbnail_url' not in video_info:
 919                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
 920                         video_thumbnail = ''
 921                 else:   # don't panic if we can't find it
 922                         video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
 923
 924                 # upload date
 925                 upload_date = u'NA'
 926                 mobj = re.search(r'id="eow-date".*?>(.*?)</span>', video_webpage, re.DOTALL)
 927                 if mobj is not None:
 928                         upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
 929                         format_expressions = ['%d %B %Y', '%B %d %Y']
 930                         for expression in format_expressions:
 931                                 try:
 932                                         upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
 933                                 except:
 934                                         pass
 935
 936                 # description
 937                 video_description = 'No description available.'
 938                 if self._downloader.params.get('forcedescription', False):
 939                         mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
 940                         if mobj is not None:
 941                                 video_description = mobj.group(1)
 942
 943                 # token
 944                 video_token = urllib.unquote_plus(video_info['token'][0])
 945
 946                 # Decide which formats to download
 947                 req_format = self._downloader.params.get('format', None)
 948                 get_video_template = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=&ps=&asv=&fmt=%%s' % (video_id, video_token)
 949
 950                 if 'fmt_url_map' in video_info:
 951                         url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(','))
 952                         format_limit = self._downloader.params.get('format_limit', None)
 953                         if format_limit is not None and format_limit in self._available_formats:
 954                                 format_list = self._available_formats[self._available_formats.index(format_limit):]
 955                         else:
 956                                 format_list = self._available_formats
 957                         existing_formats = [x for x in format_list if x in url_map]
 958                         if len(existing_formats) == 0:
 959                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 960                                 return
 961                         if req_format is None:
 962                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
 963                         elif req_format == '-1':
 964                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
 965                         else:
 966                                 if req_format in url_map:
 967                                         video_url_list = [(req_format, url_map[req_format])] # Specific format
 968                                 else:
 969                                         video_url_list = [(req_format, get_video_template % req_format)] # Specific format
 970
 971                 elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
 972                         self.report_rtmp_download()
 973                         video_url_list = [(None, video_info['conn'][0])]
 974
 975                 else:
 976                         self._downloader.trouble(u'ERROR: no fmt_url_map or conn information found in video info')
 977                         return
 978
 979                 for format_param, video_real_url in video_url_list:
 980                         # At this point we have a new video
 981                         self._downloader.increment_downloads()
 982
 983                         # Extension
 984                         video_extension = self._video_extensions.get(format_param, 'flv')
 985
 986                         # Find the video URL in fmt_url_map or conn paramters
 987                         try:
 988                                 # Process video information
 989                                 self._downloader.process_info({
 990                                         'id':           video_id.decode('utf-8'),
 991                                         'url':          video_real_url.decode('utf-8'),
 992                                         'uploader':     video_uploader.decode('utf-8'),
 993                                         'upload_date':  upload_date,
 994                                         'title':        video_title,
 995                                         'stitle':       simple_title,
 996                                         'ext':          video_extension.decode('utf-8'),
 997                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
 998                                         'thumbnail':    video_thumbnail.decode('utf-8'),
 999                                         'description':  video_description.decode('utf-8'),
1000                                         'player_url':   player_url,
1001                                 })
1002                         except UnavailableVideoError, err:
1003                                 self._downloader.trouble(u'ERROR: unable to download video (format may not be available)')
1004
1005
1006 class MetacafeIE(InfoExtractor):
1007         """Information Extractor for metacafe.com."""
1008
1009         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
1010         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
1011         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
1012         _youtube_ie = None
1013
1014         def __init__(self, youtube_ie, downloader=None):
1015                 InfoExtractor.__init__(self, downloader)
1016                 self._youtube_ie = youtube_ie
1017
1018         @staticmethod
1019         def suitable(url):
1020                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
1021
1022         def report_disclaimer(self):
1023                 """Report disclaimer retrieval."""
1024                 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
1025
1026         def report_age_confirmation(self):
1027                 """Report attempt to confirm age."""
1028                 self._downloader.to_screen(u'[metacafe] Confirming age')
1029
1030         def report_download_webpage(self, video_id):
1031                 """Report webpage download."""
1032                 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
1033
1034         def report_extraction(self, video_id):
1035                 """Report information extraction."""
1036                 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
1037
1038         def _real_initialize(self):
1039                 # Retrieve disclaimer
1040                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
1041                 try:
1042                         self.report_disclaimer()
1043                         disclaimer = urllib2.urlopen(request).read()
1044                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1045                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1046                         return
1047
1048                 # Confirm age
1049                 disclaimer_form = {
1050                         'filters': '0',
1051                         'submit': "Continue - I'm over 18",
1052                         }
1053                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
1054                 try:
1055                         self.report_age_confirmation()
1056                         disclaimer = urllib2.urlopen(request).read()
1057                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1058                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1059                         return
1060
1061         def _real_extract(self, url):
1062                 # Extract id and simplified title from URL
1063                 mobj = re.match(self._VALID_URL, url)
1064                 if mobj is None:
1065                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1066                         return
1067
1068                 video_id = mobj.group(1)
1069
1070                 # Check if video comes from YouTube
1071                 mobj2 = re.match(r'^yt-(.*)$', video_id)
1072                 if mobj2 is not None:
1073                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1074                         return
1075
1076                 # At this point we have a new video
1077                 self._downloader.increment_downloads()
1078
1079                 simple_title = mobj.group(2).decode('utf-8')
1080
1081                 # Retrieve video webpage to extract further information
1082                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1083                 try:
1084                         self.report_download_webpage(video_id)
1085                         webpage = urllib2.urlopen(request).read()
1086                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1087                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1088                         return
1089
1090                 # Extract URL, uploader and title from webpage
1091                 self.report_extraction(video_id)
1092                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1093                 if mobj is not None:
1094                         mediaURL = urllib.unquote(mobj.group(1))
1095                         video_extension = mediaURL[-3:]
1096
1097                         # Extract gdaKey if available
1098                         mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1099                         if mobj is None:
1100                                 video_url = mediaURL
1101                         else:
1102                                 gdaKey = mobj.group(1)
1103                                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1104                 else:
1105                         mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
1106                         if mobj is None:
1107                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1108                                 return
1109                         vardict = parse_qs(mobj.group(1))
1110                         if 'mediaData' not in vardict:
1111                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1112                                 return
1113                         mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
1114                         if mobj is None:
1115                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1116                                 return
1117                         mediaURL = mobj.group(1).replace('\\/', '/')
1118                         video_extension = mediaURL[-3:]
1119                         video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
1120
1121                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1122                 if mobj is None:
1123                         self._downloader.trouble(u'ERROR: unable to extract title')
1124                         return
1125                 video_title = mobj.group(1).decode('utf-8')
1126                 video_title = sanitize_title(video_title)
1127
1128                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1129                 if mobj is None:
1130                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1131                         return
1132                 video_uploader = mobj.group(1)
1133
1134                 try:
1135                         # Process video information
1136                         self._downloader.process_info({
1137                                 'id':           video_id.decode('utf-8'),
1138                                 'url':          video_url.decode('utf-8'),
1139                                 'uploader':     video_uploader.decode('utf-8'),
1140                                 'upload_date':  u'NA',
1141                                 'title':        video_title,
1142                                 'stitle':       simple_title,
1143                                 'ext':          video_extension.decode('utf-8'),
1144                                 'format':       u'NA',
1145                                 'player_url':   None,
1146                         })
1147                 except UnavailableVideoError:
1148                         self._downloader.trouble(u'ERROR: unable to download video')
1149
1150
1151 class DailymotionIE(InfoExtractor):
1152         """Information Extractor for Dailymotion"""
1153
1154         _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1155
1156         def __init__(self, downloader=None):
1157                 InfoExtractor.__init__(self, downloader)
1158
1159         @staticmethod
1160         def suitable(url):
1161                 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1162
1163         def report_download_webpage(self, video_id):
1164                 """Report webpage download."""
1165                 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
1166
1167         def report_extraction(self, video_id):
1168                 """Report information extraction."""
1169                 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
1170
1171         def _real_initialize(self):
1172                 return
1173
1174         def _real_extract(self, url):
1175                 # Extract id and simplified title from URL
1176                 mobj = re.match(self._VALID_URL, url)
1177                 if mobj is None:
1178                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1179                         return
1180
1181                 # At this point we have a new video
1182                 self._downloader.increment_downloads()
1183                 video_id = mobj.group(1)
1184
1185                 simple_title = mobj.group(2).decode('utf-8')
1186                 video_extension = 'flv'
1187
1188                 # Retrieve video webpage to extract further information
1189                 request = urllib2.Request(url)
1190                 try:
1191                         self.report_download_webpage(video_id)
1192                         webpage = urllib2.urlopen(request).read()
1193                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1194                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1195                         return
1196
1197                 # Extract URL, uploader and title from webpage
1198                 self.report_extraction(video_id)
1199                 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1200                 if mobj is None:
1201                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1202                         return
1203                 mediaURL = urllib.unquote(mobj.group(1))
1204
1205                 # if needed add http://www.dailymotion.com/ if relative URL
1206
1207                 video_url = mediaURL
1208
1209                 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1210                 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1211                 if mobj is None:
1212                         self._downloader.trouble(u'ERROR: unable to extract title')
1213                         return
1214                 video_title = mobj.group(1).decode('utf-8')
1215                 video_title = sanitize_title(video_title)
1216
1217                 mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a>', webpage)
1218                 if mobj is None:
1219                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1220                         return
1221                 video_uploader = mobj.group(1)
1222
1223                 try:
1224                         # Process video information
1225                         self._downloader.process_info({
1226                                 'id':           video_id.decode('utf-8'),
1227                                 'url':          video_url.decode('utf-8'),
1228                                 'uploader':     video_uploader.decode('utf-8'),
1229                                 'upload_date':  u'NA',
1230                                 'title':        video_title,
1231                                 'stitle':       simple_title,
1232                                 'ext':          video_extension.decode('utf-8'),
1233                                 'format':       u'NA',
1234                                 'player_url':   None,
1235                         })
1236                 except UnavailableVideoError:
1237                         self._downloader.trouble(u'ERROR: unable to download video')
1238
1239 class GoogleIE(InfoExtractor):
1240         """Information extractor for video.google.com."""
1241
1242         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1243
1244         def __init__(self, downloader=None):
1245                 InfoExtractor.__init__(self, downloader)
1246
1247         @staticmethod
1248         def suitable(url):
1249                 return (re.match(GoogleIE._VALID_URL, url) is not None)
1250
1251         def report_download_webpage(self, video_id):
1252                 """Report webpage download."""
1253                 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
1254
1255         def report_extraction(self, video_id):
1256                 """Report information extraction."""
1257                 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
1258
1259         def _real_initialize(self):
1260                 return
1261
1262         def _real_extract(self, url):
1263                 # Extract id from URL
1264                 mobj = re.match(self._VALID_URL, url)
1265                 if mobj is None:
1266                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1267                         return
1268
1269                 # At this point we have a new video
1270                 self._downloader.increment_downloads()
1271                 video_id = mobj.group(1)
1272
1273                 video_extension = 'mp4'
1274
1275                 # Retrieve video webpage to extract further information
1276                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1277                 try:
1278                         self.report_download_webpage(video_id)
1279                         webpage = urllib2.urlopen(request).read()
1280                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1281                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1282                         return
1283
1284                 # Extract URL, uploader, and title from webpage
1285                 self.report_extraction(video_id)
1286                 mobj = re.search(r"download_url:'([^']+)'", webpage)
1287                 if mobj is None:
1288                         video_extension = 'flv'
1289                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1290                 if mobj is None:
1291                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1292                         return
1293                 mediaURL = urllib.unquote(mobj.group(1))
1294                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1295                 mediaURL = mediaURL.replace('\\x26', '\x26')
1296
1297                 video_url = mediaURL
1298
1299                 mobj = re.search(r'<title>(.*)</title>', webpage)
1300                 if mobj is None:
1301                         self._downloader.trouble(u'ERROR: unable to extract title')
1302                         return
1303                 video_title = mobj.group(1).decode('utf-8')
1304                 video_title = sanitize_title(video_title)
1305                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1306
1307                 # Extract video description
1308                 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1309                 if mobj is None:
1310                         self._downloader.trouble(u'ERROR: unable to extract video description')
1311                         return
1312                 video_description = mobj.group(1).decode('utf-8')
1313                 if not video_description:
1314                         video_description = 'No description available.'
1315
1316                 # Extract video thumbnail
1317                 if self._downloader.params.get('forcethumbnail', False):
1318                         request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1319                         try:
1320                                 webpage = urllib2.urlopen(request).read()
1321                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1322                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1323                                 return
1324                         mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1325                         if mobj is None:
1326                                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1327                                 return
1328                         video_thumbnail = mobj.group(1)
1329                 else:   # we need something to pass to process_info
1330                         video_thumbnail = ''
1331
1332
1333                 try:
1334                         # Process video information
1335                         self._downloader.process_info({
1336                                 'id':           video_id.decode('utf-8'),
1337                                 'url':          video_url.decode('utf-8'),
1338                                 'uploader':     u'NA',
1339                                 'upload_date':  u'NA',
1340                                 'title':        video_title,
1341                                 'stitle':       simple_title,
1342                                 'ext':          video_extension.decode('utf-8'),
1343                                 'format':       u'NA',
1344                                 'player_url':   None,
1345                         })
1346                 except UnavailableVideoError:
1347                         self._downloader.trouble(u'ERROR: unable to download video')
1348
1349
1350 class PhotobucketIE(InfoExtractor):
1351         """Information extractor for photobucket.com."""
1352
1353         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1354
1355         def __init__(self, downloader=None):
1356                 InfoExtractor.__init__(self, downloader)
1357
1358         @staticmethod
1359         def suitable(url):
1360                 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1361
1362         def report_download_webpage(self, video_id):
1363                 """Report webpage download."""
1364                 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
1365
1366         def report_extraction(self, video_id):
1367                 """Report information extraction."""
1368                 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
1369
1370         def _real_initialize(self):
1371                 return
1372
1373         def _real_extract(self, url):
1374                 # Extract id from URL
1375                 mobj = re.match(self._VALID_URL, url)
1376                 if mobj is None:
1377                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1378                         return
1379
1380                 # At this point we have a new video
1381                 self._downloader.increment_downloads()
1382                 video_id = mobj.group(1)
1383
1384                 video_extension = 'flv'
1385
1386                 # Retrieve video webpage to extract further information
1387                 request = urllib2.Request(url)
1388                 try:
1389                         self.report_download_webpage(video_id)
1390                         webpage = urllib2.urlopen(request).read()
1391                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1392                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1393                         return
1394
1395                 # Extract URL, uploader, and title from webpage
1396                 self.report_extraction(video_id)
1397                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1398                 if mobj is None:
1399                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1400                         return
1401                 mediaURL = urllib.unquote(mobj.group(1))
1402
1403                 video_url = mediaURL
1404
1405                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1406                 if mobj is None:
1407                         self._downloader.trouble(u'ERROR: unable to extract title')
1408                         return
1409                 video_title = mobj.group(1).decode('utf-8')
1410                 video_title = sanitize_title(video_title)
1411                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1412
1413                 video_uploader = mobj.group(2).decode('utf-8')
1414
1415                 try:
1416                         # Process video information
1417                         self._downloader.process_info({
1418                                 'id':           video_id.decode('utf-8'),
1419                                 'url':          video_url.decode('utf-8'),
1420                                 'uploader':     video_uploader,
1421                                 'upload_date':  u'NA',
1422                                 'title':        video_title,
1423                                 'stitle':       simple_title,
1424                                 'ext':          video_extension.decode('utf-8'),
1425                                 'format':       u'NA',
1426                                 'player_url':   None,
1427                         })
1428                 except UnavailableVideoError:
1429                         self._downloader.trouble(u'ERROR: unable to download video')
1430
1431
1432 class YahooIE(InfoExtractor):
1433         """Information extractor for video.yahoo.com."""
1434
1435         # _VALID_URL matches all Yahoo! Video URLs
1436         # _VPAGE_URL matches only the extractable '/watch/' URLs
1437         _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1438         _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1439
1440         def __init__(self, downloader=None):
1441                 InfoExtractor.__init__(self, downloader)
1442
1443         @staticmethod
1444         def suitable(url):
1445                 return (re.match(YahooIE._VALID_URL, url) is not None)
1446
1447         def report_download_webpage(self, video_id):
1448                 """Report webpage download."""
1449                 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
1450
1451         def report_extraction(self, video_id):
1452                 """Report information extraction."""
1453                 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
1454
1455         def _real_initialize(self):
1456                 return
1457
1458         def _real_extract(self, url, new_video=True):
1459                 # Extract ID from URL
1460                 mobj = re.match(self._VALID_URL, url)
1461                 if mobj is None:
1462                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1463                         return
1464
1465                 # At this point we have a new video
1466                 self._downloader.increment_downloads()
1467                 video_id = mobj.group(2)
1468                 video_extension = 'flv'
1469
1470                 # Rewrite valid but non-extractable URLs as
1471                 # extractable English language /watch/ URLs
1472                 if re.match(self._VPAGE_URL, url) is None:
1473                         request = urllib2.Request(url)
1474                         try:
1475                                 webpage = urllib2.urlopen(request).read()
1476                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1477                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1478                                 return
1479
1480                         mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1481                         if mobj is None:
1482                                 self._downloader.trouble(u'ERROR: Unable to extract id field')
1483                                 return
1484                         yahoo_id = mobj.group(1)
1485
1486                         mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1487                         if mobj is None:
1488                                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1489                                 return
1490                         yahoo_vid = mobj.group(1)
1491
1492                         url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1493                         return self._real_extract(url, new_video=False)
1494
1495                 # Retrieve video webpage to extract further information
1496                 request = urllib2.Request(url)
1497                 try:
1498                         self.report_download_webpage(video_id)
1499                         webpage = urllib2.urlopen(request).read()
1500                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1501                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1502                         return
1503
1504                 # Extract uploader and title from webpage
1505                 self.report_extraction(video_id)
1506                 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1507                 if mobj is None:
1508                         self._downloader.trouble(u'ERROR: unable to extract video title')
1509                         return
1510                 video_title = mobj.group(1).decode('utf-8')
1511                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1512
1513                 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1514                 if mobj is None:
1515                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
1516                         return
1517                 video_uploader = mobj.group(1).decode('utf-8')
1518
1519                 # Extract video thumbnail
1520                 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1521                 if mobj is None:
1522                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1523                         return
1524                 video_thumbnail = mobj.group(1).decode('utf-8')
1525
1526                 # Extract video description
1527                 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1528                 if mobj is None:
1529                         self._downloader.trouble(u'ERROR: unable to extract video description')
1530                         return
1531                 video_description = mobj.group(1).decode('utf-8')
1532                 if not video_description: video_description = 'No description available.'
1533
1534                 # Extract video height and width
1535                 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1536                 if mobj is None:
1537                         self._downloader.trouble(u'ERROR: unable to extract video height')
1538                         return
1539                 yv_video_height = mobj.group(1)
1540
1541                 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1542                 if mobj is None:
1543                         self._downloader.trouble(u'ERROR: unable to extract video width')
1544                         return
1545                 yv_video_width = mobj.group(1)
1546
1547                 # Retrieve video playlist to extract media URL
1548                 # I'm not completely sure what all these options are, but we
1549                 # seem to need most of them, otherwise the server sends a 401.
1550                 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
1551                 yv_bitrate = '700'  # according to Wikipedia this is hard-coded
1552                 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1553                                           '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1554                                           '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1555                 try:
1556                         self.report_download_webpage(video_id)
1557                         webpage = urllib2.urlopen(request).read()
1558                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1559                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1560                         return
1561
1562                 # Extract media URL from playlist XML
1563                 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1564                 if mobj is None:
1565                         self._downloader.trouble(u'ERROR: Unable to extract media URL')
1566                         return
1567                 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1568                 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1569
1570                 try:
1571                         # Process video information
1572                         self._downloader.process_info({
1573                                 'id':           video_id.decode('utf-8'),
1574                                 'url':          video_url,
1575                                 'uploader':     video_uploader,
1576                                 'upload_date':  u'NA',
1577                                 'title':        video_title,
1578                                 'stitle':       simple_title,
1579                                 'ext':          video_extension.decode('utf-8'),
1580                                 'thumbnail':    video_thumbnail.decode('utf-8'),
1581                                 'description':  video_description,
1582                                 'thumbnail':    video_thumbnail,
1583                                 'description':  video_description,
1584                                 'player_url':   None,
1585                         })
1586                 except UnavailableVideoError:
1587                         self._downloader.trouble(u'ERROR: unable to download video')
1588
1589
1590 class GenericIE(InfoExtractor):
1591         """Generic last-resort information extractor."""
1592
1593         def __init__(self, downloader=None):
1594                 InfoExtractor.__init__(self, downloader)
1595
1596         @staticmethod
1597         def suitable(url):
1598                 return True
1599
1600         def report_download_webpage(self, video_id):
1601                 """Report webpage download."""
1602                 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1603                 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1604
1605         def report_extraction(self, video_id):
1606                 """Report information extraction."""
1607                 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1608
1609         def _real_initialize(self):
1610                 return
1611
1612         def _real_extract(self, url):
1613                 # At this point we have a new video
1614                 self._downloader.increment_downloads()
1615
1616                 video_id = url.split('/')[-1]
1617                 request = urllib2.Request(url)
1618                 try:
1619                         self.report_download_webpage(video_id)
1620                         webpage = urllib2.urlopen(request).read()
1621                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1622                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1623                         return
1624                 except ValueError, err:
1625                         # since this is the last-resort InfoExtractor, if
1626                         # this error is thrown, it'll be thrown here
1627                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1628                         return
1629
1630                 self.report_extraction(video_id)
1631                 # Start with something easy: JW Player in SWFObject
1632                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1633                 if mobj is None:
1634                         # Broaden the search a little bit
1635                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1636                 if mobj is None:
1637                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1638                         return
1639
1640                 # It's possible that one of the regexes
1641                 # matched, but returned an empty group:
1642                 if mobj.group(1) is None:
1643                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1644                         return
1645
1646                 video_url = urllib.unquote(mobj.group(1))
1647                 video_id  = os.path.basename(video_url)
1648
1649                 # here's a fun little line of code for you:
1650                 video_extension = os.path.splitext(video_id)[1][1:]
1651                 video_id        = os.path.splitext(video_id)[0]
1652
1653                 # it's tempting to parse this further, but you would
1654                 # have to take into account all the variations like
1655                 #   Video Title - Site Name
1656                 #   Site Name | Video Title
1657                 #   Video Title - Tagline | Site Name
1658                 # and so on and so forth; it's just not practical
1659                 mobj = re.search(r'<title>(.*)</title>', webpage)
1660                 if mobj is None:
1661                         self._downloader.trouble(u'ERROR: unable to extract title')
1662                         return
1663                 video_title = mobj.group(1).decode('utf-8')
1664                 video_title = sanitize_title(video_title)
1665                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1666
1667                 # video uploader is domain name
1668                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1669                 if mobj is None:
1670                         self._downloader.trouble(u'ERROR: unable to extract title')
1671                         return
1672                 video_uploader = mobj.group(1).decode('utf-8')
1673
1674                 try:
1675                         # Process video information
1676                         self._downloader.process_info({
1677                                 'id':           video_id.decode('utf-8'),
1678                                 'url':          video_url.decode('utf-8'),
1679                                 'uploader':     video_uploader,
1680                                 'upload_date':  u'NA',
1681                                 'title':        video_title,
1682                                 'stitle':       simple_title,
1683                                 'ext':          video_extension.decode('utf-8'),
1684                                 'format':       u'NA',
1685                                 'player_url':   None,
1686                         })
1687                 except UnavailableVideoError, err:
1688                         self._downloader.trouble(u'ERROR: unable to download video')
1689
1690
1691 class YoutubeSearchIE(InfoExtractor):
1692         """Information Extractor for YouTube search queries."""
1693         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1694         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1695         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1696         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1697         _youtube_ie = None
1698         _max_youtube_results = 1000
1699
1700         def __init__(self, youtube_ie, downloader=None):
1701                 InfoExtractor.__init__(self, downloader)
1702                 self._youtube_ie = youtube_ie
1703
1704         @staticmethod
1705         def suitable(url):
1706                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1707
1708         def report_download_page(self, query, pagenum):
1709                 """Report attempt to download playlist page with given number."""
1710                 query = query.decode(preferredencoding())
1711                 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1712
1713         def _real_initialize(self):
1714                 self._youtube_ie.initialize()
1715
1716         def _real_extract(self, query):
1717                 mobj = re.match(self._VALID_QUERY, query)
1718                 if mobj is None:
1719                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1720                         return
1721
1722                 prefix, query = query.split(':')
1723                 prefix = prefix[8:]
1724                 query  = query.encode('utf-8')
1725                 if prefix == '':
1726                         self._download_n_results(query, 1)
1727                         return
1728                 elif prefix == 'all':
1729                         self._download_n_results(query, self._max_youtube_results)
1730                         return
1731                 else:
1732                         try:
1733                                 n = long(prefix)
1734                                 if n <= 0:
1735                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1736                                         return
1737                                 elif n > self._max_youtube_results:
1738                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
1739                                         n = self._max_youtube_results
1740                                 self._download_n_results(query, n)
1741                                 return
1742                         except ValueError: # parsing prefix as integer fails
1743                                 self._download_n_results(query, 1)
1744                                 return
1745
1746         def _download_n_results(self, query, n):
1747                 """Downloads a specified number of results for a query"""
1748
1749                 video_ids = []
1750                 already_seen = set()
1751                 pagenum = 1
1752
1753                 while True:
1754                         self.report_download_page(query, pagenum)
1755                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1756                         request = urllib2.Request(result_url, None, std_headers)
1757                         try:
1758                                 page = urllib2.urlopen(request).read()
1759                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1760                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1761                                 return
1762
1763                         # Extract video identifiers
1764                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1765                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1766                                 if video_id not in already_seen:
1767                                         video_ids.append(video_id)
1768                                         already_seen.add(video_id)
1769                                         if len(video_ids) == n:
1770                                                 # Specified n videos reached
1771                                                 for id in video_ids:
1772                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1773                                                 return
1774
1775                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1776                                 for id in video_ids:
1777                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1778                                 return
1779
1780                         pagenum = pagenum + 1
1781
1782 class GoogleSearchIE(InfoExtractor):
1783         """Information Extractor for Google Video search queries."""
1784         _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
1785         _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1786         _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
1787         _MORE_PAGES_INDICATOR = r'<span>Next</span>'
1788         _google_ie = None
1789         _max_google_results = 1000
1790
1791         def __init__(self, google_ie, downloader=None):
1792                 InfoExtractor.__init__(self, downloader)
1793                 self._google_ie = google_ie
1794
1795         @staticmethod
1796         def suitable(url):
1797                 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
1798
1799         def report_download_page(self, query, pagenum):
1800                 """Report attempt to download playlist page with given number."""
1801                 query = query.decode(preferredencoding())
1802                 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1803
1804         def _real_initialize(self):
1805                 self._google_ie.initialize()
1806
1807         def _real_extract(self, query):
1808                 mobj = re.match(self._VALID_QUERY, query)
1809                 if mobj is None:
1810                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1811                         return
1812
1813                 prefix, query = query.split(':')
1814                 prefix = prefix[8:]
1815                 query  = query.encode('utf-8')
1816                 if prefix == '':
1817                         self._download_n_results(query, 1)
1818                         return
1819                 elif prefix == 'all':
1820                         self._download_n_results(query, self._max_google_results)
1821                         return
1822                 else:
1823                         try:
1824                                 n = long(prefix)
1825                                 if n <= 0:
1826                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1827                                         return
1828                                 elif n > self._max_google_results:
1829                                         self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)'  % (self._max_google_results, n))
1830                                         n = self._max_google_results
1831                                 self._download_n_results(query, n)
1832                                 return
1833                         except ValueError: # parsing prefix as integer fails
1834                                 self._download_n_results(query, 1)
1835                                 return
1836
1837         def _download_n_results(self, query, n):
1838                 """Downloads a specified number of results for a query"""
1839
1840                 video_ids = []
1841                 already_seen = set()
1842                 pagenum = 1
1843
1844                 while True:
1845                         self.report_download_page(query, pagenum)
1846                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1847                         request = urllib2.Request(result_url, None, std_headers)
1848                         try:
1849                                 page = urllib2.urlopen(request).read()
1850                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1851                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1852                                 return
1853
1854                         # Extract video identifiers
1855                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1856                                 video_id = mobj.group(1)
1857                                 if video_id not in already_seen:
1858                                         video_ids.append(video_id)
1859                                         already_seen.add(video_id)
1860                                         if len(video_ids) == n:
1861                                                 # Specified n videos reached
1862                                                 for id in video_ids:
1863                                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1864                                                 return
1865
1866                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1867                                 for id in video_ids:
1868                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1869                                 return
1870
1871                         pagenum = pagenum + 1
1872
1873 class YahooSearchIE(InfoExtractor):
1874         """Information Extractor for Yahoo! Video search queries."""
1875         _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
1876         _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1877         _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1878         _MORE_PAGES_INDICATOR = r'\s*Next'
1879         _yahoo_ie = None
1880         _max_yahoo_results = 1000
1881
1882         def __init__(self, yahoo_ie, downloader=None):
1883                 InfoExtractor.__init__(self, downloader)
1884                 self._yahoo_ie = yahoo_ie
1885
1886         @staticmethod
1887         def suitable(url):
1888                 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
1889
1890         def report_download_page(self, query, pagenum):
1891                 """Report attempt to download playlist page with given number."""
1892                 query = query.decode(preferredencoding())
1893                 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1894
1895         def _real_initialize(self):
1896                 self._yahoo_ie.initialize()
1897
1898         def _real_extract(self, query):
1899                 mobj = re.match(self._VALID_QUERY, query)
1900                 if mobj is None:
1901                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1902                         return
1903
1904                 prefix, query = query.split(':')
1905                 prefix = prefix[8:]
1906                 query  = query.encode('utf-8')
1907                 if prefix == '':
1908                         self._download_n_results(query, 1)
1909                         return
1910                 elif prefix == 'all':
1911                         self._download_n_results(query, self._max_yahoo_results)
1912                         return
1913                 else:
1914                         try:
1915                                 n = long(prefix)
1916                                 if n <= 0:
1917                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1918                                         return
1919                                 elif n > self._max_yahoo_results:
1920                                         self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)'  % (self._max_yahoo_results, n))
1921                                         n = self._max_yahoo_results
1922                                 self._download_n_results(query, n)
1923                                 return
1924                         except ValueError: # parsing prefix as integer fails
1925                                 self._download_n_results(query, 1)
1926                                 return
1927
1928         def _download_n_results(self, query, n):
1929                 """Downloads a specified number of results for a query"""
1930
1931                 video_ids = []
1932                 already_seen = set()
1933                 pagenum = 1
1934
1935                 while True:
1936                         self.report_download_page(query, pagenum)
1937                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1938                         request = urllib2.Request(result_url, None, std_headers)
1939                         try:
1940                                 page = urllib2.urlopen(request).read()
1941                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1942                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1943                                 return
1944
1945                         # Extract video identifiers
1946                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1947                                 video_id = mobj.group(1)
1948                                 if video_id not in already_seen:
1949                                         video_ids.append(video_id)
1950                                         already_seen.add(video_id)
1951                                         if len(video_ids) == n:
1952                                                 # Specified n videos reached
1953                                                 for id in video_ids:
1954                                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1955                                                 return
1956
1957                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1958                                 for id in video_ids:
1959                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1960                                 return
1961
1962                         pagenum = pagenum + 1
1963
1964 class YoutubePlaylistIE(InfoExtractor):
1965         """Information Extractor for YouTube playlists."""
1966
1967         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
1968         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
1969         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
1970         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1971         _youtube_ie = None
1972
1973         def __init__(self, youtube_ie, downloader=None):
1974                 InfoExtractor.__init__(self, downloader)
1975                 self._youtube_ie = youtube_ie
1976
1977         @staticmethod
1978         def suitable(url):
1979                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
1980
1981         def report_download_page(self, playlist_id, pagenum):
1982                 """Report attempt to download playlist page with given number."""
1983                 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1984
1985         def _real_initialize(self):
1986                 self._youtube_ie.initialize()
1987
1988         def _real_extract(self, url):
1989                 # Extract playlist id
1990                 mobj = re.match(self._VALID_URL, url)
1991                 if mobj is None:
1992                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1993                         return
1994
1995                 # Download playlist pages
1996                 playlist_id = mobj.group(1)
1997                 video_ids = []
1998                 pagenum = 1
1999
2000                 while True:
2001                         self.report_download_page(playlist_id, pagenum)
2002                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
2003                         try:
2004                                 page = urllib2.urlopen(request).read()
2005                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2006                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2007                                 return
2008
2009                         # Extract video identifiers
2010                         ids_in_page = []
2011                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2012                                 if mobj.group(1) not in ids_in_page:
2013                                         ids_in_page.append(mobj.group(1))
2014                         video_ids.extend(ids_in_page)
2015
2016                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2017                                 break
2018                         pagenum = pagenum + 1
2019
2020                 playliststart = self._downloader.params.get('playliststart', 1) - 1
2021                 playlistend = self._downloader.params.get('playlistend', -1)
2022                 video_ids = video_ids[playliststart:playlistend]
2023
2024                 for id in video_ids:
2025                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2026                 return
2027
2028 class YoutubeUserIE(InfoExtractor):
2029         """Information Extractor for YouTube users."""
2030
2031         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)'
2032         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
2033         _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this.
2034         _youtube_ie = None
2035
2036         def __init__(self, youtube_ie, downloader=None):
2037                 InfoExtractor.__init__(self, downloader)
2038                 self._youtube_ie = youtube_ie
2039
2040         @staticmethod
2041         def suitable(url):
2042                 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
2043
2044         def report_download_page(self, username):
2045                 """Report attempt to download user page."""
2046                 self._downloader.to_screen(u'[youtube] user %s: Downloading page ' % (username))
2047
2048         def _real_initialize(self):
2049                 self._youtube_ie.initialize()
2050
2051         def _real_extract(self, url):
2052                 # Extract username
2053                 mobj = re.match(self._VALID_URL, url)
2054                 if mobj is None:
2055                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2056                         return
2057
2058                 # Download user page
2059                 username = mobj.group(1)
2060                 video_ids = []
2061                 pagenum = 1
2062
2063                 self.report_download_page(username)
2064                 request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
2065                 try:
2066                         page = urllib2.urlopen(request).read()
2067                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2068                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2069                         return
2070
2071                 # Extract video identifiers
2072                 ids_in_page = []
2073
2074                 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2075                         if mobj.group(1) not in ids_in_page:
2076                                 ids_in_page.append(mobj.group(1))
2077                 video_ids.extend(ids_in_page)
2078
2079                 playliststart = self._downloader.params.get('playliststart', 1) - 1
2080                 playlistend = self._downloader.params.get('playlistend', -1)
2081                 video_ids = video_ids[playliststart:playlistend]
2082
2083                 for id in video_ids:
2084                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2085                 return
2086
2087 class DepositFilesIE(InfoExtractor):
2088         """Information extractor for depositfiles.com"""
2089
2090         _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)'
2091
2092         def __init__(self, downloader=None):
2093                 InfoExtractor.__init__(self, downloader)
2094
2095         @staticmethod
2096         def suitable(url):
2097                 return (re.match(DepositFilesIE._VALID_URL, url) is not None)
2098
2099         def report_download_webpage(self, file_id):
2100                 """Report webpage download."""
2101                 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
2102
2103         def report_extraction(self, file_id):
2104                 """Report information extraction."""
2105                 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
2106
2107         def _real_initialize(self):
2108                 return
2109
2110         def _real_extract(self, url):
2111                 # At this point we have a new file
2112                 self._downloader.increment_downloads()
2113
2114                 file_id = url.split('/')[-1]
2115                 # Rebuild url in english locale
2116                 url = 'http://depositfiles.com/en/files/' + file_id
2117
2118                 # Retrieve file webpage with 'Free download' button pressed
2119                 free_download_indication = { 'gateway_result' : '1' }
2120                 request = urllib2.Request(url, urllib.urlencode(free_download_indication), std_headers)
2121                 try:
2122                         self.report_download_webpage(file_id)
2123                         webpage = urllib2.urlopen(request).read()
2124                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2125                         self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
2126                         return
2127
2128                 # Search for the real file URL
2129                 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
2130                 if (mobj is None) or (mobj.group(1) is None):
2131                         # Try to figure out reason of the error.
2132                         mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
2133                         if (mobj is not None) and (mobj.group(1) is not None):
2134                                 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
2135                                 self._downloader.trouble(u'ERROR: %s' % restriction_message)
2136                         else:
2137                                 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
2138                         return
2139
2140                 file_url = mobj.group(1)
2141                 file_extension = os.path.splitext(file_url)[1][1:]
2142
2143                 # Search for file title
2144                 mobj = re.search(r'<b title="(.*?)">', webpage)
2145                 if mobj is None:
2146                         self._downloader.trouble(u'ERROR: unable to extract title')
2147                         return
2148                 file_title = mobj.group(1).decode('utf-8')
2149
2150                 try:
2151                         # Process file information
2152                         self._downloader.process_info({
2153                                 'id':           file_id.decode('utf-8'),
2154                                 'url':          file_url.decode('utf-8'),
2155                                 'uploader':     u'NA',
2156                                 'upload_date':  u'NA',
2157                                 'title':        file_title,
2158                                 'stitle':       file_title,
2159                                 'ext':          file_extension.decode('utf-8'),
2160                                 'format':       u'NA',
2161                                 'player_url':   None,
2162                         })
2163                 except UnavailableVideoError, err:
2164                         self._downloader.trouble(u'ERROR: unable to download file')
2165
2166 class PostProcessor(object):
2167         """Post Processor class.
2168
2169         PostProcessor objects can be added to downloaders with their
2170         add_post_processor() method. When the downloader has finished a
2171         successful download, it will take its internal chain of PostProcessors
2172         and start calling the run() method on each one of them, first with
2173         an initial argument and then with the returned value of the previous
2174         PostProcessor.
2175
2176         The chain will be stopped if one of them ever returns None or the end
2177         of the chain is reached.
2178
2179         PostProcessor objects follow a "mutual registration" process similar
2180         to InfoExtractor objects.
2181         """
2182
2183         _downloader = None
2184
2185         def __init__(self, downloader=None):
2186                 self._downloader = downloader
2187
2188         def set_downloader(self, downloader):
2189                 """Sets the downloader for this PP."""
2190                 self._downloader = downloader
2191
2192         def run(self, information):
2193                 """Run the PostProcessor.
2194
2195                 The "information" argument is a dictionary like the ones
2196                 composed by InfoExtractors. The only difference is that this
2197                 one has an extra field called "filepath" that points to the
2198                 downloaded file.
2199
2200                 When this method returns None, the postprocessing chain is
2201                 stopped. However, this method may return an information
2202                 dictionary that will be passed to the next postprocessing
2203                 object in the chain. It can be the one it received after
2204                 changing some fields.
2205
2206                 In addition, this method may raise a PostProcessingError
2207                 exception that will be taken into account by the downloader
2208                 it was called from.
2209                 """
2210                 return information # by default, do nothing
2211
2212 ### MAIN PROGRAM ###
2213 if __name__ == '__main__':
2214         try:
2215                 # Modules needed only when running the main program
2216                 import getpass
2217                 import optparse
2218
2219                 # Function to update the program file with the latest version from bitbucket.org
2220                 def update_self(downloader, filename):
2221                         # Note: downloader only used for options
2222                         if not os.access (filename, os.W_OK):
2223                                 sys.exit('ERROR: no write permissions on %s' % filename)
2224
2225                         downloader.to_screen('Updating to latest stable version...')
2226                         latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
2227                         latest_version = urllib.urlopen(latest_url).read().strip()
2228                         prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
2229                         newcontent = urllib.urlopen(prog_url).read()
2230                         stream = open(filename, 'w')
2231                         stream.write(newcontent)
2232                         stream.close()
2233                         downloader.to_screen('Updated to version %s' % latest_version)
2234
2235                 # Parse command line
2236                 parser = optparse.OptionParser(
2237                         usage='Usage: %prog [options] url...',
2238                         version='2010.12.09',
2239                         conflict_handler='resolve',
2240                 )
2241
2242                 parser.add_option('-h', '--help',
2243                                 action='help', help='print this help text and exit')
2244                 parser.add_option('-v', '--version',
2245                                 action='version', help='print program version and exit')
2246                 parser.add_option('-U', '--update',
2247                                 action='store_true', dest='update_self', help='update this program to latest stable version')
2248                 parser.add_option('-i', '--ignore-errors',
2249                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
2250                 parser.add_option('-r', '--rate-limit',
2251                                 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
2252                 parser.add_option('-R', '--retries',
2253                                 dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
2254                 parser.add_option('--playlist-start',
2255                                 dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
2256                 parser.add_option('--playlist-end',
2257                                 dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
2258
2259                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
2260                 authentication.add_option('-u', '--username',
2261                                 dest='username', metavar='USERNAME', help='account username')
2262                 authentication.add_option('-p', '--password',
2263                                 dest='password', metavar='PASSWORD', help='account password')
2264                 authentication.add_option('-n', '--netrc',
2265                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
2266                 parser.add_option_group(authentication)
2267
2268                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
2269                 video_format.add_option('-f', '--format',
2270                                 action='store', dest='format', metavar='FORMAT', help='video format code')
2271                 video_format.add_option('-m', '--mobile-version',
2272                                 action='store_const', dest='format', help='alias for -f 17', const='17')
2273                 video_format.add_option('--all-formats',
2274                                 action='store_const', dest='format', help='download all available video formats', const='-1')
2275                 video_format.add_option('--max-quality',
2276                                 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
2277                 video_format.add_option('-b', '--best-quality',
2278                                 action='store_true', dest='bestquality', help='download the best video quality (DEPRECATED)')
2279                 parser.add_option_group(video_format)
2280
2281                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
2282                 verbosity.add_option('-q', '--quiet',
2283                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
2284                 verbosity.add_option('-s', '--simulate',
2285                                 action='store_true', dest='simulate', help='do not download video', default=False)
2286                 verbosity.add_option('-g', '--get-url',
2287                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
2288                 verbosity.add_option('-e', '--get-title',
2289                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
2290                 verbosity.add_option('--get-thumbnail',
2291                                 action='store_true', dest='getthumbnail', help='simulate, quiet but print thumbnail URL', default=False)
2292                 verbosity.add_option('--get-description',
2293                                 action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False)
2294                 verbosity.add_option('--no-progress',
2295                                 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
2296                 parser.add_option_group(verbosity)
2297
2298                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
2299                 filesystem.add_option('-t', '--title',
2300                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
2301                 filesystem.add_option('-l', '--literal',
2302                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
2303                 filesystem.add_option('-A', '--auto-number',
2304                                 action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False)
2305                 filesystem.add_option('-o', '--output',
2306                                 dest='outtmpl', metavar='TEMPLATE', help='output filename template')
2307                 filesystem.add_option('-a', '--batch-file',
2308                                 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
2309                 filesystem.add_option('-w', '--no-overwrites',
2310                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
2311                 filesystem.add_option('-c', '--continue',
2312                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
2313                 filesystem.add_option('--cookies',
2314                                 dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
2315                 parser.add_option_group(filesystem)
2316
2317                 (opts, args) = parser.parse_args()
2318
2319                 # Open appropriate CookieJar
2320                 if opts.cookiefile is None:
2321                         jar = cookielib.CookieJar()
2322                 else:
2323                         try:
2324                                 jar = cookielib.MozillaCookieJar(opts.cookiefile)
2325                                 if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
2326                                         jar.load()
2327                         except (IOError, OSError), err:
2328                                 sys.exit(u'ERROR: unable to open cookie file')
2329
2330                 # General configuration
2331                 cookie_processor = urllib2.HTTPCookieProcessor(jar)
2332                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
2333                 urllib2.install_opener(urllib2.build_opener(cookie_processor))
2334                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
2335
2336                 # Batch file verification
2337                 batchurls = []
2338                 if opts.batchfile is not None:
2339                         try:
2340                                 if opts.batchfile == '-':
2341                                         batchfd = sys.stdin
2342                                 else:
2343                                         batchfd = open(opts.batchfile, 'r')
2344                                 batchurls = batchfd.readlines()
2345                                 batchurls = [x.strip() for x in batchurls]
2346                                 batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
2347                         except IOError:
2348                                 sys.exit(u'ERROR: batch file could not be read')
2349                 all_urls = batchurls + args
2350
2351                 # Conflicting, missing and erroneous options
2352                 if opts.bestquality:
2353                         print >>sys.stderr, u'\nWARNING: -b/--best-quality IS DEPRECATED AS IT IS THE DEFAULT BEHAVIOR NOW\n'
2354                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
2355                         parser.error(u'using .netrc conflicts with giving username/password')
2356                 if opts.password is not None and opts.username is None:
2357                         parser.error(u'account username missing')
2358                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber):
2359                         parser.error(u'using output template conflicts with using title, literal title or auto number')
2360                 if opts.usetitle and opts.useliteral:
2361                         parser.error(u'using title conflicts with using literal title')
2362                 if opts.username is not None and opts.password is None:
2363                         opts.password = getpass.getpass(u'Type account password and press return:')
2364                 if opts.ratelimit is not None:
2365                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
2366                         if numeric_limit is None:
2367                                 parser.error(u'invalid rate limit specified')
2368                         opts.ratelimit = numeric_limit
2369                 if opts.retries is not None:
2370                         try:
2371                                 opts.retries = long(opts.retries)
2372                         except (TypeError, ValueError), err:
2373                                 parser.error(u'invalid retry count specified')
2374                 try:
2375                         opts.playliststart = long(opts.playliststart)
2376                         if opts.playliststart <= 0:
2377                                 raise ValueError
2378                 except (TypeError, ValueError), err:
2379                         parser.error(u'invalid playlist start number specified')
2380                 try:
2381                         opts.playlistend = long(opts.playlistend)
2382                         if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
2383                                 raise ValueError
2384                 except (TypeError, ValueError), err:
2385                         parser.error(u'invalid playlist end number specified')
2386
2387                 # Information extractors
2388                 youtube_ie = YoutubeIE()
2389                 metacafe_ie = MetacafeIE(youtube_ie)
2390                 dailymotion_ie = DailymotionIE()
2391                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
2392                 youtube_user_ie = YoutubeUserIE(youtube_ie)
2393                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
2394                 google_ie = GoogleIE()
2395                 google_search_ie = GoogleSearchIE(google_ie)
2396                 photobucket_ie = PhotobucketIE()
2397                 yahoo_ie = YahooIE()
2398                 yahoo_search_ie = YahooSearchIE(yahoo_ie)
2399                 deposit_files_ie = DepositFilesIE()
2400                 generic_ie = GenericIE()
2401
2402                 # File downloader
2403                 fd = FileDownloader({
2404                         'usenetrc': opts.usenetrc,
2405                         'username': opts.username,
2406                         'password': opts.password,
2407                         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2408                         'forceurl': opts.geturl,
2409                         'forcetitle': opts.gettitle,
2410                         'forcethumbnail': opts.getthumbnail,
2411                         'forcedescription': opts.getdescription,
2412                         'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2413                         'format': opts.format,
2414                         'format_limit': opts.format_limit,
2415                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
2416                                 or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
2417                                 or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
2418                                 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
2419                                 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
2420                                 or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
2421                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
2422                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
2423                                 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
2424                                 or u'%(id)s.%(ext)s'),
2425                         'ignoreerrors': opts.ignoreerrors,
2426                         'ratelimit': opts.ratelimit,
2427                         'nooverwrites': opts.nooverwrites,
2428                         'retries': opts.retries,
2429                         'continuedl': opts.continue_dl,
2430                         'noprogress': opts.noprogress,
2431                         'playliststart': opts.playliststart,
2432                         'playlistend': opts.playlistend,
2433                         'logtostderr': opts.outtmpl == '-',
2434                         })
2435                 fd.add_info_extractor(youtube_search_ie)
2436                 fd.add_info_extractor(youtube_pl_ie)
2437                 fd.add_info_extractor(youtube_user_ie)
2438                 fd.add_info_extractor(metacafe_ie)
2439                 fd.add_info_extractor(dailymotion_ie)
2440                 fd.add_info_extractor(youtube_ie)
2441                 fd.add_info_extractor(google_ie)
2442                 fd.add_info_extractor(google_search_ie)
2443                 fd.add_info_extractor(photobucket_ie)
2444                 fd.add_info_extractor(yahoo_ie)
2445                 fd.add_info_extractor(yahoo_search_ie)
2446                 fd.add_info_extractor(deposit_files_ie)
2447
2448                 # This must come last since it's the
2449                 # fallback if none of the others work
2450                 fd.add_info_extractor(generic_ie)
2451
2452                 # Update version
2453                 if opts.update_self:
2454                         update_self(fd, sys.argv[0])
2455
2456                 # Maybe do nothing
2457                 if len(all_urls) < 1:
2458                         if not opts.update_self:
2459                                 parser.error(u'you must provide at least one URL')
2460                         else:
2461                                 sys.exit()
2462                 retcode = fd.download(all_urls)
2463
2464                 # Dump cookie jar if requested
2465                 if opts.cookiefile is not None:
2466                         try:
2467                                 jar.save()
2468                         except (IOError, OSError), err:
2469                                 sys.exit(u'ERROR: unable to save cookie jar')
2470
2471                 sys.exit(retcode)
2472
2473         except DownloadError:
2474                 sys.exit(1)
2475         except SameFileError:
2476                 sys.exit(u'ERROR: fixed output name but more than one file to download')
2477         except KeyboardInterrupt:
2478                 sys.exit(u'\nERROR: Interrupted by user')