youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # Author: Benjamin Johnson
   6 # Author: Vasyl' Vavrychuk
   7 # Author: Witold Baryluk
   8 # Author: Paweł Paprota
   9 # License: Public domain code
  10 import cookielib
  11 import ctypes
  12 import datetime
  13 import email.utils
  14 import gzip
  15 import htmlentitydefs
  16 import httplib
  17 import locale
  18 import math
  19 import netrc
  20 import os
  21 import os.path
  22 import re
  23 import socket
  24 import string
  25 import StringIO
  26 import subprocess
  27 import sys
  28 import time
  29 import urllib
  30 import urllib2
  31 import zlib
  32
  33 # parse_qs was moved from the cgi module to the urlparse module recently.
  34 try:
  35         from urlparse import parse_qs
  36 except ImportError:
  37         from cgi import parse_qs
  38
  39 std_headers = {
  40         'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12',
  41         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  42         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  43         'Accept-Encoding': 'gzip, deflate',
  44         'Accept-Language': 'en-us,en;q=0.5',
  45 }
  46
  47 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  48
  49 def preferredencoding():
  50         """Get preferred encoding.
  51
  52         Returns the best encoding scheme for the system, based on
  53         locale.getpreferredencoding() and some further tweaks.
  54         """
  55         def yield_preferredencoding():
  56                 try:
  57                         pref = locale.getpreferredencoding()
  58                         u'TEST'.encode(pref)
  59                 except:
  60                         pref = 'UTF-8'
  61                 while True:
  62                         yield pref
  63         return yield_preferredencoding().next()
  64
  65 def htmlentity_transform(matchobj):
  66         """Transforms an HTML entity to a Unicode character.
  67
  68         This function receives a match object and is intended to be used with
  69         the re.sub() function.
  70         """
  71         entity = matchobj.group(1)
  72
  73         # Known non-numeric HTML entity
  74         if entity in htmlentitydefs.name2codepoint:
  75                 return unichr(htmlentitydefs.name2codepoint[entity])
  76
  77         # Unicode character
  78         mobj = re.match(ur'(?u)#(x?\d+)', entity)
  79         if mobj is not None:
  80                 numstr = mobj.group(1)
  81                 if numstr.startswith(u'x'):
  82                         base = 16
  83                         numstr = u'0%s' % numstr
  84                 else:
  85                         base = 10
  86                 return unichr(long(numstr, base))
  87
  88         # Unknown entity in name, return its literal representation
  89         return (u'&%s;' % entity)
  90
  91 def sanitize_title(utitle):
  92         """Sanitizes a video title so it could be used as part of a filename."""
  93         utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
  94         return utitle.replace(unicode(os.sep), u'%')
  95
  96 def sanitize_open(filename, open_mode):
  97         """Try to open the given filename, and slightly tweak it if this fails.
  98
  99         Attempts to open the given filename. If this fails, it tries to change
 100         the filename slightly, step by step, until it's either able to open it
 101         or it fails and raises a final exception, like the standard open()
 102         function.
 103
 104         It returns the tuple (stream, definitive_file_name).
 105         """
 106         try:
 107                 if filename == u'-':
 108                         if sys.platform == 'win32':
 109                                 import msvcrt
 110                                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 111                         return (sys.stdout, filename)
 112                 stream = open(filename, open_mode)
 113                 return (stream, filename)
 114         except (IOError, OSError), err:
 115                 # In case of error, try to remove win32 forbidden chars
 116                 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 117
 118                 # An exception here should be caught in the caller
 119                 stream = open(filename, open_mode)
 120                 return (stream, filename)
 121
 122 def timeconvert(timestr):
 123     """Convert RFC 2822 defined time string into system timestamp"""
 124     timestamp = None
 125     timetuple = email.utils.parsedate_tz(timestr)
 126     if timetuple is not None:
 127         timestamp = email.utils.mktime_tz(timetuple)
 128     return timestamp
 129
 130 class DownloadError(Exception):
 131         """Download Error exception.
 132
 133         This exception may be thrown by FileDownloader objects if they are not
 134         configured to continue on errors. They will contain the appropriate
 135         error message.
 136         """
 137         pass
 138
 139 class SameFileError(Exception):
 140         """Same File exception.
 141
 142         This exception will be thrown by FileDownloader objects if they detect
 143         multiple files would have to be downloaded to the same file on disk.
 144         """
 145         pass
 146
 147 class PostProcessingError(Exception):
 148         """Post Processing exception.
 149
 150         This exception may be raised by PostProcessor's .run() method to
 151         indicate an error in the postprocessing task.
 152         """
 153         pass
 154
 155 class UnavailableVideoError(Exception):
 156         """Unavailable Format exception.
 157
 158         This exception will be thrown when a video is requested
 159         in a format that is not available for that video.
 160         """
 161         pass
 162
 163 class ContentTooShortError(Exception):
 164         """Content Too Short exception.
 165
 166         This exception may be raised by FileDownloader objects when a file they
 167         download is too small for what the server announced first, indicating
 168         the connection was probably interrupted.
 169         """
 170         # Both in bytes
 171         downloaded = None
 172         expected = None
 173
 174         def __init__(self, downloaded, expected):
 175                 self.downloaded = downloaded
 176                 self.expected = expected
 177
 178 class YoutubeDLHandler(urllib2.HTTPHandler):
 179         """Handler for HTTP requests and responses.
 180
 181         This class, when installed with an OpenerDirector, automatically adds
 182         the standard headers to every HTTP request and handles gzipped and
 183         deflated responses from web servers. If compression is to be avoided in
 184         a particular request, the original request in the program code only has
 185         to include the HTTP header "Youtubedl-No-Compression", which will be
 186         removed before making the real request.
 187
 188         Part of this code was copied from:
 189
 190           http://techknack.net/python-urllib2-handlers/
 191
 192         Andrew Rowls, the author of that code, agreed to release it to the
 193         public domain.
 194         """
 195
 196         @staticmethod
 197         def deflate(data):
 198                 try:
 199                         return zlib.decompress(data, -zlib.MAX_WBITS)
 200                 except zlib.error:
 201                         return zlib.decompress(data)
 202
 203         @staticmethod
 204         def addinfourl_wrapper(stream, headers, url, code):
 205                 if hasattr(urllib2.addinfourl, 'getcode'):
 206                         return urllib2.addinfourl(stream, headers, url, code)
 207                 ret = urllib2.addinfourl(stream, headers, url)
 208                 ret.code = code
 209                 return ret
 210
 211         def http_request(self, req):
 212                 for h in std_headers:
 213                         if h in req.headers:
 214                                 del req.headers[h]
 215                         req.add_header(h, std_headers[h])
 216                 if 'Youtubedl-no-compression' in req.headers:
 217                         if 'Accept-encoding' in req.headers:
 218                                 del req.headers['Accept-encoding']
 219                         del req.headers['Youtubedl-no-compression']
 220                 return req
 221
 222         def http_response(self, req, resp):
 223                 old_resp = resp
 224                 # gzip
 225                 if resp.headers.get('Content-encoding', '') == 'gzip':
 226                         gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
 227                         resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 228                         resp.msg = old_resp.msg
 229                 # deflate
 230                 if resp.headers.get('Content-encoding', '') == 'deflate':
 231                         gz = StringIO.StringIO(self.deflate(resp.read()))
 232                         resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 233                         resp.msg = old_resp.msg
 234                 return resp
 235
 236 class FileDownloader(object):
 237         """File Downloader class.
 238
 239         File downloader objects are the ones responsible of downloading the
 240         actual video file and writing it to disk if the user has requested
 241         it, among some other tasks. In most cases there should be one per
 242         program. As, given a video URL, the downloader doesn't know how to
 243         extract all the needed information, task that InfoExtractors do, it
 244         has to pass the URL to one of them.
 245
 246         For this, file downloader objects have a method that allows
 247         InfoExtractors to be registered in a given order. When it is passed
 248         a URL, the file downloader handles it to the first InfoExtractor it
 249         finds that reports being able to handle it. The InfoExtractor extracts
 250         all the information about the video or videos the URL refers to, and
 251         asks the FileDownloader to process the video information, possibly
 252         downloading the video.
 253
 254         File downloaders accept a lot of parameters. In order not to saturate
 255         the object constructor with arguments, it receives a dictionary of
 256         options instead. These options are available through the params
 257         attribute for the InfoExtractors to use. The FileDownloader also
 258         registers itself as the downloader in charge for the InfoExtractors
 259         that are added to it, so this is a "mutual registration".
 260
 261         Available options:
 262
 263         username:         Username for authentication purposes.
 264         password:         Password for authentication purposes.
 265         usenetrc:         Use netrc for authentication instead.
 266         quiet:            Do not print messages to stdout.
 267         forceurl:         Force printing final URL.
 268         forcetitle:       Force printing title.
 269         forcethumbnail:   Force printing thumbnail URL.
 270         forcedescription: Force printing description.
 271         forcefilename:    Force printing final filename.
 272         simulate:         Do not download the video files.
 273         format:           Video format code.
 274         format_limit:     Highest quality format to try.
 275         outtmpl:          Template for output names.
 276         ignoreerrors:     Do not stop on download errors.
 277         ratelimit:        Download speed limit, in bytes/sec.
 278         nooverwrites:     Prevent overwriting files.
 279         retries:          Number of times to retry for HTTP error 5xx
 280         continuedl:       Try to continue downloads if possible.
 281         noprogress:       Do not print the progress bar.
 282         playliststart:    Playlist item to start at.
 283         playlistend:      Playlist item to end at.
 284         logtostderr:      Log messages to stderr instead of stdout.
 285         consoletitle:     Display progress in console window's titlebar.
 286         nopart:           Do not use temporary .part files.
 287         updatetime:       Use the Last-modified header to set output file timestamps.
 288         """
 289
 290         params = None
 291         _ies = []
 292         _pps = []
 293         _download_retcode = None
 294         _num_downloads = None
 295         _screen_file = None
 296
 297         def __init__(self, params):
 298                 """Create a FileDownloader object with the given options."""
 299                 self._ies = []
 300                 self._pps = []
 301                 self._download_retcode = 0
 302                 self._num_downloads = 0
 303                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 304                 self.params = params
 305
 306         @staticmethod
 307         def pmkdir(filename):
 308                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 309                 components = filename.split(os.sep)
 310                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 311                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 312                 for dir in aggregate:
 313                         if not os.path.exists(dir):
 314                                 os.mkdir(dir)
 315
 316         @staticmethod
 317         def format_bytes(bytes):
 318                 if bytes is None:
 319                         return 'N/A'
 320                 if type(bytes) is str:
 321                         bytes = float(bytes)
 322                 if bytes == 0.0:
 323                         exponent = 0
 324                 else:
 325                         exponent = long(math.log(bytes, 1024.0))
 326                 suffix = 'bkMGTPEZY'[exponent]
 327                 converted = float(bytes) / float(1024**exponent)
 328                 return '%.2f%s' % (converted, suffix)
 329
 330         @staticmethod
 331         def calc_percent(byte_counter, data_len):
 332                 if data_len is None:
 333                         return '---.-%'
 334                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 335
 336         @staticmethod
 337         def calc_eta(start, now, total, current):
 338                 if total is None:
 339                         return '--:--'
 340                 dif = now - start
 341                 if current == 0 or dif < 0.001: # One millisecond
 342                         return '--:--'
 343                 rate = float(current) / dif
 344                 eta = long((float(total) - float(current)) / rate)
 345                 (eta_mins, eta_secs) = divmod(eta, 60)
 346                 if eta_mins > 99:
 347                         return '--:--'
 348                 return '%02d:%02d' % (eta_mins, eta_secs)
 349
 350         @staticmethod
 351         def calc_speed(start, now, bytes):
 352                 dif = now - start
 353                 if bytes == 0 or dif < 0.001: # One millisecond
 354                         return '%10s' % '---b/s'
 355                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 356
 357         @staticmethod
 358         def best_block_size(elapsed_time, bytes):
 359                 new_min = max(bytes / 2.0, 1.0)
 360                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 361                 if elapsed_time < 0.001:
 362                         return long(new_max)
 363                 rate = bytes / elapsed_time
 364                 if rate > new_max:
 365                         return long(new_max)
 366                 if rate < new_min:
 367                         return long(new_min)
 368                 return long(rate)
 369
 370         @staticmethod
 371         def parse_bytes(bytestr):
 372                 """Parse a string indicating a byte quantity into a long integer."""
 373                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 374                 if matchobj is None:
 375                         return None
 376                 number = float(matchobj.group(1))
 377                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 378                 return long(round(number * multiplier))
 379
 380         def add_info_extractor(self, ie):
 381                 """Add an InfoExtractor object to the end of the list."""
 382                 self._ies.append(ie)
 383                 ie.set_downloader(self)
 384
 385         def add_post_processor(self, pp):
 386                 """Add a PostProcessor object to the end of the chain."""
 387                 self._pps.append(pp)
 388                 pp.set_downloader(self)
 389
 390         def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
 391                 """Print message to stdout if not in quiet mode."""
 392                 try:
 393                         if not self.params.get('quiet', False):
 394                                 terminator = [u'\n', u''][skip_eol]
 395                                 print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
 396                         self._screen_file.flush()
 397                 except (UnicodeEncodeError), err:
 398                         if not ignore_encoding_errors:
 399                                 raise
 400
 401         def to_stderr(self, message):
 402                 """Print message to stderr."""
 403                 print >>sys.stderr, message.encode(preferredencoding())
 404
 405         def to_cons_title(self, message):
 406                 """Set console/terminal window title to message."""
 407                 if not self.params.get('consoletitle', False):
 408                         return
 409                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 410                         # c_wchar_p() might not be necessary if `message` is
 411                         # already of type unicode()
 412                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 413                 elif 'TERM' in os.environ:
 414                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 415
 416         def fixed_template(self):
 417                 """Checks if the output template is fixed."""
 418                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 419
 420         def trouble(self, message=None):
 421                 """Determine action to take when a download problem appears.
 422
 423                 Depending on if the downloader has been configured to ignore
 424                 download errors or not, this method may throw an exception or
 425                 not when errors are found, after printing the message.
 426                 """
 427                 if message is not None:
 428                         self.to_stderr(message)
 429                 if not self.params.get('ignoreerrors', False):
 430                         raise DownloadError(message)
 431                 self._download_retcode = 1
 432
 433         def slow_down(self, start_time, byte_counter):
 434                 """Sleep if the download speed is over the rate limit."""
 435                 rate_limit = self.params.get('ratelimit', None)
 436                 if rate_limit is None or byte_counter == 0:
 437                         return
 438                 now = time.time()
 439                 elapsed = now - start_time
 440                 if elapsed <= 0.0:
 441                         return
 442                 speed = float(byte_counter) / elapsed
 443                 if speed > rate_limit:
 444                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 445
 446         def temp_name(self, filename):
 447                 """Returns a temporary filename for the given filename."""
 448                 if self.params.get('nopart', False) or filename == u'-' or \
 449                                 (os.path.exists(filename) and not os.path.isfile(filename)):
 450                         return filename
 451                 return filename + u'.part'
 452
 453         def undo_temp_name(self, filename):
 454                 if filename.endswith(u'.part'):
 455                         return filename[:-len(u'.part')]
 456                 return filename
 457
 458         def try_rename(self, old_filename, new_filename):
 459                 try:
 460                         if old_filename == new_filename:
 461                                 return
 462                         os.rename(old_filename, new_filename)
 463                 except (IOError, OSError), err:
 464                         self.trouble(u'ERROR: unable to rename file')
 465
 466         def try_utime(self, filename, last_modified_hdr):
 467                 """Try to set the last-modified time of the given file."""
 468                 if last_modified_hdr is None:
 469                         return
 470                 if not os.path.isfile(filename):
 471                         return
 472                 timestr = last_modified_hdr
 473                 if timestr is None:
 474                         return
 475                 filetime = timeconvert(timestr)
 476                 if filetime is None:
 477                         return
 478                 try:
 479                         os.utime(filename,(time.time(), filetime))
 480                 except:
 481                         pass
 482
 483         def report_destination(self, filename):
 484                 """Report destination filename."""
 485                 self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
 486
 487         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 488                 """Report download progress."""
 489                 if self.params.get('noprogress', False):
 490                         return
 491                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 492                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 493                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 494                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 495
 496         def report_resuming_byte(self, resume_len):
 497                 """Report attempt to resume at given byte."""
 498                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 499
 500         def report_retry(self, count, retries):
 501                 """Report retry in case of HTTP error 5xx"""
 502                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 503
 504         def report_file_already_downloaded(self, file_name):
 505                 """Report file has already been fully downloaded."""
 506                 try:
 507                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 508                 except (UnicodeEncodeError), err:
 509                         self.to_screen(u'[download] The file has already been downloaded')
 510
 511         def report_unable_to_resume(self):
 512                 """Report it was impossible to resume download."""
 513                 self.to_screen(u'[download] Unable to resume')
 514
 515         def report_finish(self):
 516                 """Report download finished."""
 517                 if self.params.get('noprogress', False):
 518                         self.to_screen(u'[download] Download completed')
 519                 else:
 520                         self.to_screen(u'')
 521
 522         def increment_downloads(self):
 523                 """Increment the ordinal that assigns a number to each file."""
 524                 self._num_downloads += 1
 525
 526         def prepare_filename(self, info_dict):
 527                 """Generate the output filename."""
 528                 try:
 529                         template_dict = dict(info_dict)
 530                         template_dict['epoch'] = unicode(long(time.time()))
 531                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 532                         filename = self.params['outtmpl'] % template_dict
 533                         return filename
 534                 except (ValueError, KeyError), err:
 535                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 536                         return None
 537
 538         def process_info(self, info_dict):
 539                 """Process a single dictionary returned by an InfoExtractor."""
 540                 filename = self.prepare_filename(info_dict)
 541                 # Do nothing else if in simulate mode
 542                 if self.params.get('simulate', False):
 543                         # Forced printings
 544                         if self.params.get('forcetitle', False):
 545                                 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
 546                         if self.params.get('forceurl', False):
 547                                 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
 548                         if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 549                                 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
 550                         if self.params.get('forcedescription', False) and 'description' in info_dict:
 551                                 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
 552                         if self.params.get('forcefilename', False) and filename is not None:
 553                                 print filename.encode(preferredencoding(), 'xmlcharrefreplace')
 554
 555                         return
 556
 557                 if filename is None:
 558                         return
 559                 if self.params.get('nooverwrites', False) and os.path.exists(filename):
 560                         self.to_stderr(u'WARNING: file exists and will be skipped')
 561                         return
 562
 563                 try:
 564                         self.pmkdir(filename)
 565                 except (OSError, IOError), err:
 566                         self.trouble(u'ERROR: unable to create directories: %s' % str(err))
 567                         return
 568
 569                 try:
 570                         success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
 571                 except (OSError, IOError), err:
 572                         raise UnavailableVideoError
 573                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 574                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 575                         return
 576                 except (ContentTooShortError, ), err:
 577                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 578                         return
 579
 580                 if success:
 581                         try:
 582                                 self.post_process(filename, info_dict)
 583                         except (PostProcessingError), err:
 584                                 self.trouble(u'ERROR: postprocessing: %s' % str(err))
 585                                 return
 586
 587         def download(self, url_list):
 588                 """Download a given list of URLs."""
 589                 if len(url_list) > 1 and self.fixed_template():
 590                         raise SameFileError(self.params['outtmpl'])
 591
 592                 for url in url_list:
 593                         suitable_found = False
 594                         for ie in self._ies:
 595                                 # Go to next InfoExtractor if not suitable
 596                                 if not ie.suitable(url):
 597                                         continue
 598
 599                                 # Suitable InfoExtractor found
 600                                 suitable_found = True
 601
 602                                 # Extract information from URL and process it
 603                                 ie.extract(url)
 604
 605                                 # Suitable InfoExtractor had been found; go to next URL
 606                                 break
 607
 608                         if not suitable_found:
 609                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 610
 611                 return self._download_retcode
 612
 613         def post_process(self, filename, ie_info):
 614                 """Run the postprocessing chain on the given file."""
 615                 info = dict(ie_info)
 616                 info['filepath'] = filename
 617                 for pp in self._pps:
 618                         info = pp.run(info)
 619                         if info is None:
 620                                 break
 621
 622         def _download_with_rtmpdump(self, filename, url, player_url):
 623                 self.report_destination(filename)
 624                 tmpfilename = self.temp_name(filename)
 625
 626                 # Check for rtmpdump first
 627                 try:
 628                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 629                 except (OSError, IOError):
 630                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 631                         return False
 632
 633                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 634                 # the connection was interrumpted and resuming appears to be
 635                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 636                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 637                 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
 638                 while retval == 2 or retval == 1:
 639                         prevsize = os.path.getsize(tmpfilename)
 640                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 641                         time.sleep(5.0) # This seems to be needed
 642                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 643                         cursize = os.path.getsize(tmpfilename)
 644                         if prevsize == cursize and retval == 1:
 645                                 break
 646                 if retval == 0:
 647                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
 648                         self.try_rename(tmpfilename, filename)
 649                         return True
 650                 else:
 651                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 652                         return False
 653
 654         def _do_download(self, filename, url, player_url):
 655                 # Check file already present
 656                 if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
 657                         self.report_file_already_downloaded(filename)
 658                         return True
 659
 660                 # Attempt to download using rtmpdump
 661                 if url.startswith('rtmp'):
 662                         return self._download_with_rtmpdump(filename, url, player_url)
 663
 664                 tmpfilename = self.temp_name(filename)
 665                 stream = None
 666                 open_mode = 'wb'
 667
 668                 # Do not include the Accept-Encoding header
 669                 headers = {'Youtubedl-no-compression': 'True'}
 670                 basic_request = urllib2.Request(url, None, headers)
 671                 request = urllib2.Request(url, None, headers)
 672
 673                 # Establish possible resume length
 674                 if os.path.isfile(tmpfilename):
 675                         resume_len = os.path.getsize(tmpfilename)
 676                 else:
 677                         resume_len = 0
 678
 679                 # Request parameters in case of being able to resume
 680                 if self.params.get('continuedl', False) and resume_len != 0:
 681                         self.report_resuming_byte(resume_len)
 682                         request.add_header('Range','bytes=%d-' % resume_len)
 683                         open_mode = 'ab'
 684
 685                 count = 0
 686                 retries = self.params.get('retries', 0)
 687                 while count <= retries:
 688                         # Establish connection
 689                         try:
 690                                 data = urllib2.urlopen(request)
 691                                 break
 692                         except (urllib2.HTTPError, ), err:
 693                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 694                                         # Unexpected HTTP error
 695                                         raise
 696                                 elif err.code == 416:
 697                                         # Unable to resume (requested range not satisfiable)
 698                                         try:
 699                                                 # Open the connection again without the range header
 700                                                 data = urllib2.urlopen(basic_request)
 701                                                 content_length = data.info()['Content-Length']
 702                                         except (urllib2.HTTPError, ), err:
 703                                                 if err.code < 500 or err.code >= 600:
 704                                                         raise
 705                                         else:
 706                                                 # Examine the reported length
 707                                                 if (content_length is not None and
 708                                                     (resume_len - 100 < long(content_length) < resume_len + 100)):
 709                                                         # The file had already been fully downloaded.
 710                                                         # Explanation to the above condition: in issue #175 it was revealed that
 711                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 712                                                         # changing the file size slightly and causing problems for some users. So
 713                                                         # I decided to implement a suggested change and consider the file
 714                                                         # completely downloaded if the file size differs less than 100 bytes from
 715                                                         # the one in the hard drive.
 716                                                         self.report_file_already_downloaded(filename)
 717                                                         self.try_rename(tmpfilename, filename)
 718                                                         return True
 719                                                 else:
 720                                                         # The length does not match, we start the download over
 721                                                         self.report_unable_to_resume()
 722                                                         open_mode = 'wb'
 723                                                         break
 724                         # Retry
 725                         count += 1
 726                         if count <= retries:
 727                                 self.report_retry(count, retries)
 728
 729                 if count > retries:
 730                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 731                         return False
 732
 733                 data_len = data.info().get('Content-length', None)
 734                 if data_len is not None:
 735                         data_len = long(data_len) + resume_len
 736                 data_len_str = self.format_bytes(data_len)
 737                 byte_counter = 0 + resume_len
 738                 block_size = 1024
 739                 start = time.time()
 740                 while True:
 741                         # Download and write
 742                         before = time.time()
 743                         data_block = data.read(block_size)
 744                         after = time.time()
 745                         if len(data_block) == 0:
 746                                 break
 747                         byte_counter += len(data_block)
 748
 749                         # Open file just in time
 750                         if stream is None:
 751                                 try:
 752                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 753                                         filename = self.undo_temp_name(tmpfilename)
 754                                         self.report_destination(filename)
 755                                 except (OSError, IOError), err:
 756                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 757                                         return False
 758                         try:
 759                                 stream.write(data_block)
 760                         except (IOError, OSError), err:
 761                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 762                                 return False
 763                         block_size = self.best_block_size(after - before, len(data_block))
 764
 765                         # Progress message
 766                         percent_str = self.calc_percent(byte_counter, data_len)
 767                         eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 768                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 769                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 770
 771                         # Apply rate limit
 772                         self.slow_down(start, byte_counter - resume_len)
 773
 774                 stream.close()
 775                 self.report_finish()
 776                 if data_len is not None and byte_counter != data_len:
 777                         raise ContentTooShortError(byte_counter, long(data_len))
 778                 self.try_rename(tmpfilename, filename)
 779
 780                 # Update file modification time
 781                 if self.params.get('updatetime', True):
 782                         self.try_utime(filename, data.info().get('last-modified', None))
 783
 784                 return True
 785
 786 class InfoExtractor(object):
 787         """Information Extractor class.
 788
 789         Information extractors are the classes that, given a URL, extract
 790         information from the video (or videos) the URL refers to. This
 791         information includes the real video URL, the video title and simplified
 792         title, author and others. The information is stored in a dictionary
 793         which is then passed to the FileDownloader. The FileDownloader
 794         processes this information possibly downloading the video to the file
 795         system, among other possible outcomes. The dictionaries must include
 796         the following fields:
 797
 798         id:             Video identifier.
 799         url:            Final video URL.
 800         uploader:       Nickname of the video uploader.
 801         title:          Literal title.
 802         stitle:         Simplified title.
 803         ext:            Video filename extension.
 804         format:         Video format.
 805         player_url:     SWF Player URL (may be None).
 806
 807         The following fields are optional. Their primary purpose is to allow
 808         youtube-dl to serve as the backend for a video search function, such
 809         as the one in youtube2mp3.  They are only used when their respective
 810         forced printing functions are called:
 811
 812         thumbnail:      Full URL to a video thumbnail image.
 813         description:    One-line video description.
 814
 815         Subclasses of this one should re-define the _real_initialize() and
 816         _real_extract() methods, as well as the suitable() static method.
 817         Probably, they should also be instantiated and added to the main
 818         downloader.
 819         """
 820
 821         _ready = False
 822         _downloader = None
 823
 824         def __init__(self, downloader=None):
 825                 """Constructor. Receives an optional downloader."""
 826                 self._ready = False
 827                 self.set_downloader(downloader)
 828
 829         @staticmethod
 830         def suitable(url):
 831                 """Receives a URL and returns True if suitable for this IE."""
 832                 return False
 833
 834         def initialize(self):
 835                 """Initializes an instance (authentication, etc)."""
 836                 if not self._ready:
 837                         self._real_initialize()
 838                         self._ready = True
 839
 840         def extract(self, url):
 841                 """Extracts URL information and returns it in list of dicts."""
 842                 self.initialize()
 843                 return self._real_extract(url)
 844
 845         def set_downloader(self, downloader):
 846                 """Sets the downloader for this IE."""
 847                 self._downloader = downloader
 848
 849         def _real_initialize(self):
 850                 """Real initialization process. Redefine in subclasses."""
 851                 pass
 852
 853         def _real_extract(self, url):
 854                 """Real extraction process. Redefine in subclasses."""
 855                 pass
 856
 857 class YoutubeIE(InfoExtractor):
 858         """Information extractor for youtube.com."""
 859
 860         _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
 861         _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 862         _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
 863         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 864         _NETRC_MACHINE = 'youtube'
 865         # Listed in order of quality
 866         _available_formats = ['38', '37', '22', '45', '35', '34', '43', '18', '6', '5', '17', '13']
 867         _video_extensions = {
 868                 '13': '3gp',
 869                 '17': 'mp4',
 870                 '18': 'mp4',
 871                 '22': 'mp4',
 872                 '37': 'mp4',
 873                 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
 874                 '43': 'webm',
 875                 '45': 'webm',
 876         }
 877
 878         @staticmethod
 879         def suitable(url):
 880                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 881
 882         def report_lang(self):
 883                 """Report attempt to set language."""
 884                 self._downloader.to_screen(u'[youtube] Setting language')
 885
 886         def report_login(self):
 887                 """Report attempt to log in."""
 888                 self._downloader.to_screen(u'[youtube] Logging in')
 889
 890         def report_age_confirmation(self):
 891                 """Report attempt to confirm age."""
 892                 self._downloader.to_screen(u'[youtube] Confirming age')
 893
 894         def report_video_webpage_download(self, video_id):
 895                 """Report attempt to download video webpage."""
 896                 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
 897
 898         def report_video_info_webpage_download(self, video_id):
 899                 """Report attempt to download video info webpage."""
 900                 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
 901
 902         def report_information_extraction(self, video_id):
 903                 """Report attempt to extract video information."""
 904                 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
 905
 906         def report_unavailable_format(self, video_id, format):
 907                 """Report extracted video URL."""
 908                 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
 909
 910         def report_rtmp_download(self):
 911                 """Indicate the download will use the RTMP protocol."""
 912                 self._downloader.to_screen(u'[youtube] RTMP download detected')
 913
 914         def _real_initialize(self):
 915                 if self._downloader is None:
 916                         return
 917
 918                 username = None
 919                 password = None
 920                 downloader_params = self._downloader.params
 921
 922                 # Attempt to use provided username and password or .netrc data
 923                 if downloader_params.get('username', None) is not None:
 924                         username = downloader_params['username']
 925                         password = downloader_params['password']
 926                 elif downloader_params.get('usenetrc', False):
 927                         try:
 928                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 929                                 if info is not None:
 930                                         username = info[0]
 931                                         password = info[2]
 932                                 else:
 933                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 934                         except (IOError, netrc.NetrcParseError), err:
 935                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 936                                 return
 937
 938                 # Set language
 939                 request = urllib2.Request(self._LANG_URL)
 940                 try:
 941                         self.report_lang()
 942                         urllib2.urlopen(request).read()
 943                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 944                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 945                         return
 946
 947                 # No authentication to be performed
 948                 if username is None:
 949                         return
 950
 951                 # Log in
 952                 login_form = {
 953                                 'current_form': 'loginForm',
 954                                 'next':         '/',
 955                                 'action_login': 'Log In',
 956                                 'username':     username,
 957                                 'password':     password,
 958                                 }
 959                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
 960                 try:
 961                         self.report_login()
 962                         login_results = urllib2.urlopen(request).read()
 963                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 964                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 965                                 return
 966                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 967                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 968                         return
 969
 970                 # Confirm age
 971                 age_form = {
 972                                 'next_url':             '/',
 973                                 'action_confirm':       'Confirm',
 974                                 }
 975                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
 976                 try:
 977                         self.report_age_confirmation()
 978                         age_results = urllib2.urlopen(request).read()
 979                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 980                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 981                         return
 982
 983         def _real_extract(self, url):
 984                 # Extract video id from URL
 985                 mobj = re.match(self._VALID_URL, url)
 986                 if mobj is None:
 987                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 988                         return
 989                 video_id = mobj.group(2)
 990
 991                 # Get video webpage
 992                 self.report_video_webpage_download(video_id)
 993                 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&amp;has_verified=1' % video_id)
 994                 try:
 995                         video_webpage = urllib2.urlopen(request).read()
 996                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 997                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 998                         return
 999
1000                 # Attempt to extract SWF player URL
1001                 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1002                 if mobj is not None:
1003                         player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1004                 else:
1005                         player_url = None
1006
1007                 # Get video info
1008                 self.report_video_info_webpage_download(video_id)
1009                 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1010                         video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1011                                            % (video_id, el_type))
1012                         request = urllib2.Request(video_info_url)
1013                         try:
1014                                 video_info_webpage = urllib2.urlopen(request).read()
1015                                 video_info = parse_qs(video_info_webpage)
1016                                 if 'token' in video_info:
1017                                         break
1018                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1019                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
1020                                 return
1021                 if 'token' not in video_info:
1022                         if 'reason' in video_info:
1023                                 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
1024                         else:
1025                                 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
1026                         return
1027
1028                 # Start extracting information
1029                 self.report_information_extraction(video_id)
1030
1031                 # uploader
1032                 if 'author' not in video_info:
1033                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1034                         return
1035                 video_uploader = urllib.unquote_plus(video_info['author'][0])
1036
1037                 # title
1038                 if 'title' not in video_info:
1039                         self._downloader.trouble(u'ERROR: unable to extract video title')
1040                         return
1041                 video_title = urllib.unquote_plus(video_info['title'][0])
1042                 video_title = video_title.decode('utf-8')
1043                 video_title = sanitize_title(video_title)
1044
1045                 # simplified title
1046                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1047                 simple_title = simple_title.strip(ur'_')
1048
1049                 # thumbnail image
1050                 if 'thumbnail_url' not in video_info:
1051                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
1052                         video_thumbnail = ''
1053                 else:   # don't panic if we can't find it
1054                         video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
1055
1056                 # upload date
1057                 upload_date = u'NA'
1058                 mobj = re.search(r'id="eow-date".*?>(.*?)</span>', video_webpage, re.DOTALL)
1059                 if mobj is not None:
1060                         upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1061                         format_expressions = ['%d %B %Y', '%B %d %Y']
1062                         for expression in format_expressions:
1063                                 try:
1064                                         upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
1065                                 except:
1066                                         pass
1067
1068                 # description
1069                 video_description = 'No description available.'
1070                 if self._downloader.params.get('forcedescription', False):
1071                         mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
1072                         if mobj is not None:
1073                                 video_description = mobj.group(1)
1074
1075                 # token
1076                 video_token = urllib.unquote_plus(video_info['token'][0])
1077
1078                 # Decide which formats to download
1079                 req_format = self._downloader.params.get('format', None)
1080
1081                 if 'fmt_url_map' in video_info:
1082                         url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(','))
1083                         format_limit = self._downloader.params.get('format_limit', None)
1084                         if format_limit is not None and format_limit in self._available_formats:
1085                                 format_list = self._available_formats[self._available_formats.index(format_limit):]
1086                         else:
1087                                 format_list = self._available_formats
1088                         existing_formats = [x for x in format_list if x in url_map]
1089                         if len(existing_formats) == 0:
1090                                 self._downloader.trouble(u'ERROR: no known formats available for video')
1091                                 return
1092                         if req_format is None:
1093                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
1094                         elif req_format == '-1':
1095                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1096                         else:
1097                                 # Specific format
1098                                 if req_format not in url_map:
1099                                         self._downloader.trouble(u'ERROR: requested format not available')
1100                                         return
1101                                 video_url_list = [(req_format, url_map[req_format])] # Specific format
1102
1103                 elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1104                         self.report_rtmp_download()
1105                         video_url_list = [(None, video_info['conn'][0])]
1106
1107                 else:
1108                         self._downloader.trouble(u'ERROR: no fmt_url_map or conn information found in video info')
1109                         return
1110
1111                 for format_param, video_real_url in video_url_list:
1112                         # At this point we have a new video
1113                         self._downloader.increment_downloads()
1114
1115                         # Extension
1116                         video_extension = self._video_extensions.get(format_param, 'flv')
1117
1118                         # Find the video URL in fmt_url_map or conn paramters
1119                         try:
1120                                 # Process video information
1121                                 self._downloader.process_info({
1122                                         'id':           video_id.decode('utf-8'),
1123                                         'url':          video_real_url.decode('utf-8'),
1124                                         'uploader':     video_uploader.decode('utf-8'),
1125                                         'upload_date':  upload_date,
1126                                         'title':        video_title,
1127                                         'stitle':       simple_title,
1128                                         'ext':          video_extension.decode('utf-8'),
1129                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
1130                                         'thumbnail':    video_thumbnail.decode('utf-8'),
1131                                         'description':  video_description.decode('utf-8'),
1132                                         'player_url':   player_url,
1133                                 })
1134                         except UnavailableVideoError, err:
1135                                 self._downloader.trouble(u'\nERROR: unable to download video')
1136
1137
1138 class MetacafeIE(InfoExtractor):
1139         """Information Extractor for metacafe.com."""
1140
1141         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
1142         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
1143         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
1144         _youtube_ie = None
1145
1146         def __init__(self, youtube_ie, downloader=None):
1147                 InfoExtractor.__init__(self, downloader)
1148                 self._youtube_ie = youtube_ie
1149
1150         @staticmethod
1151         def suitable(url):
1152                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
1153
1154         def report_disclaimer(self):
1155                 """Report disclaimer retrieval."""
1156                 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
1157
1158         def report_age_confirmation(self):
1159                 """Report attempt to confirm age."""
1160                 self._downloader.to_screen(u'[metacafe] Confirming age')
1161
1162         def report_download_webpage(self, video_id):
1163                 """Report webpage download."""
1164                 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
1165
1166         def report_extraction(self, video_id):
1167                 """Report information extraction."""
1168                 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
1169
1170         def _real_initialize(self):
1171                 # Retrieve disclaimer
1172                 request = urllib2.Request(self._DISCLAIMER)
1173                 try:
1174                         self.report_disclaimer()
1175                         disclaimer = urllib2.urlopen(request).read()
1176                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1177                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1178                         return
1179
1180                 # Confirm age
1181                 disclaimer_form = {
1182                         'filters': '0',
1183                         'submit': "Continue - I'm over 18",
1184                         }
1185                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
1186                 try:
1187                         self.report_age_confirmation()
1188                         disclaimer = urllib2.urlopen(request).read()
1189                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1190                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1191                         return
1192
1193         def _real_extract(self, url):
1194                 # Extract id and simplified title from URL
1195                 mobj = re.match(self._VALID_URL, url)
1196                 if mobj is None:
1197                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1198                         return
1199
1200                 video_id = mobj.group(1)
1201
1202                 # Check if video comes from YouTube
1203                 mobj2 = re.match(r'^yt-(.*)$', video_id)
1204                 if mobj2 is not None:
1205                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1206                         return
1207
1208                 # At this point we have a new video
1209                 self._downloader.increment_downloads()
1210
1211                 simple_title = mobj.group(2).decode('utf-8')
1212
1213                 # Retrieve video webpage to extract further information
1214                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1215                 try:
1216                         self.report_download_webpage(video_id)
1217                         webpage = urllib2.urlopen(request).read()
1218                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1219                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1220                         return
1221
1222                 # Extract URL, uploader and title from webpage
1223                 self.report_extraction(video_id)
1224                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1225                 if mobj is not None:
1226                         mediaURL = urllib.unquote(mobj.group(1))
1227                         video_extension = mediaURL[-3:]
1228
1229                         # Extract gdaKey if available
1230                         mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1231                         if mobj is None:
1232                                 video_url = mediaURL
1233                         else:
1234                                 gdaKey = mobj.group(1)
1235                                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1236                 else:
1237                         mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
1238                         if mobj is None:
1239                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1240                                 return
1241                         vardict = parse_qs(mobj.group(1))
1242                         if 'mediaData' not in vardict:
1243                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1244                                 return
1245                         mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
1246                         if mobj is None:
1247                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1248                                 return
1249                         mediaURL = mobj.group(1).replace('\\/', '/')
1250                         video_extension = mediaURL[-3:]
1251                         video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
1252
1253                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1254                 if mobj is None:
1255                         self._downloader.trouble(u'ERROR: unable to extract title')
1256                         return
1257                 video_title = mobj.group(1).decode('utf-8')
1258                 video_title = sanitize_title(video_title)
1259
1260                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1261                 if mobj is None:
1262                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1263                         return
1264                 video_uploader = mobj.group(1)
1265
1266                 try:
1267                         # Process video information
1268                         self._downloader.process_info({
1269                                 'id':           video_id.decode('utf-8'),
1270                                 'url':          video_url.decode('utf-8'),
1271                                 'uploader':     video_uploader.decode('utf-8'),
1272                                 'upload_date':  u'NA',
1273                                 'title':        video_title,
1274                                 'stitle':       simple_title,
1275                                 'ext':          video_extension.decode('utf-8'),
1276                                 'format':       u'NA',
1277                                 'player_url':   None,
1278                         })
1279                 except UnavailableVideoError:
1280                         self._downloader.trouble(u'\nERROR: unable to download video')
1281
1282
1283 class DailymotionIE(InfoExtractor):
1284         """Information Extractor for Dailymotion"""
1285
1286         _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1287
1288         def __init__(self, downloader=None):
1289                 InfoExtractor.__init__(self, downloader)
1290
1291         @staticmethod
1292         def suitable(url):
1293                 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1294
1295         def report_download_webpage(self, video_id):
1296                 """Report webpage download."""
1297                 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
1298
1299         def report_extraction(self, video_id):
1300                 """Report information extraction."""
1301                 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
1302
1303         def _real_initialize(self):
1304                 return
1305
1306         def _real_extract(self, url):
1307                 # Extract id and simplified title from URL
1308                 mobj = re.match(self._VALID_URL, url)
1309                 if mobj is None:
1310                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1311                         return
1312
1313                 # At this point we have a new video
1314                 self._downloader.increment_downloads()
1315                 video_id = mobj.group(1)
1316
1317                 simple_title = mobj.group(2).decode('utf-8')
1318                 video_extension = 'flv'
1319
1320                 # Retrieve video webpage to extract further information
1321                 request = urllib2.Request(url)
1322                 try:
1323                         self.report_download_webpage(video_id)
1324                         webpage = urllib2.urlopen(request).read()
1325                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1326                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1327                         return
1328
1329                 # Extract URL, uploader and title from webpage
1330                 self.report_extraction(video_id)
1331                 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1332                 if mobj is None:
1333                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1334                         return
1335                 mediaURL = urllib.unquote(mobj.group(1))
1336
1337                 # if needed add http://www.dailymotion.com/ if relative URL
1338
1339                 video_url = mediaURL
1340
1341                 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1342                 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1343                 if mobj is None:
1344                         self._downloader.trouble(u'ERROR: unable to extract title')
1345                         return
1346                 video_title = mobj.group(1).decode('utf-8')
1347                 video_title = sanitize_title(video_title)
1348
1349                 mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
1350                 if mobj is None:
1351                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1352                         return
1353                 video_uploader = mobj.group(1)
1354
1355                 try:
1356                         # Process video information
1357                         self._downloader.process_info({
1358                                 'id':           video_id.decode('utf-8'),
1359                                 'url':          video_url.decode('utf-8'),
1360                                 'uploader':     video_uploader.decode('utf-8'),
1361                                 'upload_date':  u'NA',
1362                                 'title':        video_title,
1363                                 'stitle':       simple_title,
1364                                 'ext':          video_extension.decode('utf-8'),
1365                                 'format':       u'NA',
1366                                 'player_url':   None,
1367                         })
1368                 except UnavailableVideoError:
1369                         self._downloader.trouble(u'\nERROR: unable to download video')
1370
1371 class GoogleIE(InfoExtractor):
1372         """Information extractor for video.google.com."""
1373
1374         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1375
1376         def __init__(self, downloader=None):
1377                 InfoExtractor.__init__(self, downloader)
1378
1379         @staticmethod
1380         def suitable(url):
1381                 return (re.match(GoogleIE._VALID_URL, url) is not None)
1382
1383         def report_download_webpage(self, video_id):
1384                 """Report webpage download."""
1385                 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
1386
1387         def report_extraction(self, video_id):
1388                 """Report information extraction."""
1389                 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
1390
1391         def _real_initialize(self):
1392                 return
1393
1394         def _real_extract(self, url):
1395                 # Extract id from URL
1396                 mobj = re.match(self._VALID_URL, url)
1397                 if mobj is None:
1398                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1399                         return
1400
1401                 # At this point we have a new video
1402                 self._downloader.increment_downloads()
1403                 video_id = mobj.group(1)
1404
1405                 video_extension = 'mp4'
1406
1407                 # Retrieve video webpage to extract further information
1408                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1409                 try:
1410                         self.report_download_webpage(video_id)
1411                         webpage = urllib2.urlopen(request).read()
1412                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1413                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1414                         return
1415
1416                 # Extract URL, uploader, and title from webpage
1417                 self.report_extraction(video_id)
1418                 mobj = re.search(r"download_url:'([^']+)'", webpage)
1419                 if mobj is None:
1420                         video_extension = 'flv'
1421                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1422                 if mobj is None:
1423                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1424                         return
1425                 mediaURL = urllib.unquote(mobj.group(1))
1426                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1427                 mediaURL = mediaURL.replace('\\x26', '\x26')
1428
1429                 video_url = mediaURL
1430
1431                 mobj = re.search(r'<title>(.*)</title>', webpage)
1432                 if mobj is None:
1433                         self._downloader.trouble(u'ERROR: unable to extract title')
1434                         return
1435                 video_title = mobj.group(1).decode('utf-8')
1436                 video_title = sanitize_title(video_title)
1437                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1438
1439                 # Extract video description
1440                 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1441                 if mobj is None:
1442                         self._downloader.trouble(u'ERROR: unable to extract video description')
1443                         return
1444                 video_description = mobj.group(1).decode('utf-8')
1445                 if not video_description:
1446                         video_description = 'No description available.'
1447
1448                 # Extract video thumbnail
1449                 if self._downloader.params.get('forcethumbnail', False):
1450                         request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1451                         try:
1452                                 webpage = urllib2.urlopen(request).read()
1453                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1454                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1455                                 return
1456                         mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1457                         if mobj is None:
1458                                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1459                                 return
1460                         video_thumbnail = mobj.group(1)
1461                 else:   # we need something to pass to process_info
1462                         video_thumbnail = ''
1463
1464
1465                 try:
1466                         # Process video information
1467                         self._downloader.process_info({
1468                                 'id':           video_id.decode('utf-8'),
1469                                 'url':          video_url.decode('utf-8'),
1470                                 'uploader':     u'NA',
1471                                 'upload_date':  u'NA',
1472                                 'title':        video_title,
1473                                 'stitle':       simple_title,
1474                                 'ext':          video_extension.decode('utf-8'),
1475                                 'format':       u'NA',
1476                                 'player_url':   None,
1477                         })
1478                 except UnavailableVideoError:
1479                         self._downloader.trouble(u'\nERROR: unable to download video')
1480
1481
1482 class PhotobucketIE(InfoExtractor):
1483         """Information extractor for photobucket.com."""
1484
1485         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1486
1487         def __init__(self, downloader=None):
1488                 InfoExtractor.__init__(self, downloader)
1489
1490         @staticmethod
1491         def suitable(url):
1492                 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1493
1494         def report_download_webpage(self, video_id):
1495                 """Report webpage download."""
1496                 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
1497
1498         def report_extraction(self, video_id):
1499                 """Report information extraction."""
1500                 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
1501
1502         def _real_initialize(self):
1503                 return
1504
1505         def _real_extract(self, url):
1506                 # Extract id from URL
1507                 mobj = re.match(self._VALID_URL, url)
1508                 if mobj is None:
1509                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1510                         return
1511
1512                 # At this point we have a new video
1513                 self._downloader.increment_downloads()
1514                 video_id = mobj.group(1)
1515
1516                 video_extension = 'flv'
1517
1518                 # Retrieve video webpage to extract further information
1519                 request = urllib2.Request(url)
1520                 try:
1521                         self.report_download_webpage(video_id)
1522                         webpage = urllib2.urlopen(request).read()
1523                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1524                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1525                         return
1526
1527                 # Extract URL, uploader, and title from webpage
1528                 self.report_extraction(video_id)
1529                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1530                 if mobj is None:
1531                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1532                         return
1533                 mediaURL = urllib.unquote(mobj.group(1))
1534
1535                 video_url = mediaURL
1536
1537                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1538                 if mobj is None:
1539                         self._downloader.trouble(u'ERROR: unable to extract title')
1540                         return
1541                 video_title = mobj.group(1).decode('utf-8')
1542                 video_title = sanitize_title(video_title)
1543                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1544
1545                 video_uploader = mobj.group(2).decode('utf-8')
1546
1547                 try:
1548                         # Process video information
1549                         self._downloader.process_info({
1550                                 'id':           video_id.decode('utf-8'),
1551                                 'url':          video_url.decode('utf-8'),
1552                                 'uploader':     video_uploader,
1553                                 'upload_date':  u'NA',
1554                                 'title':        video_title,
1555                                 'stitle':       simple_title,
1556                                 'ext':          video_extension.decode('utf-8'),
1557                                 'format':       u'NA',
1558                                 'player_url':   None,
1559                         })
1560                 except UnavailableVideoError:
1561                         self._downloader.trouble(u'\nERROR: unable to download video')
1562
1563
1564 class YahooIE(InfoExtractor):
1565         """Information extractor for video.yahoo.com."""
1566
1567         # _VALID_URL matches all Yahoo! Video URLs
1568         # _VPAGE_URL matches only the extractable '/watch/' URLs
1569         _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1570         _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1571
1572         def __init__(self, downloader=None):
1573                 InfoExtractor.__init__(self, downloader)
1574
1575         @staticmethod
1576         def suitable(url):
1577                 return (re.match(YahooIE._VALID_URL, url) is not None)
1578
1579         def report_download_webpage(self, video_id):
1580                 """Report webpage download."""
1581                 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
1582
1583         def report_extraction(self, video_id):
1584                 """Report information extraction."""
1585                 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
1586
1587         def _real_initialize(self):
1588                 return
1589
1590         def _real_extract(self, url, new_video=True):
1591                 # Extract ID from URL
1592                 mobj = re.match(self._VALID_URL, url)
1593                 if mobj is None:
1594                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1595                         return
1596
1597                 # At this point we have a new video
1598                 self._downloader.increment_downloads()
1599                 video_id = mobj.group(2)
1600                 video_extension = 'flv'
1601
1602                 # Rewrite valid but non-extractable URLs as
1603                 # extractable English language /watch/ URLs
1604                 if re.match(self._VPAGE_URL, url) is None:
1605                         request = urllib2.Request(url)
1606                         try:
1607                                 webpage = urllib2.urlopen(request).read()
1608                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1609                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1610                                 return
1611
1612                         mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1613                         if mobj is None:
1614                                 self._downloader.trouble(u'ERROR: Unable to extract id field')
1615                                 return
1616                         yahoo_id = mobj.group(1)
1617
1618                         mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1619                         if mobj is None:
1620                                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1621                                 return
1622                         yahoo_vid = mobj.group(1)
1623
1624                         url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1625                         return self._real_extract(url, new_video=False)
1626
1627                 # Retrieve video webpage to extract further information
1628                 request = urllib2.Request(url)
1629                 try:
1630                         self.report_download_webpage(video_id)
1631                         webpage = urllib2.urlopen(request).read()
1632                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1633                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1634                         return
1635
1636                 # Extract uploader and title from webpage
1637                 self.report_extraction(video_id)
1638                 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1639                 if mobj is None:
1640                         self._downloader.trouble(u'ERROR: unable to extract video title')
1641                         return
1642                 video_title = mobj.group(1).decode('utf-8')
1643                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1644
1645                 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1646                 if mobj is None:
1647                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
1648                         return
1649                 video_uploader = mobj.group(1).decode('utf-8')
1650
1651                 # Extract video thumbnail
1652                 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1653                 if mobj is None:
1654                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1655                         return
1656                 video_thumbnail = mobj.group(1).decode('utf-8')
1657
1658                 # Extract video description
1659                 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1660                 if mobj is None:
1661                         self._downloader.trouble(u'ERROR: unable to extract video description')
1662                         return
1663                 video_description = mobj.group(1).decode('utf-8')
1664                 if not video_description: video_description = 'No description available.'
1665
1666                 # Extract video height and width
1667                 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1668                 if mobj is None:
1669                         self._downloader.trouble(u'ERROR: unable to extract video height')
1670                         return
1671                 yv_video_height = mobj.group(1)
1672
1673                 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1674                 if mobj is None:
1675                         self._downloader.trouble(u'ERROR: unable to extract video width')
1676                         return
1677                 yv_video_width = mobj.group(1)
1678
1679                 # Retrieve video playlist to extract media URL
1680                 # I'm not completely sure what all these options are, but we
1681                 # seem to need most of them, otherwise the server sends a 401.
1682                 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
1683                 yv_bitrate = '700'  # according to Wikipedia this is hard-coded
1684                 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1685                                           '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1686                                           '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1687                 try:
1688                         self.report_download_webpage(video_id)
1689                         webpage = urllib2.urlopen(request).read()
1690                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1691                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1692                         return
1693
1694                 # Extract media URL from playlist XML
1695                 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1696                 if mobj is None:
1697                         self._downloader.trouble(u'ERROR: Unable to extract media URL')
1698                         return
1699                 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1700                 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1701
1702                 try:
1703                         # Process video information
1704                         self._downloader.process_info({
1705                                 'id':           video_id.decode('utf-8'),
1706                                 'url':          video_url,
1707                                 'uploader':     video_uploader,
1708                                 'upload_date':  u'NA',
1709                                 'title':        video_title,
1710                                 'stitle':       simple_title,
1711                                 'ext':          video_extension.decode('utf-8'),
1712                                 'thumbnail':    video_thumbnail.decode('utf-8'),
1713                                 'description':  video_description,
1714                                 'thumbnail':    video_thumbnail,
1715                                 'description':  video_description,
1716                                 'player_url':   None,
1717                         })
1718                 except UnavailableVideoError:
1719                         self._downloader.trouble(u'\nERROR: unable to download video')
1720
1721
1722 class GenericIE(InfoExtractor):
1723         """Generic last-resort information extractor."""
1724
1725         def __init__(self, downloader=None):
1726                 InfoExtractor.__init__(self, downloader)
1727
1728         @staticmethod
1729         def suitable(url):
1730                 return True
1731
1732         def report_download_webpage(self, video_id):
1733                 """Report webpage download."""
1734                 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1735                 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1736
1737         def report_extraction(self, video_id):
1738                 """Report information extraction."""
1739                 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1740
1741         def _real_initialize(self):
1742                 return
1743
1744         def _real_extract(self, url):
1745                 # At this point we have a new video
1746                 self._downloader.increment_downloads()
1747
1748                 video_id = url.split('/')[-1]
1749                 request = urllib2.Request(url)
1750                 try:
1751                         self.report_download_webpage(video_id)
1752                         webpage = urllib2.urlopen(request).read()
1753                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1754                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1755                         return
1756                 except ValueError, err:
1757                         # since this is the last-resort InfoExtractor, if
1758                         # this error is thrown, it'll be thrown here
1759                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1760                         return
1761
1762                 self.report_extraction(video_id)
1763                 # Start with something easy: JW Player in SWFObject
1764                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1765                 if mobj is None:
1766                         # Broaden the search a little bit
1767                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1768                 if mobj is None:
1769                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1770                         return
1771
1772                 # It's possible that one of the regexes
1773                 # matched, but returned an empty group:
1774                 if mobj.group(1) is None:
1775                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1776                         return
1777
1778                 video_url = urllib.unquote(mobj.group(1))
1779                 video_id  = os.path.basename(video_url)
1780
1781                 # here's a fun little line of code for you:
1782                 video_extension = os.path.splitext(video_id)[1][1:]
1783                 video_id        = os.path.splitext(video_id)[0]
1784
1785                 # it's tempting to parse this further, but you would
1786                 # have to take into account all the variations like
1787                 #   Video Title - Site Name
1788                 #   Site Name | Video Title
1789                 #   Video Title - Tagline | Site Name
1790                 # and so on and so forth; it's just not practical
1791                 mobj = re.search(r'<title>(.*)</title>', webpage)
1792                 if mobj is None:
1793                         self._downloader.trouble(u'ERROR: unable to extract title')
1794                         return
1795                 video_title = mobj.group(1).decode('utf-8')
1796                 video_title = sanitize_title(video_title)
1797                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1798
1799                 # video uploader is domain name
1800                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1801                 if mobj is None:
1802                         self._downloader.trouble(u'ERROR: unable to extract title')
1803                         return
1804                 video_uploader = mobj.group(1).decode('utf-8')
1805
1806                 try:
1807                         # Process video information
1808                         self._downloader.process_info({
1809                                 'id':           video_id.decode('utf-8'),
1810                                 'url':          video_url.decode('utf-8'),
1811                                 'uploader':     video_uploader,
1812                                 'upload_date':  u'NA',
1813                                 'title':        video_title,
1814                                 'stitle':       simple_title,
1815                                 'ext':          video_extension.decode('utf-8'),
1816                                 'format':       u'NA',
1817                                 'player_url':   None,
1818                         })
1819                 except UnavailableVideoError, err:
1820                         self._downloader.trouble(u'\nERROR: unable to download video')
1821
1822
1823 class YoutubeSearchIE(InfoExtractor):
1824         """Information Extractor for YouTube search queries."""
1825         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1826         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1827         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1828         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1829         _youtube_ie = None
1830         _max_youtube_results = 1000
1831
1832         def __init__(self, youtube_ie, downloader=None):
1833                 InfoExtractor.__init__(self, downloader)
1834                 self._youtube_ie = youtube_ie
1835
1836         @staticmethod
1837         def suitable(url):
1838                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1839
1840         def report_download_page(self, query, pagenum):
1841                 """Report attempt to download playlist page with given number."""
1842                 query = query.decode(preferredencoding())
1843                 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1844
1845         def _real_initialize(self):
1846                 self._youtube_ie.initialize()
1847
1848         def _real_extract(self, query):
1849                 mobj = re.match(self._VALID_QUERY, query)
1850                 if mobj is None:
1851                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1852                         return
1853
1854                 prefix, query = query.split(':')
1855                 prefix = prefix[8:]
1856                 query  = query.encode('utf-8')
1857                 if prefix == '':
1858                         self._download_n_results(query, 1)
1859                         return
1860                 elif prefix == 'all':
1861                         self._download_n_results(query, self._max_youtube_results)
1862                         return
1863                 else:
1864                         try:
1865                                 n = long(prefix)
1866                                 if n <= 0:
1867                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1868                                         return
1869                                 elif n > self._max_youtube_results:
1870                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
1871                                         n = self._max_youtube_results
1872                                 self._download_n_results(query, n)
1873                                 return
1874                         except ValueError: # parsing prefix as integer fails
1875                                 self._download_n_results(query, 1)
1876                                 return
1877
1878         def _download_n_results(self, query, n):
1879                 """Downloads a specified number of results for a query"""
1880
1881                 video_ids = []
1882                 already_seen = set()
1883                 pagenum = 1
1884
1885                 while True:
1886                         self.report_download_page(query, pagenum)
1887                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1888                         request = urllib2.Request(result_url)
1889                         try:
1890                                 page = urllib2.urlopen(request).read()
1891                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1892                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1893                                 return
1894
1895                         # Extract video identifiers
1896                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1897                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1898                                 if video_id not in already_seen:
1899                                         video_ids.append(video_id)
1900                                         already_seen.add(video_id)
1901                                         if len(video_ids) == n:
1902                                                 # Specified n videos reached
1903                                                 for id in video_ids:
1904                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1905                                                 return
1906
1907                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1908                                 for id in video_ids:
1909                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1910                                 return
1911
1912                         pagenum = pagenum + 1
1913
1914 class GoogleSearchIE(InfoExtractor):
1915         """Information Extractor for Google Video search queries."""
1916         _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
1917         _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1918         _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
1919         _MORE_PAGES_INDICATOR = r'<span>Next</span>'
1920         _google_ie = None
1921         _max_google_results = 1000
1922
1923         def __init__(self, google_ie, downloader=None):
1924                 InfoExtractor.__init__(self, downloader)
1925                 self._google_ie = google_ie
1926
1927         @staticmethod
1928         def suitable(url):
1929                 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
1930
1931         def report_download_page(self, query, pagenum):
1932                 """Report attempt to download playlist page with given number."""
1933                 query = query.decode(preferredencoding())
1934                 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1935
1936         def _real_initialize(self):
1937                 self._google_ie.initialize()
1938
1939         def _real_extract(self, query):
1940                 mobj = re.match(self._VALID_QUERY, query)
1941                 if mobj is None:
1942                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1943                         return
1944
1945                 prefix, query = query.split(':')
1946                 prefix = prefix[8:]
1947                 query  = query.encode('utf-8')
1948                 if prefix == '':
1949                         self._download_n_results(query, 1)
1950                         return
1951                 elif prefix == 'all':
1952                         self._download_n_results(query, self._max_google_results)
1953                         return
1954                 else:
1955                         try:
1956                                 n = long(prefix)
1957                                 if n <= 0:
1958                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1959                                         return
1960                                 elif n > self._max_google_results:
1961                                         self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)'  % (self._max_google_results, n))
1962                                         n = self._max_google_results
1963                                 self._download_n_results(query, n)
1964                                 return
1965                         except ValueError: # parsing prefix as integer fails
1966                                 self._download_n_results(query, 1)
1967                                 return
1968
1969         def _download_n_results(self, query, n):
1970                 """Downloads a specified number of results for a query"""
1971
1972                 video_ids = []
1973                 already_seen = set()
1974                 pagenum = 1
1975
1976                 while True:
1977                         self.report_download_page(query, pagenum)
1978                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1979                         request = urllib2.Request(result_url)
1980                         try:
1981                                 page = urllib2.urlopen(request).read()
1982                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1983                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1984                                 return
1985
1986                         # Extract video identifiers
1987                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1988                                 video_id = mobj.group(1)
1989                                 if video_id not in already_seen:
1990                                         video_ids.append(video_id)
1991                                         already_seen.add(video_id)
1992                                         if len(video_ids) == n:
1993                                                 # Specified n videos reached
1994                                                 for id in video_ids:
1995                                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1996                                                 return
1997
1998                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1999                                 for id in video_ids:
2000                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
2001                                 return
2002
2003                         pagenum = pagenum + 1
2004
2005 class YahooSearchIE(InfoExtractor):
2006         """Information Extractor for Yahoo! Video search queries."""
2007         _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
2008         _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
2009         _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
2010         _MORE_PAGES_INDICATOR = r'\s*Next'
2011         _yahoo_ie = None
2012         _max_yahoo_results = 1000
2013
2014         def __init__(self, yahoo_ie, downloader=None):
2015                 InfoExtractor.__init__(self, downloader)
2016                 self._yahoo_ie = yahoo_ie
2017
2018         @staticmethod
2019         def suitable(url):
2020                 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
2021
2022         def report_download_page(self, query, pagenum):
2023                 """Report attempt to download playlist page with given number."""
2024                 query = query.decode(preferredencoding())
2025                 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
2026
2027         def _real_initialize(self):
2028                 self._yahoo_ie.initialize()
2029
2030         def _real_extract(self, query):
2031                 mobj = re.match(self._VALID_QUERY, query)
2032                 if mobj is None:
2033                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2034                         return
2035
2036                 prefix, query = query.split(':')
2037                 prefix = prefix[8:]
2038                 query  = query.encode('utf-8')
2039                 if prefix == '':
2040                         self._download_n_results(query, 1)
2041                         return
2042                 elif prefix == 'all':
2043                         self._download_n_results(query, self._max_yahoo_results)
2044                         return
2045                 else:
2046                         try:
2047                                 n = long(prefix)
2048                                 if n <= 0:
2049                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2050                                         return
2051                                 elif n > self._max_yahoo_results:
2052                                         self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)'  % (self._max_yahoo_results, n))
2053                                         n = self._max_yahoo_results
2054                                 self._download_n_results(query, n)
2055                                 return
2056                         except ValueError: # parsing prefix as integer fails
2057                                 self._download_n_results(query, 1)
2058                                 return
2059
2060         def _download_n_results(self, query, n):
2061                 """Downloads a specified number of results for a query"""
2062
2063                 video_ids = []
2064                 already_seen = set()
2065                 pagenum = 1
2066
2067                 while True:
2068                         self.report_download_page(query, pagenum)
2069                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2070                         request = urllib2.Request(result_url)
2071                         try:
2072                                 page = urllib2.urlopen(request).read()
2073                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2074                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2075                                 return
2076
2077                         # Extract video identifiers
2078                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2079                                 video_id = mobj.group(1)
2080                                 if video_id not in already_seen:
2081                                         video_ids.append(video_id)
2082                                         already_seen.add(video_id)
2083                                         if len(video_ids) == n:
2084                                                 # Specified n videos reached
2085                                                 for id in video_ids:
2086                                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
2087                                                 return
2088
2089                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2090                                 for id in video_ids:
2091                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
2092                                 return
2093
2094                         pagenum = pagenum + 1
2095
2096 class YoutubePlaylistIE(InfoExtractor):
2097         """Information Extractor for YouTube playlists."""
2098
2099         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/|p/)([^&]+).*'
2100         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
2101         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
2102         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
2103         _youtube_ie = None
2104
2105         def __init__(self, youtube_ie, downloader=None):
2106                 InfoExtractor.__init__(self, downloader)
2107                 self._youtube_ie = youtube_ie
2108
2109         @staticmethod
2110         def suitable(url):
2111                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
2112
2113         def report_download_page(self, playlist_id, pagenum):
2114                 """Report attempt to download playlist page with given number."""
2115                 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
2116
2117         def _real_initialize(self):
2118                 self._youtube_ie.initialize()
2119
2120         def _real_extract(self, url):
2121                 # Extract playlist id
2122                 mobj = re.match(self._VALID_URL, url)
2123                 if mobj is None:
2124                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2125                         return
2126
2127                 # Download playlist pages
2128                 playlist_id = mobj.group(1)
2129                 video_ids = []
2130                 pagenum = 1
2131
2132                 while True:
2133                         self.report_download_page(playlist_id, pagenum)
2134                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum))
2135                         try:
2136                                 page = urllib2.urlopen(request).read()
2137                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2138                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2139                                 return
2140
2141                         # Extract video identifiers
2142                         ids_in_page = []
2143                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2144                                 if mobj.group(1) not in ids_in_page:
2145                                         ids_in_page.append(mobj.group(1))
2146                         video_ids.extend(ids_in_page)
2147
2148                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2149                                 break
2150                         pagenum = pagenum + 1
2151
2152                 playliststart = self._downloader.params.get('playliststart', 1) - 1
2153                 playlistend = self._downloader.params.get('playlistend', -1)
2154                 video_ids = video_ids[playliststart:playlistend]
2155
2156                 for id in video_ids:
2157                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2158                 return
2159
2160 class YoutubeUserIE(InfoExtractor):
2161         """Information Extractor for YouTube users."""
2162
2163         _VALID_URL = r'(?:(?:(?:http://)?(?:\w+\.)?youtube.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
2164         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
2165         _GDATA_PAGE_SIZE = 50
2166         _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
2167         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
2168         _youtube_ie = None
2169
2170         def __init__(self, youtube_ie, downloader=None):
2171                 InfoExtractor.__init__(self, downloader)
2172                 self._youtube_ie = youtube_ie
2173
2174         @staticmethod
2175         def suitable(url):
2176                 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
2177
2178         def report_download_page(self, username, start_index):
2179                 """Report attempt to download user page."""
2180                 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
2181                                            (username, start_index, start_index + self._GDATA_PAGE_SIZE))
2182
2183         def _real_initialize(self):
2184                 self._youtube_ie.initialize()
2185
2186         def _real_extract(self, url):
2187                 # Extract username
2188                 mobj = re.match(self._VALID_URL, url)
2189                 if mobj is None:
2190                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2191                         return
2192
2193                 username = mobj.group(1)
2194
2195                 # Download video ids using YouTube Data API. Result size per
2196                 # query is limited (currently to 50 videos) so we need to query
2197                 # page by page until there are no video ids - it means we got
2198                 # all of them.
2199
2200                 video_ids = []
2201                 pagenum = 0
2202
2203                 while True:
2204                         start_index = pagenum * self._GDATA_PAGE_SIZE + 1
2205                         self.report_download_page(username, start_index)
2206
2207                         request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
2208
2209                         try:
2210                                 page = urllib2.urlopen(request).read()
2211                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2212                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2213                                 return
2214
2215                         # Extract video identifiers
2216                         ids_in_page = []
2217
2218                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2219                                 if mobj.group(1) not in ids_in_page:
2220                                         ids_in_page.append(mobj.group(1))
2221
2222                         video_ids.extend(ids_in_page)
2223
2224                         # A little optimization - if current page is not
2225                         # "full", ie. does not contain PAGE_SIZE video ids then
2226                         # we can assume that this page is the last one - there
2227                         # are no more ids on further pages - no need to query
2228                         # again.
2229
2230                         if len(ids_in_page) < self._GDATA_PAGE_SIZE:
2231                                 break
2232
2233                         pagenum += 1
2234
2235                 all_ids_count = len(video_ids)
2236                 playliststart = self._downloader.params.get('playliststart', 1) - 1
2237                 playlistend = self._downloader.params.get('playlistend', -1)
2238
2239                 if playlistend == -1:
2240                         video_ids = video_ids[playliststart:]
2241                 else:
2242                         video_ids = video_ids[playliststart:playlistend]
2243
2244                 self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" %
2245                                            (username, all_ids_count, len(video_ids)))
2246
2247                 for video_id in video_ids:
2248                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id)
2249
2250
2251 class DepositFilesIE(InfoExtractor):
2252         """Information extractor for depositfiles.com"""
2253
2254         _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)'
2255
2256         def __init__(self, downloader=None):
2257                 InfoExtractor.__init__(self, downloader)
2258
2259         @staticmethod
2260         def suitable(url):
2261                 return (re.match(DepositFilesIE._VALID_URL, url) is not None)
2262
2263         def report_download_webpage(self, file_id):
2264                 """Report webpage download."""
2265                 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
2266
2267         def report_extraction(self, file_id):
2268                 """Report information extraction."""
2269                 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
2270
2271         def _real_initialize(self):
2272                 return
2273
2274         def _real_extract(self, url):
2275                 # At this point we have a new file
2276                 self._downloader.increment_downloads()
2277
2278                 file_id = url.split('/')[-1]
2279                 # Rebuild url in english locale
2280                 url = 'http://depositfiles.com/en/files/' + file_id
2281
2282                 # Retrieve file webpage with 'Free download' button pressed
2283                 free_download_indication = { 'gateway_result' : '1' }
2284                 request = urllib2.Request(url, urllib.urlencode(free_download_indication))
2285                 try:
2286                         self.report_download_webpage(file_id)
2287                         webpage = urllib2.urlopen(request).read()
2288                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2289                         self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
2290                         return
2291
2292                 # Search for the real file URL
2293                 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
2294                 if (mobj is None) or (mobj.group(1) is None):
2295                         # Try to figure out reason of the error.
2296                         mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
2297                         if (mobj is not None) and (mobj.group(1) is not None):
2298                                 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
2299                                 self._downloader.trouble(u'ERROR: %s' % restriction_message)
2300                         else:
2301                                 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
2302                         return
2303
2304                 file_url = mobj.group(1)
2305                 file_extension = os.path.splitext(file_url)[1][1:]
2306
2307                 # Search for file title
2308                 mobj = re.search(r'<b title="(.*?)">', webpage)
2309                 if mobj is None:
2310                         self._downloader.trouble(u'ERROR: unable to extract title')
2311                         return
2312                 file_title = mobj.group(1).decode('utf-8')
2313
2314                 try:
2315                         # Process file information
2316                         self._downloader.process_info({
2317                                 'id':           file_id.decode('utf-8'),
2318                                 'url':          file_url.decode('utf-8'),
2319                                 'uploader':     u'NA',
2320                                 'upload_date':  u'NA',
2321                                 'title':        file_title,
2322                                 'stitle':       file_title,
2323                                 'ext':          file_extension.decode('utf-8'),
2324                                 'format':       u'NA',
2325                                 'player_url':   None,
2326                         })
2327                 except UnavailableVideoError, err:
2328                         self._downloader.trouble(u'ERROR: unable to download file')
2329
2330 class PostProcessor(object):
2331         """Post Processor class.
2332
2333         PostProcessor objects can be added to downloaders with their
2334         add_post_processor() method. When the downloader has finished a
2335         successful download, it will take its internal chain of PostProcessors
2336         and start calling the run() method on each one of them, first with
2337         an initial argument and then with the returned value of the previous
2338         PostProcessor.
2339
2340         The chain will be stopped if one of them ever returns None or the end
2341         of the chain is reached.
2342
2343         PostProcessor objects follow a "mutual registration" process similar
2344         to InfoExtractor objects.
2345         """
2346
2347         _downloader = None
2348
2349         def __init__(self, downloader=None):
2350                 self._downloader = downloader
2351
2352         def set_downloader(self, downloader):
2353                 """Sets the downloader for this PP."""
2354                 self._downloader = downloader
2355
2356         def run(self, information):
2357                 """Run the PostProcessor.
2358
2359                 The "information" argument is a dictionary like the ones
2360                 composed by InfoExtractors. The only difference is that this
2361                 one has an extra field called "filepath" that points to the
2362                 downloaded file.
2363
2364                 When this method returns None, the postprocessing chain is
2365                 stopped. However, this method may return an information
2366                 dictionary that will be passed to the next postprocessing
2367                 object in the chain. It can be the one it received after
2368                 changing some fields.
2369
2370                 In addition, this method may raise a PostProcessingError
2371                 exception that will be taken into account by the downloader
2372                 it was called from.
2373                 """
2374                 return information # by default, do nothing
2375
2376 ### MAIN PROGRAM ###
2377 if __name__ == '__main__':
2378         try:
2379                 # Modules needed only when running the main program
2380                 import getpass
2381                 import optparse
2382
2383                 # Function to update the program file with the latest version from the repository.
2384                 def update_self(downloader, filename):
2385                         # Note: downloader only used for options
2386                         if not os.access(filename, os.W_OK):
2387                                 sys.exit('ERROR: no write permissions on %s' % filename)
2388
2389                         downloader.to_screen('Updating to latest stable version...')
2390                         try:
2391                                 latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
2392                                 latest_version = urllib.urlopen(latest_url).read().strip()
2393                                 prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
2394                                 newcontent = urllib.urlopen(prog_url).read()
2395                         except (IOError, OSError), err:
2396                                 sys.exit('ERROR: unable to download latest version')
2397                         try:
2398                                 stream = open(filename, 'w')
2399                                 stream.write(newcontent)
2400                                 stream.close()
2401                         except (IOError, OSError), err:
2402                                 sys.exit('ERROR: unable to overwrite current version')
2403                         downloader.to_screen('Updated to version %s' % latest_version)
2404
2405                 # Parse command line
2406                 parser = optparse.OptionParser(
2407                         usage='Usage: %prog [options] url...',
2408                         version='2010.12.09',
2409                         conflict_handler='resolve',
2410                 )
2411
2412                 parser.add_option('-h', '--help',
2413                                 action='help', help='print this help text and exit')
2414                 parser.add_option('-v', '--version',
2415                                 action='version', help='print program version and exit')
2416                 parser.add_option('-U', '--update',
2417                                 action='store_true', dest='update_self', help='update this program to latest stable version')
2418                 parser.add_option('-i', '--ignore-errors',
2419                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
2420                 parser.add_option('-r', '--rate-limit',
2421                                 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
2422                 parser.add_option('-R', '--retries',
2423                                 dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
2424                 parser.add_option('--playlist-start',
2425                                 dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
2426                 parser.add_option('--playlist-end',
2427                                 dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
2428                 parser.add_option('--dump-user-agent',
2429                                 action='store_true', dest='dump_user_agent',
2430                                 help='display the current browser identification', default=False)
2431
2432                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
2433                 authentication.add_option('-u', '--username',
2434                                 dest='username', metavar='USERNAME', help='account username')
2435                 authentication.add_option('-p', '--password',
2436                                 dest='password', metavar='PASSWORD', help='account password')
2437                 authentication.add_option('-n', '--netrc',
2438                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
2439                 parser.add_option_group(authentication)
2440
2441                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
2442                 video_format.add_option('-f', '--format',
2443                                 action='store', dest='format', metavar='FORMAT', help='video format code')
2444                 video_format.add_option('--all-formats',
2445                                 action='store_const', dest='format', help='download all available video formats', const='-1')
2446                 video_format.add_option('--max-quality',
2447                                 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
2448                 parser.add_option_group(video_format)
2449
2450                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
2451                 verbosity.add_option('-q', '--quiet',
2452                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
2453                 verbosity.add_option('-s', '--simulate',
2454                                 action='store_true', dest='simulate', help='do not download video', default=False)
2455                 verbosity.add_option('-g', '--get-url',
2456                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
2457                 verbosity.add_option('-e', '--get-title',
2458                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
2459                 verbosity.add_option('--get-thumbnail',
2460                                 action='store_true', dest='getthumbnail',
2461                                 help='simulate, quiet but print thumbnail URL', default=False)
2462                 verbosity.add_option('--get-description',
2463                                 action='store_true', dest='getdescription',
2464                                 help='simulate, quiet but print video description', default=False)
2465                 verbosity.add_option('--get-filename',
2466                                 action='store_true', dest='getfilename',
2467                                 help='simulate, quiet but print output filename', default=False)
2468                 verbosity.add_option('--no-progress',
2469                                 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
2470                 verbosity.add_option('--console-title',
2471                                 action='store_true', dest='consoletitle',
2472                                 help='display progress in console titlebar', default=False)
2473                 parser.add_option_group(verbosity)
2474
2475                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
2476                 filesystem.add_option('-t', '--title',
2477                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
2478                 filesystem.add_option('-l', '--literal',
2479                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
2480                 filesystem.add_option('-A', '--auto-number',
2481                                 action='store_true', dest='autonumber',
2482                                 help='number downloaded files starting from 00000', default=False)
2483                 filesystem.add_option('-o', '--output',
2484                                 dest='outtmpl', metavar='TEMPLATE', help='output filename template')
2485                 filesystem.add_option('-a', '--batch-file',
2486                                 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
2487                 filesystem.add_option('-w', '--no-overwrites',
2488                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
2489                 filesystem.add_option('-c', '--continue',
2490                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
2491                 filesystem.add_option('--cookies',
2492                                 dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
2493                 filesystem.add_option('--no-part',
2494                                 action='store_true', dest='nopart', help='do not use .part files', default=False)
2495                 filesystem.add_option('--no-mtime',
2496                                 action='store_false', dest='updatetime',
2497                                 help='do not use the Last-modified header to set the file modification time', default=True)
2498                 parser.add_option_group(filesystem)
2499
2500                 (opts, args) = parser.parse_args()
2501
2502                 # Open appropriate CookieJar
2503                 if opts.cookiefile is None:
2504                         jar = cookielib.CookieJar()
2505                 else:
2506                         try:
2507                                 jar = cookielib.MozillaCookieJar(opts.cookiefile)
2508                                 if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
2509                                         jar.load()
2510                         except (IOError, OSError), err:
2511                                 sys.exit(u'ERROR: unable to open cookie file')
2512
2513                 # Dump user agent
2514                 if opts.dump_user_agent:
2515                         print std_headers['User-Agent']
2516                         sys.exit(0)
2517
2518                 # General configuration
2519                 cookie_processor = urllib2.HTTPCookieProcessor(jar)
2520                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler()))
2521                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
2522
2523                 # Batch file verification
2524                 batchurls = []
2525                 if opts.batchfile is not None:
2526                         try:
2527                                 if opts.batchfile == '-':
2528                                         batchfd = sys.stdin
2529                                 else:
2530                                         batchfd = open(opts.batchfile, 'r')
2531                                 batchurls = batchfd.readlines()
2532                                 batchurls = [x.strip() for x in batchurls]
2533                                 batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
2534                         except IOError:
2535                                 sys.exit(u'ERROR: batch file could not be read')
2536                 all_urls = batchurls + args
2537
2538                 # Conflicting, missing and erroneous options
2539                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
2540                         parser.error(u'using .netrc conflicts with giving username/password')
2541                 if opts.password is not None and opts.username is None:
2542                         parser.error(u'account username missing')
2543                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber):
2544                         parser.error(u'using output template conflicts with using title, literal title or auto number')
2545                 if opts.usetitle and opts.useliteral:
2546                         parser.error(u'using title conflicts with using literal title')
2547                 if opts.username is not None and opts.password is None:
2548                         opts.password = getpass.getpass(u'Type account password and press return:')
2549                 if opts.ratelimit is not None:
2550                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
2551                         if numeric_limit is None:
2552                                 parser.error(u'invalid rate limit specified')
2553                         opts.ratelimit = numeric_limit
2554                 if opts.retries is not None:
2555                         try:
2556                                 opts.retries = long(opts.retries)
2557                         except (TypeError, ValueError), err:
2558                                 parser.error(u'invalid retry count specified')
2559                 try:
2560                         opts.playliststart = long(opts.playliststart)
2561                         if opts.playliststart <= 0:
2562                                 raise ValueError
2563                 except (TypeError, ValueError), err:
2564                         parser.error(u'invalid playlist start number specified')
2565                 try:
2566                         opts.playlistend = long(opts.playlistend)
2567                         if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
2568                                 raise ValueError
2569                 except (TypeError, ValueError), err:
2570                         parser.error(u'invalid playlist end number specified')
2571
2572                 # Information extractors
2573                 youtube_ie = YoutubeIE()
2574                 metacafe_ie = MetacafeIE(youtube_ie)
2575                 dailymotion_ie = DailymotionIE()
2576                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
2577                 youtube_user_ie = YoutubeUserIE(youtube_ie)
2578                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
2579                 google_ie = GoogleIE()
2580                 google_search_ie = GoogleSearchIE(google_ie)
2581                 photobucket_ie = PhotobucketIE()
2582                 yahoo_ie = YahooIE()
2583                 yahoo_search_ie = YahooSearchIE(yahoo_ie)
2584                 deposit_files_ie = DepositFilesIE()
2585                 generic_ie = GenericIE()
2586
2587                 # File downloader
2588                 fd = FileDownloader({
2589                         'usenetrc': opts.usenetrc,
2590                         'username': opts.username,
2591                         'password': opts.password,
2592                         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
2593                         'forceurl': opts.geturl,
2594                         'forcetitle': opts.gettitle,
2595                         'forcethumbnail': opts.getthumbnail,
2596                         'forcedescription': opts.getdescription,
2597                         'forcefilename': opts.getfilename,
2598                         'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
2599                         'format': opts.format,
2600                         'format_limit': opts.format_limit,
2601                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
2602                                 or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
2603                                 or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
2604                                 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
2605                                 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
2606                                 or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
2607                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
2608                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
2609                                 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
2610                                 or u'%(id)s.%(ext)s'),
2611                         'ignoreerrors': opts.ignoreerrors,
2612                         'ratelimit': opts.ratelimit,
2613                         'nooverwrites': opts.nooverwrites,
2614                         'retries': opts.retries,
2615                         'continuedl': opts.continue_dl,
2616                         'noprogress': opts.noprogress,
2617                         'playliststart': opts.playliststart,
2618                         'playlistend': opts.playlistend,
2619                         'logtostderr': opts.outtmpl == '-',
2620                         'consoletitle': opts.consoletitle,
2621                         'nopart': opts.nopart,
2622                         'updatetime': opts.updatetime,
2623                         })
2624                 fd.add_info_extractor(youtube_search_ie)
2625                 fd.add_info_extractor(youtube_pl_ie)
2626                 fd.add_info_extractor(youtube_user_ie)
2627                 fd.add_info_extractor(metacafe_ie)
2628                 fd.add_info_extractor(dailymotion_ie)
2629                 fd.add_info_extractor(youtube_ie)
2630                 fd.add_info_extractor(google_ie)
2631                 fd.add_info_extractor(google_search_ie)
2632                 fd.add_info_extractor(photobucket_ie)
2633                 fd.add_info_extractor(yahoo_ie)
2634                 fd.add_info_extractor(yahoo_search_ie)
2635                 fd.add_info_extractor(deposit_files_ie)
2636
2637                 # This must come last since it's the
2638                 # fallback if none of the others work
2639                 fd.add_info_extractor(generic_ie)
2640
2641                 # Update version
2642                 if opts.update_self:
2643                         update_self(fd, sys.argv[0])
2644
2645                 # Maybe do nothing
2646                 if len(all_urls) < 1:
2647                         if not opts.update_self:
2648                                 parser.error(u'you must provide at least one URL')
2649                         else:
2650                                 sys.exit()
2651                 retcode = fd.download(all_urls)
2652
2653                 # Dump cookie jar if requested
2654                 if opts.cookiefile is not None:
2655                         try:
2656                                 jar.save()
2657                         except (IOError, OSError), err:
2658                                 sys.exit(u'ERROR: unable to save cookie jar')
2659
2660                 sys.exit(retcode)
2661
2662         except DownloadError:
2663                 sys.exit(1)
2664         except SameFileError:
2665                 sys.exit(u'ERROR: fixed output name but more than one file to download')
2666         except KeyboardInterrupt:
2667                 sys.exit(u'\nERROR: Interrupted by user')