youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __author__  = (
   5         'Ricardo Garcia Gonzalez',
   6         'Danny Colligan',
   7         'Benjamin Johnson',
   8         'Vasyl\' Vavrychuk',
   9         'Witold Baryluk',
  10         'Paweł Paprota',
  11         'Gergely Imreh',
  12         'Rogério Brito',
  13         'Philipp Hagemeister',
  14         'Sören Schulze',
  15         )
  16
  17 __license__ = 'Public Domain'
  18 __version__ = '2011.09.13'
  19
  20 UPDATE_URL = 'https://raw.github.com/phihag/youtube-dl/master/youtube-dl'
  21
  22 import cookielib
  23 import datetime
  24 import gzip
  25 import htmlentitydefs
  26 import httplib
  27 import locale
  28 import math
  29 import netrc
  30 import os
  31 import os.path
  32 import re
  33 import socket
  34 import string
  35 import subprocess
  36 import sys
  37 import time
  38 import urllib
  39 import urllib2
  40 import warnings
  41 import zlib
  42
  43 if os.name == 'nt':
  44         import ctypes
  45
  46 try:
  47         import email.utils
  48 except ImportError: # Python 2.4
  49         import email.Utils
  50 try:
  51         import cStringIO as StringIO
  52 except ImportError:
  53         import StringIO
  54
  55 # parse_qs was moved from the cgi module to the urlparse module recently.
  56 try:
  57         from urlparse import parse_qs
  58 except ImportError:
  59         from cgi import parse_qs
  60
  61 try:
  62         import lxml.etree
  63 except ImportError:
  64         pass # Handled below
  65
  66 try:
  67         import xml.etree.ElementTree
  68 except ImportError: # Python<2.5
  69         pass # Not officially supported, but let it slip
  70
  71 std_headers = {
  72         'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
  73         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  74         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  75         'Accept-Encoding': 'gzip, deflate',
  76         'Accept-Language': 'en-us,en;q=0.5',
  77 }
  78
  79 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  80
  81 try:
  82         import json
  83 except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
  84         import re
  85         class json(object):
  86                 @staticmethod
  87                 def loads(s):
  88                         s = s.decode('UTF-8')
  89                         def raiseError(msg, i):
  90                                 raise ValueError(msg + ' at position ' + str(i) + ' of ' + repr(s) + ': ' + repr(s[i:]))
  91                         def skipSpace(i, expectMore=True):
  92                                 while i < len(s) and s[i] in ' \t\r\n':
  93                                         i += 1
  94                                 if expectMore:
  95                                         if i >= len(s):
  96                                                 raiseError('Premature end', i)
  97                                 return i
  98                         def decodeEscape(match):
  99                                 esc = match.group(1)
 100                                 _STATIC = {
 101                                         '"': '"',
 102                                         '\\': '\\',
 103                                         '/': '/',
 104                                         'b': unichr(0x8),
 105                                         'f': unichr(0xc),
 106                                         'n': '\n',
 107                                         'r': '\r',
 108                                         't': '\t',
 109                                 }
 110                                 if esc in _STATIC:
 111                                         return _STATIC[esc]
 112                                 if esc[0] == 'u':
 113                                         if len(esc) == 1+4:
 114                                                 return unichr(int(esc[1:5], 16))
 115                                         if len(esc) == 5+6 and esc[5:7] == '\\u':
 116                                                 hi = int(esc[1:5], 16)
 117                                                 low = int(esc[7:11], 16)
 118                                                 return unichr((hi - 0xd800) * 0x400 + low - 0xdc00 + 0x10000)
 119                                 raise ValueError('Unknown escape ' + str(esc))
 120                         def parseString(i):
 121                                 i += 1
 122                                 e = i
 123                                 while True:
 124                                         e = s.index('"', e)
 125                                         bslashes = 0
 126                                         while s[e-bslashes-1] == '\\':
 127                                                 bslashes += 1
 128                                         if bslashes % 2 == 1:
 129                                                 e += 1
 130                                                 continue
 131                                         break
 132                                 rexp = re.compile(r'\\(u[dD][89aAbB][0-9a-fA-F]{2}\\u[0-9a-fA-F]{4}|u[0-9a-fA-F]{4}|.|$)')
 133                                 stri = rexp.sub(decodeEscape, s[i:e])
 134                                 return (e+1,stri)
 135                         def parseObj(i):
 136                                 i += 1
 137                                 res = {}
 138                                 i = skipSpace(i)
 139                                 if s[i] == '}': # Empty dictionary
 140                                         return (i+1,res)
 141                                 while True:
 142                                         if s[i] != '"':
 143                                                 raiseError('Expected a string object key', i)
 144                                         i,key = parseString(i)
 145                                         i = skipSpace(i)
 146                                         if i >= len(s) or s[i] != ':':
 147                                                 raiseError('Expected a colon', i)
 148                                         i,val = parse(i+1)
 149                                         res[key] = val
 150                                         i = skipSpace(i)
 151                                         if s[i] == '}':
 152                                                 return (i+1, res)
 153                                         if s[i] != ',':
 154                                                 raiseError('Expected comma or closing curly brace', i)
 155                                         i = skipSpace(i+1)
 156                         def parseArray(i):
 157                                 res = []
 158                                 i = skipSpace(i+1)
 159                                 if s[i] == ']': # Empty array
 160                                         return (i+1,res)
 161                                 while True:
 162                                         i,val = parse(i)
 163                                         res.append(val)
 164                                         i = skipSpace(i) # Raise exception if premature end
 165                                         if s[i] == ']':
 166                                                 return (i+1, res)
 167                                         if s[i] != ',':
 168                                                 raiseError('Expected a comma or closing bracket', i)
 169                                         i = skipSpace(i+1)
 170                         def parseDiscrete(i):
 171                                 for k,v in {'true': True, 'false': False, 'null': None}.items():
 172                                         if s.startswith(k, i):
 173                                                 return (i+len(k), v)
 174                                 raiseError('Not a boolean (or null)', i)
 175                         def parseNumber(i):
 176                                 mobj = re.match('^(-?(0|[1-9][0-9]*)(\.[0-9]*)?([eE][+-]?[0-9]+)?)', s[i:])
 177                                 if mobj is None:
 178                                         raiseError('Not a number', i)
 179                                 nums = mobj.group(1)
 180                                 if '.' in nums or 'e' in nums or 'E' in nums:
 181                                         return (i+len(nums), float(nums))
 182                                 return (i+len(nums), int(nums))
 183                         CHARMAP = {'{': parseObj, '[': parseArray, '"': parseString, 't': parseDiscrete, 'f': parseDiscrete, 'n': parseDiscrete}
 184                         def parse(i):
 185                                 i = skipSpace(i)
 186                                 i,res = CHARMAP.get(s[i], parseNumber)(i)
 187                                 i = skipSpace(i, False)
 188                                 return (i,res)
 189                         i,res = parse(0)
 190                         if i < len(s):
 191                                 raise ValueError('Extra data at end of input (index ' + str(i) + ' of ' + repr(s) + ': ' + repr(s[i:]) + ')')
 192                         return res
 193
 194 def preferredencoding():
 195         """Get preferred encoding.
 196
 197         Returns the best encoding scheme for the system, based on
 198         locale.getpreferredencoding() and some further tweaks.
 199         """
 200         def yield_preferredencoding():
 201                 try:
 202                         pref = locale.getpreferredencoding()
 203                         u'TEST'.encode(pref)
 204                 except:
 205                         pref = 'UTF-8'
 206                 while True:
 207                         yield pref
 208         return yield_preferredencoding().next()
 209
 210
 211 def htmlentity_transform(matchobj):
 212         """Transforms an HTML entity to a Unicode character.
 213
 214         This function receives a match object and is intended to be used with
 215         the re.sub() function.
 216         """
 217         entity = matchobj.group(1)
 218
 219         # Known non-numeric HTML entity
 220         if entity in htmlentitydefs.name2codepoint:
 221                 return unichr(htmlentitydefs.name2codepoint[entity])
 222
 223         # Unicode character
 224         mobj = re.match(ur'(?u)#(x?\d+)', entity)
 225         if mobj is not None:
 226                 numstr = mobj.group(1)
 227                 if numstr.startswith(u'x'):
 228                         base = 16
 229                         numstr = u'0%s' % numstr
 230                 else:
 231                         base = 10
 232                 return unichr(long(numstr, base))
 233
 234         # Unknown entity in name, return its literal representation
 235         return (u'&%s;' % entity)
 236
 237
 238 def sanitize_title(utitle):
 239         """Sanitizes a video title so it could be used as part of a filename."""
 240         utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
 241         return utitle.replace(unicode(os.sep), u'%')
 242
 243
 244 def sanitize_open(filename, open_mode):
 245         """Try to open the given filename, and slightly tweak it if this fails.
 246
 247         Attempts to open the given filename. If this fails, it tries to change
 248         the filename slightly, step by step, until it's either able to open it
 249         or it fails and raises a final exception, like the standard open()
 250         function.
 251
 252         It returns the tuple (stream, definitive_file_name).
 253         """
 254         try:
 255                 if filename == u'-':
 256                         if sys.platform == 'win32':
 257                                 import msvcrt
 258                                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 259                         return (sys.stdout, filename)
 260                 stream = open(filename, open_mode)
 261                 return (stream, filename)
 262         except (IOError, OSError), err:
 263                 # In case of error, try to remove win32 forbidden chars
 264                 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 265
 266                 # An exception here should be caught in the caller
 267                 stream = open(filename, open_mode)
 268                 return (stream, filename)
 269
 270
 271 def timeconvert(timestr):
 272         """Convert RFC 2822 defined time string into system timestamp"""
 273         timestamp = None
 274         timetuple = email.utils.parsedate_tz(timestr)
 275         if timetuple is not None:
 276                 timestamp = email.utils.mktime_tz(timetuple)
 277         return timestamp
 278
 279
 280 class DownloadError(Exception):
 281         """Download Error exception.
 282
 283         This exception may be thrown by FileDownloader objects if they are not
 284         configured to continue on errors. They will contain the appropriate
 285         error message.
 286         """
 287         pass
 288
 289
 290 class SameFileError(Exception):
 291         """Same File exception.
 292
 293         This exception will be thrown by FileDownloader objects if they detect
 294         multiple files would have to be downloaded to the same file on disk.
 295         """
 296         pass
 297
 298
 299 class PostProcessingError(Exception):
 300         """Post Processing exception.
 301
 302         This exception may be raised by PostProcessor's .run() method to
 303         indicate an error in the postprocessing task.
 304         """
 305         pass
 306
 307
 308 class UnavailableVideoError(Exception):
 309         """Unavailable Format exception.
 310
 311         This exception will be thrown when a video is requested
 312         in a format that is not available for that video.
 313         """
 314         pass
 315
 316
 317 class ContentTooShortError(Exception):
 318         """Content Too Short exception.
 319
 320         This exception may be raised by FileDownloader objects when a file they
 321         download is too small for what the server announced first, indicating
 322         the connection was probably interrupted.
 323         """
 324         # Both in bytes
 325         downloaded = None
 326         expected = None
 327
 328         def __init__(self, downloaded, expected):
 329                 self.downloaded = downloaded
 330                 self.expected = expected
 331
 332
 333 class YoutubeDLHandler(urllib2.HTTPHandler):
 334         """Handler for HTTP requests and responses.
 335
 336         This class, when installed with an OpenerDirector, automatically adds
 337         the standard headers to every HTTP request and handles gzipped and
 338         deflated responses from web servers. If compression is to be avoided in
 339         a particular request, the original request in the program code only has
 340         to include the HTTP header "Youtubedl-No-Compression", which will be
 341         removed before making the real request.
 342
 343         Part of this code was copied from:
 344
 345         http://techknack.net/python-urllib2-handlers/
 346
 347         Andrew Rowls, the author of that code, agreed to release it to the
 348         public domain.
 349         """
 350
 351         @staticmethod
 352         def deflate(data):
 353                 try:
 354                         return zlib.decompress(data, -zlib.MAX_WBITS)
 355                 except zlib.error:
 356                         return zlib.decompress(data)
 357
 358         @staticmethod
 359         def addinfourl_wrapper(stream, headers, url, code):
 360                 if hasattr(urllib2.addinfourl, 'getcode'):
 361                         return urllib2.addinfourl(stream, headers, url, code)
 362                 ret = urllib2.addinfourl(stream, headers, url)
 363                 ret.code = code
 364                 return ret
 365
 366         def http_request(self, req):
 367                 for h in std_headers:
 368                         if h in req.headers:
 369                                 del req.headers[h]
 370                         req.add_header(h, std_headers[h])
 371                 if 'Youtubedl-no-compression' in req.headers:
 372                         if 'Accept-encoding' in req.headers:
 373                                 del req.headers['Accept-encoding']
 374                         del req.headers['Youtubedl-no-compression']
 375                 return req
 376
 377         def http_response(self, req, resp):
 378                 old_resp = resp
 379                 # gzip
 380                 if resp.headers.get('Content-encoding', '') == 'gzip':
 381                         gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
 382                         resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 383                         resp.msg = old_resp.msg
 384                 # deflate
 385                 if resp.headers.get('Content-encoding', '') == 'deflate':
 386                         gz = StringIO.StringIO(self.deflate(resp.read()))
 387                         resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 388                         resp.msg = old_resp.msg
 389                 return resp
 390
 391
 392 class FileDownloader(object):
 393         """File Downloader class.
 394
 395         File downloader objects are the ones responsible of downloading the
 396         actual video file and writing it to disk if the user has requested
 397         it, among some other tasks. In most cases there should be one per
 398         program. As, given a video URL, the downloader doesn't know how to
 399         extract all the needed information, task that InfoExtractors do, it
 400         has to pass the URL to one of them.
 401
 402         For this, file downloader objects have a method that allows
 403         InfoExtractors to be registered in a given order. When it is passed
 404         a URL, the file downloader handles it to the first InfoExtractor it
 405         finds that reports being able to handle it. The InfoExtractor extracts
 406         all the information about the video or videos the URL refers to, and
 407         asks the FileDownloader to process the video information, possibly
 408         downloading the video.
 409
 410         File downloaders accept a lot of parameters. In order not to saturate
 411         the object constructor with arguments, it receives a dictionary of
 412         options instead. These options are available through the params
 413         attribute for the InfoExtractors to use. The FileDownloader also
 414         registers itself as the downloader in charge for the InfoExtractors
 415         that are added to it, so this is a "mutual registration".
 416
 417         Available options:
 418
 419         username:         Username for authentication purposes.
 420         password:         Password for authentication purposes.
 421         usenetrc:         Use netrc for authentication instead.
 422         quiet:            Do not print messages to stdout.
 423         forceurl:         Force printing final URL.
 424         forcetitle:       Force printing title.
 425         forcethumbnail:   Force printing thumbnail URL.
 426         forcedescription: Force printing description.
 427         forcefilename:    Force printing final filename.
 428         simulate:         Do not download the video files.
 429         format:           Video format code.
 430         format_limit:     Highest quality format to try.
 431         outtmpl:          Template for output names.
 432         ignoreerrors:     Do not stop on download errors.
 433         ratelimit:        Download speed limit, in bytes/sec.
 434         nooverwrites:     Prevent overwriting files.
 435         retries:          Number of times to retry for HTTP error 5xx
 436         continuedl:       Try to continue downloads if possible.
 437         noprogress:       Do not print the progress bar.
 438         playliststart:    Playlist item to start at.
 439         playlistend:      Playlist item to end at.
 440         logtostderr:      Log messages to stderr instead of stdout.
 441         consoletitle:     Display progress in console window's titlebar.
 442         nopart:           Do not use temporary .part files.
 443         updatetime:       Use the Last-modified header to set output file timestamps.
 444         writedescription: Write the video description to a .description file
 445         writeinfojson:    Write the video description to a .info.json file
 446         """
 447
 448         params = None
 449         _ies = []
 450         _pps = []
 451         _download_retcode = None
 452         _num_downloads = None
 453         _screen_file = None
 454
 455         def __init__(self, params):
 456                 """Create a FileDownloader object with the given options."""
 457                 self._ies = []
 458                 self._pps = []
 459                 self._download_retcode = 0
 460                 self._num_downloads = 0
 461                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 462                 self.params = params
 463
 464         @staticmethod
 465         def format_bytes(bytes):
 466                 if bytes is None:
 467                         return 'N/A'
 468                 if type(bytes) is str:
 469                         bytes = float(bytes)
 470                 if bytes == 0.0:
 471                         exponent = 0
 472                 else:
 473                         exponent = long(math.log(bytes, 1024.0))
 474                 suffix = 'bkMGTPEZY'[exponent]
 475                 converted = float(bytes) / float(1024 ** exponent)
 476                 return '%.2f%s' % (converted, suffix)
 477
 478         @staticmethod
 479         def calc_percent(byte_counter, data_len):
 480                 if data_len is None:
 481                         return '---.-%'
 482                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 483
 484         @staticmethod
 485         def calc_eta(start, now, total, current):
 486                 if total is None:
 487                         return '--:--'
 488                 dif = now - start
 489                 if current == 0 or dif < 0.001: # One millisecond
 490                         return '--:--'
 491                 rate = float(current) / dif
 492                 eta = long((float(total) - float(current)) / rate)
 493                 (eta_mins, eta_secs) = divmod(eta, 60)
 494                 if eta_mins > 99:
 495                         return '--:--'
 496                 return '%02d:%02d' % (eta_mins, eta_secs)
 497
 498         @staticmethod
 499         def calc_speed(start, now, bytes):
 500                 dif = now - start
 501                 if bytes == 0 or dif < 0.001: # One millisecond
 502                         return '%10s' % '---b/s'
 503                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 504
 505         @staticmethod
 506         def best_block_size(elapsed_time, bytes):
 507                 new_min = max(bytes / 2.0, 1.0)
 508                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 509                 if elapsed_time < 0.001:
 510                         return long(new_max)
 511                 rate = bytes / elapsed_time
 512                 if rate > new_max:
 513                         return long(new_max)
 514                 if rate < new_min:
 515                         return long(new_min)
 516                 return long(rate)
 517
 518         @staticmethod
 519         def parse_bytes(bytestr):
 520                 """Parse a string indicating a byte quantity into a long integer."""
 521                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 522                 if matchobj is None:
 523                         return None
 524                 number = float(matchobj.group(1))
 525                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 526                 return long(round(number * multiplier))
 527
 528         def add_info_extractor(self, ie):
 529                 """Add an InfoExtractor object to the end of the list."""
 530                 self._ies.append(ie)
 531                 ie.set_downloader(self)
 532
 533         def add_post_processor(self, pp):
 534                 """Add a PostProcessor object to the end of the chain."""
 535                 self._pps.append(pp)
 536                 pp.set_downloader(self)
 537
 538         def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
 539                 """Print message to stdout if not in quiet mode."""
 540                 try:
 541                         if not self.params.get('quiet', False):
 542                                 terminator = [u'\n', u''][skip_eol]
 543                                 print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
 544                         self._screen_file.flush()
 545                 except (UnicodeEncodeError), err:
 546                         if not ignore_encoding_errors:
 547                                 raise
 548
 549         def to_stderr(self, message):
 550                 """Print message to stderr."""
 551                 print >>sys.stderr, message.encode(preferredencoding())
 552
 553         def to_cons_title(self, message):
 554                 """Set console/terminal window title to message."""
 555                 if not self.params.get('consoletitle', False):
 556                         return
 557                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 558                         # c_wchar_p() might not be necessary if `message` is
 559                         # already of type unicode()
 560                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 561                 elif 'TERM' in os.environ:
 562                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 563
 564         def fixed_template(self):
 565                 """Checks if the output template is fixed."""
 566                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 567
 568         def trouble(self, message=None):
 569                 """Determine action to take when a download problem appears.
 570
 571                 Depending on if the downloader has been configured to ignore
 572                 download errors or not, this method may throw an exception or
 573                 not when errors are found, after printing the message.
 574                 """
 575                 if message is not None:
 576                         self.to_stderr(message)
 577                 if not self.params.get('ignoreerrors', False):
 578                         raise DownloadError(message)
 579                 self._download_retcode = 1
 580
 581         def slow_down(self, start_time, byte_counter):
 582                 """Sleep if the download speed is over the rate limit."""
 583                 rate_limit = self.params.get('ratelimit', None)
 584                 if rate_limit is None or byte_counter == 0:
 585                         return
 586                 now = time.time()
 587                 elapsed = now - start_time
 588                 if elapsed <= 0.0:
 589                         return
 590                 speed = float(byte_counter) / elapsed
 591                 if speed > rate_limit:
 592                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 593
 594         def temp_name(self, filename):
 595                 """Returns a temporary filename for the given filename."""
 596                 if self.params.get('nopart', False) or filename == u'-' or \
 597                                 (os.path.exists(filename) and not os.path.isfile(filename)):
 598                         return filename
 599                 return filename + u'.part'
 600
 601         def undo_temp_name(self, filename):
 602                 if filename.endswith(u'.part'):
 603                         return filename[:-len(u'.part')]
 604                 return filename
 605
 606         def try_rename(self, old_filename, new_filename):
 607                 try:
 608                         if old_filename == new_filename:
 609                                 return
 610                         os.rename(old_filename, new_filename)
 611                 except (IOError, OSError), err:
 612                         self.trouble(u'ERROR: unable to rename file')
 613
 614         def try_utime(self, filename, last_modified_hdr):
 615                 """Try to set the last-modified time of the given file."""
 616                 if last_modified_hdr is None:
 617                         return
 618                 if not os.path.isfile(filename):
 619                         return
 620                 timestr = last_modified_hdr
 621                 if timestr is None:
 622                         return
 623                 filetime = timeconvert(timestr)
 624                 if filetime is None:
 625                         return
 626                 try:
 627                         os.utime(filename, (time.time(), filetime))
 628                 except:
 629                         pass
 630
 631         def report_writedescription(self, descfn):
 632                 """ Report that the description file is being written """
 633                 self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True)
 634
 635         def report_writeinfojson(self, infofn):
 636                 """ Report that the metadata file has been written """
 637                 self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True)
 638
 639         def report_destination(self, filename):
 640                 """Report destination filename."""
 641                 self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
 642
 643         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 644                 """Report download progress."""
 645                 if self.params.get('noprogress', False):
 646                         return
 647                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 648                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 649                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 650                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 651
 652         def report_resuming_byte(self, resume_len):
 653                 """Report attempt to resume at given byte."""
 654                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 655
 656         def report_retry(self, count, retries):
 657                 """Report retry in case of HTTP error 5xx"""
 658                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 659
 660         def report_file_already_downloaded(self, file_name):
 661                 """Report file has already been fully downloaded."""
 662                 try:
 663                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 664                 except (UnicodeEncodeError), err:
 665                         self.to_screen(u'[download] The file has already been downloaded')
 666
 667         def report_unable_to_resume(self):
 668                 """Report it was impossible to resume download."""
 669                 self.to_screen(u'[download] Unable to resume')
 670
 671         def report_finish(self):
 672                 """Report download finished."""
 673                 if self.params.get('noprogress', False):
 674                         self.to_screen(u'[download] Download completed')
 675                 else:
 676                         self.to_screen(u'')
 677
 678         def increment_downloads(self):
 679                 """Increment the ordinal that assigns a number to each file."""
 680                 self._num_downloads += 1
 681
 682         def prepare_filename(self, info_dict):
 683                 """Generate the output filename."""
 684                 try:
 685                         template_dict = dict(info_dict)
 686                         template_dict['epoch'] = unicode(long(time.time()))
 687                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 688                         filename = self.params['outtmpl'] % template_dict
 689                         return filename
 690                 except (ValueError, KeyError), err:
 691                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 692                         return None
 693
 694         def process_info(self, info_dict):
 695                 """Process a single dictionary returned by an InfoExtractor."""
 696                 filename = self.prepare_filename(info_dict)
 697                 # Do nothing else if in simulate mode
 698                 if self.params.get('simulate', False):
 699                         # Forced printings
 700                         if self.params.get('forcetitle', False):
 701                                 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
 702                         if self.params.get('forceurl', False):
 703                                 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
 704                         if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 705                                 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
 706                         if self.params.get('forcedescription', False) and 'description' in info_dict:
 707                                 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
 708                         if self.params.get('forcefilename', False) and filename is not None:
 709                                 print filename.encode(preferredencoding(), 'xmlcharrefreplace')
 710
 711                         return
 712
 713                 if filename is None:
 714                         return
 715                 if self.params.get('nooverwrites', False) and os.path.exists(filename):
 716                         self.to_stderr(u'WARNING: file exists and will be skipped')
 717                         return
 718
 719                 try:
 720                         dn = os.path.dirname(filename)
 721                         if dn != '' and not os.path.exists(dn):
 722                                 os.makedirs(dn)
 723                 except (OSError, IOError), err:
 724                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 725                         return
 726
 727                 if self.params.get('writedescription', False):
 728                         try:
 729                                 descfn = filename + '.description'
 730                                 self.report_writedescription(descfn)
 731                                 descfile = open(descfn, 'wb')
 732                                 try:
 733                                         descfile.write(info_dict['description'].encode('utf-8'))
 734                                 finally:
 735                                         descfile.close()
 736                         except (OSError, IOError):
 737                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 738                                 return
 739
 740                 if self.params.get('writeinfojson', False):
 741                         infofn = filename + '.info.json'
 742                         self.report_writeinfojson(infofn)
 743                         try:
 744                                 json.dump
 745                         except (NameError,AttributeError):
 746                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 747                                 return
 748                         try:
 749                                 infof = open(infofn, 'wb')
 750                                 try:
 751                                         json.dump(info_dict, infof)
 752                                 finally:
 753                                         infof.close()
 754                         except (OSError, IOError):
 755                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 756                                 return
 757
 758                 try:
 759                         success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
 760                 except (OSError, IOError), err:
 761                         raise UnavailableVideoError
 762                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 763                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 764                         return
 765                 except (ContentTooShortError, ), err:
 766                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 767                         return
 768
 769                 if success:
 770                         try:
 771                                 self.post_process(filename, info_dict)
 772                         except (PostProcessingError), err:
 773                                 self.trouble(u'ERROR: postprocessing: %s' % str(err))
 774                                 return
 775
 776         def download(self, url_list):
 777                 """Download a given list of URLs."""
 778                 if len(url_list) > 1 and self.fixed_template():
 779                         raise SameFileError(self.params['outtmpl'])
 780
 781                 for url in url_list:
 782                         suitable_found = False
 783                         for ie in self._ies:
 784                                 # Go to next InfoExtractor if not suitable
 785                                 if not ie.suitable(url):
 786                                         continue
 787
 788                                 # Suitable InfoExtractor found
 789                                 suitable_found = True
 790
 791                                 # Extract information from URL and process it
 792                                 ie.extract(url)
 793
 794                                 # Suitable InfoExtractor had been found; go to next URL
 795                                 break
 796
 797                         if not suitable_found:
 798                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 799
 800                 return self._download_retcode
 801
 802         def post_process(self, filename, ie_info):
 803                 """Run the postprocessing chain on the given file."""
 804                 info = dict(ie_info)
 805                 info['filepath'] = filename
 806                 for pp in self._pps:
 807                         info = pp.run(info)
 808                         if info is None:
 809                                 break
 810
 811         def _download_with_rtmpdump(self, filename, url, player_url):
 812                 self.report_destination(filename)
 813                 tmpfilename = self.temp_name(filename)
 814
 815                 # Check for rtmpdump first
 816                 try:
 817                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 818                 except (OSError, IOError):
 819                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 820                         return False
 821
 822                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 823                 # the connection was interrumpted and resuming appears to be
 824                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 825                 basic_args = ['rtmpdump'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 826                 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
 827                 while retval == 2 or retval == 1:
 828                         prevsize = os.path.getsize(tmpfilename)
 829                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 830                         time.sleep(5.0) # This seems to be needed
 831                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 832                         cursize = os.path.getsize(tmpfilename)
 833                         if prevsize == cursize and retval == 1:
 834                                 break
 835                 if retval == 0:
 836                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
 837                         self.try_rename(tmpfilename, filename)
 838                         return True
 839                 else:
 840                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 841                         return False
 842
 843         def _do_download(self, filename, url, player_url):
 844                 # Check file already present
 845                 if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
 846                         self.report_file_already_downloaded(filename)
 847                         return True
 848
 849                 # Attempt to download using rtmpdump
 850                 if url.startswith('rtmp'):
 851                         return self._download_with_rtmpdump(filename, url, player_url)
 852
 853                 tmpfilename = self.temp_name(filename)
 854                 stream = None
 855                 open_mode = 'wb'
 856
 857                 # Do not include the Accept-Encoding header
 858                 headers = {'Youtubedl-no-compression': 'True'}
 859                 basic_request = urllib2.Request(url, None, headers)
 860                 request = urllib2.Request(url, None, headers)
 861
 862                 # Establish possible resume length
 863                 if os.path.isfile(tmpfilename):
 864                         resume_len = os.path.getsize(tmpfilename)
 865                 else:
 866                         resume_len = 0
 867
 868                 # Request parameters in case of being able to resume
 869                 if self.params.get('continuedl', False) and resume_len != 0:
 870                         self.report_resuming_byte(resume_len)
 871                         request.add_header('Range', 'bytes=%d-' % resume_len)
 872                         open_mode = 'ab'
 873
 874                 count = 0
 875                 retries = self.params.get('retries', 0)
 876                 while count <= retries:
 877                         # Establish connection
 878                         try:
 879                                 data = urllib2.urlopen(request)
 880                                 break
 881                         except (urllib2.HTTPError, ), err:
 882                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 883                                         # Unexpected HTTP error
 884                                         raise
 885                                 elif err.code == 416:
 886                                         # Unable to resume (requested range not satisfiable)
 887                                         try:
 888                                                 # Open the connection again without the range header
 889                                                 data = urllib2.urlopen(basic_request)
 890                                                 content_length = data.info()['Content-Length']
 891                                         except (urllib2.HTTPError, ), err:
 892                                                 if err.code < 500 or err.code >= 600:
 893                                                         raise
 894                                         else:
 895                                                 # Examine the reported length
 896                                                 if (content_length is not None and
 897                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 898                                                         # The file had already been fully downloaded.
 899                                                         # Explanation to the above condition: in issue #175 it was revealed that
 900                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 901                                                         # changing the file size slightly and causing problems for some users. So
 902                                                         # I decided to implement a suggested change and consider the file
 903                                                         # completely downloaded if the file size differs less than 100 bytes from
 904                                                         # the one in the hard drive.
 905                                                         self.report_file_already_downloaded(filename)
 906                                                         self.try_rename(tmpfilename, filename)
 907                                                         return True
 908                                                 else:
 909                                                         # The length does not match, we start the download over
 910                                                         self.report_unable_to_resume()
 911                                                         open_mode = 'wb'
 912                                                         break
 913                         # Retry
 914                         count += 1
 915                         if count <= retries:
 916                                 self.report_retry(count, retries)
 917
 918                 if count > retries:
 919                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 920                         return False
 921
 922                 data_len = data.info().get('Content-length', None)
 923                 if data_len is not None:
 924                         data_len = long(data_len) + resume_len
 925                 data_len_str = self.format_bytes(data_len)
 926                 byte_counter = 0 + resume_len
 927                 block_size = 1024
 928                 start = time.time()
 929                 while True:
 930                         # Download and write
 931                         before = time.time()
 932                         data_block = data.read(block_size)
 933                         after = time.time()
 934                         if len(data_block) == 0:
 935                                 break
 936                         byte_counter += len(data_block)
 937
 938                         # Open file just in time
 939                         if stream is None:
 940                                 try:
 941                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 942                                         assert stream is not None
 943                                         filename = self.undo_temp_name(tmpfilename)
 944                                         self.report_destination(filename)
 945                                 except (OSError, IOError), err:
 946                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 947                                         return False
 948                         try:
 949                                 stream.write(data_block)
 950                         except (IOError, OSError), err:
 951                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 952                                 return False
 953                         block_size = self.best_block_size(after - before, len(data_block))
 954
 955                         # Progress message
 956                         percent_str = self.calc_percent(byte_counter, data_len)
 957                         eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 958                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 959                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 960
 961                         # Apply rate limit
 962                         self.slow_down(start, byte_counter - resume_len)
 963
 964                 if stream is None:
 965                         self.trouble(u'\nERROR: Did not get any data blocks')
 966                         return False
 967                 stream.close()
 968                 self.report_finish()
 969                 if data_len is not None and byte_counter != data_len:
 970                         raise ContentTooShortError(byte_counter, long(data_len))
 971                 self.try_rename(tmpfilename, filename)
 972
 973                 # Update file modification time
 974                 if self.params.get('updatetime', True):
 975                         self.try_utime(filename, data.info().get('last-modified', None))
 976
 977                 return True
 978
 979
 980 class InfoExtractor(object):
 981         """Information Extractor class.
 982
 983         Information extractors are the classes that, given a URL, extract
 984         information from the video (or videos) the URL refers to. This
 985         information includes the real video URL, the video title and simplified
 986         title, author and others. The information is stored in a dictionary
 987         which is then passed to the FileDownloader. The FileDownloader
 988         processes this information possibly downloading the video to the file
 989         system, among other possible outcomes. The dictionaries must include
 990         the following fields:
 991
 992         id:             Video identifier.
 993         url:            Final video URL.
 994         uploader:       Nickname of the video uploader.
 995         title:          Literal title.
 996         stitle:         Simplified title.
 997         ext:            Video filename extension.
 998         format:         Video format.
 999         player_url:     SWF Player URL (may be None).
1000
1001         The following fields are optional. Their primary purpose is to allow
1002         youtube-dl to serve as the backend for a video search function, such
1003         as the one in youtube2mp3.  They are only used when their respective
1004         forced printing functions are called:
1005
1006         thumbnail:      Full URL to a video thumbnail image.
1007         description:    One-line video description.
1008
1009         Subclasses of this one should re-define the _real_initialize() and
1010         _real_extract() methods, as well as the suitable() static method.
1011         Probably, they should also be instantiated and added to the main
1012         downloader.
1013         """
1014
1015         _ready = False
1016         _downloader = None
1017
1018         def __init__(self, downloader=None):
1019                 """Constructor. Receives an optional downloader."""
1020                 self._ready = False
1021                 self.set_downloader(downloader)
1022
1023         @staticmethod
1024         def suitable(url):
1025                 """Receives a URL and returns True if suitable for this IE."""
1026                 return False
1027
1028         def initialize(self):
1029                 """Initializes an instance (authentication, etc)."""
1030                 if not self._ready:
1031                         self._real_initialize()
1032                         self._ready = True
1033
1034         def extract(self, url):
1035                 """Extracts URL information and returns it in list of dicts."""
1036                 self.initialize()
1037                 return self._real_extract(url)
1038
1039         def set_downloader(self, downloader):
1040                 """Sets the downloader for this IE."""
1041                 self._downloader = downloader
1042
1043         def _real_initialize(self):
1044                 """Real initialization process. Redefine in subclasses."""
1045                 pass
1046
1047         def _real_extract(self, url):
1048                 """Real extraction process. Redefine in subclasses."""
1049                 pass
1050
1051
1052 class YoutubeIE(InfoExtractor):
1053         """Information extractor for youtube.com."""
1054
1055         _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
1056         _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
1057         _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
1058         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
1059         _NETRC_MACHINE = 'youtube'
1060         # Listed in order of quality
1061         _available_formats = ['38', '37', '45', '22', '43', '35', '34', '18', '6', '5', '17', '13']
1062         _video_extensions = {
1063                 '13': '3gp',
1064                 '17': 'mp4',
1065                 '18': 'mp4',
1066                 '22': 'mp4',
1067                 '37': 'mp4',
1068                 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
1069                 '43': 'webm',
1070                 '45': 'webm',
1071         }
1072
1073         @staticmethod
1074         def suitable(url):
1075                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
1076
1077         def report_lang(self):
1078                 """Report attempt to set language."""
1079                 self._downloader.to_screen(u'[youtube] Setting language')
1080
1081         def report_login(self):
1082                 """Report attempt to log in."""
1083                 self._downloader.to_screen(u'[youtube] Logging in')
1084
1085         def report_age_confirmation(self):
1086                 """Report attempt to confirm age."""
1087                 self._downloader.to_screen(u'[youtube] Confirming age')
1088
1089         def report_video_webpage_download(self, video_id):
1090                 """Report attempt to download video webpage."""
1091                 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
1092
1093         def report_video_info_webpage_download(self, video_id):
1094                 """Report attempt to download video info webpage."""
1095                 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
1096
1097         def report_information_extraction(self, video_id):
1098                 """Report attempt to extract video information."""
1099                 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
1100
1101         def report_unavailable_format(self, video_id, format):
1102                 """Report extracted video URL."""
1103                 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
1104
1105         def report_rtmp_download(self):
1106                 """Indicate the download will use the RTMP protocol."""
1107                 self._downloader.to_screen(u'[youtube] RTMP download detected')
1108
1109         def _real_initialize(self):
1110                 if self._downloader is None:
1111                         return
1112
1113                 username = None
1114                 password = None
1115                 downloader_params = self._downloader.params
1116
1117                 # Attempt to use provided username and password or .netrc data
1118                 if downloader_params.get('username', None) is not None:
1119                         username = downloader_params['username']
1120                         password = downloader_params['password']
1121                 elif downloader_params.get('usenetrc', False):
1122                         try:
1123                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
1124                                 if info is not None:
1125                                         username = info[0]
1126                                         password = info[2]
1127                                 else:
1128                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
1129                         except (IOError, netrc.NetrcParseError), err:
1130                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
1131                                 return
1132
1133                 # Set language
1134                 request = urllib2.Request(self._LANG_URL)
1135                 try:
1136                         self.report_lang()
1137                         urllib2.urlopen(request).read()
1138                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1139                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
1140                         return
1141
1142                 # No authentication to be performed
1143                 if username is None:
1144                         return
1145
1146                 # Log in
1147                 login_form = {
1148                                 'current_form': 'loginForm',
1149                                 'next':         '/',
1150                                 'action_login': 'Log In',
1151                                 'username':     username,
1152                                 'password':     password,
1153                                 }
1154                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
1155                 try:
1156                         self.report_login()
1157                         login_results = urllib2.urlopen(request).read()
1158                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
1159                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
1160                                 return
1161                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1162                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
1163                         return
1164
1165                 # Confirm age
1166                 age_form = {
1167                                 'next_url':             '/',
1168                                 'action_confirm':       'Confirm',
1169                                 }
1170                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
1171                 try:
1172                         self.report_age_confirmation()
1173                         age_results = urllib2.urlopen(request).read()
1174                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1175                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1176                         return
1177
1178         def _real_extract(self, url):
1179                 # Extract video id from URL
1180                 mobj = re.match(self._VALID_URL, url)
1181                 if mobj is None:
1182                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1183                         return
1184                 video_id = mobj.group(2)
1185
1186                 # Get video webpage
1187                 self.report_video_webpage_download(video_id)
1188                 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&amp;has_verified=1' % video_id)
1189                 try:
1190                         video_webpage = urllib2.urlopen(request).read()
1191                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1192                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
1193                         return
1194
1195                 # Attempt to extract SWF player URL
1196                 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1197                 if mobj is not None:
1198                         player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1199                 else:
1200                         player_url = None
1201
1202                 # Get video info
1203                 self.report_video_info_webpage_download(video_id)
1204                 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1205                         video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1206                                         % (video_id, el_type))
1207                         request = urllib2.Request(video_info_url)
1208                         try:
1209                                 video_info_webpage = urllib2.urlopen(request).read()
1210                                 video_info = parse_qs(video_info_webpage)
1211                                 if 'token' in video_info:
1212                                         break
1213                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1214                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
1215                                 return
1216                 if 'token' not in video_info:
1217                         if 'reason' in video_info:
1218                                 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
1219                         else:
1220                                 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
1221                         return
1222
1223                 # Start extracting information
1224                 self.report_information_extraction(video_id)
1225
1226                 # uploader
1227                 if 'author' not in video_info:
1228                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1229                         return
1230                 video_uploader = urllib.unquote_plus(video_info['author'][0])
1231
1232                 # title
1233                 if 'title' not in video_info:
1234                         self._downloader.trouble(u'ERROR: unable to extract video title')
1235                         return
1236                 video_title = urllib.unquote_plus(video_info['title'][0])
1237                 video_title = video_title.decode('utf-8')
1238                 video_title = sanitize_title(video_title)
1239
1240                 # simplified title
1241                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1242                 simple_title = simple_title.strip(ur'_')
1243
1244                 # thumbnail image
1245                 if 'thumbnail_url' not in video_info:
1246                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
1247                         video_thumbnail = ''
1248                 else:   # don't panic if we can't find it
1249                         video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
1250
1251                 # upload date
1252                 upload_date = u'NA'
1253                 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
1254                 if mobj is not None:
1255                         upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1256                         format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
1257                         for expression in format_expressions:
1258                                 try:
1259                                         upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
1260                                 except:
1261                                         pass
1262
1263                 # description
1264                 try:
1265                         lxml.etree
1266                 except NameError:
1267                         video_description = u'No description available.'
1268                         if self._downloader.params.get('forcedescription', False) or self._downloader.params.get('writedescription', False):
1269                                 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
1270                                 if mobj is not None:
1271                                         video_description = mobj.group(1).decode('utf-8')
1272                 else:
1273                         html_parser = lxml.etree.HTMLParser(encoding='utf-8')
1274                         vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
1275                         video_description = u''.join(vwebpage_doc.xpath('id("eow-description")//text()'))
1276                         # TODO use another parser
1277
1278                 # token
1279                 video_token = urllib.unquote_plus(video_info['token'][0])
1280
1281                 # Decide which formats to download
1282                 req_format = self._downloader.params.get('format', None)
1283
1284                 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1285                         self.report_rtmp_download()
1286                         video_url_list = [(None, video_info['conn'][0])]
1287                 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
1288                         url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
1289                         url_data = [parse_qs(uds) for uds in url_data_strs]
1290                         url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
1291                         url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
1292
1293                         format_limit = self._downloader.params.get('format_limit', None)
1294                         if format_limit is not None and format_limit in self._available_formats:
1295                                 format_list = self._available_formats[self._available_formats.index(format_limit):]
1296                         else:
1297                                 format_list = self._available_formats
1298                         existing_formats = [x for x in format_list if x in url_map]
1299                         if len(existing_formats) == 0:
1300                                 self._downloader.trouble(u'ERROR: no known formats available for video')
1301                                 return
1302                         if req_format is None:
1303                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
1304                         elif req_format == '-1':
1305                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1306                         else:
1307                                 # Specific format
1308                                 if req_format not in url_map:
1309                                         self._downloader.trouble(u'ERROR: requested format not available')
1310                                         return
1311                                 video_url_list = [(req_format, url_map[req_format])] # Specific format
1312                 else:
1313                         self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
1314                         return
1315
1316                 for format_param, video_real_url in video_url_list:
1317                         # At this point we have a new video
1318                         self._downloader.increment_downloads()
1319
1320                         # Extension
1321                         video_extension = self._video_extensions.get(format_param, 'flv')
1322
1323                         try:
1324                                 # Process video information
1325                                 self._downloader.process_info({
1326                                         'id':           video_id.decode('utf-8'),
1327                                         'url':          video_real_url.decode('utf-8'),
1328                                         'uploader':     video_uploader.decode('utf-8'),
1329                                         'upload_date':  upload_date,
1330                                         'title':        video_title,
1331                                         'stitle':       simple_title,
1332                                         'ext':          video_extension.decode('utf-8'),
1333                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
1334                                         'thumbnail':    video_thumbnail.decode('utf-8'),
1335                                         'description':  video_description,
1336                                         'player_url':   player_url,
1337                                 })
1338                         except UnavailableVideoError, err:
1339                                 self._downloader.trouble(u'\nERROR: unable to download video')
1340
1341
1342 class MetacafeIE(InfoExtractor):
1343         """Information Extractor for metacafe.com."""
1344
1345         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
1346         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
1347         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
1348         _youtube_ie = None
1349
1350         def __init__(self, youtube_ie, downloader=None):
1351                 InfoExtractor.__init__(self, downloader)
1352                 self._youtube_ie = youtube_ie
1353
1354         @staticmethod
1355         def suitable(url):
1356                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
1357
1358         def report_disclaimer(self):
1359                 """Report disclaimer retrieval."""
1360                 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
1361
1362         def report_age_confirmation(self):
1363                 """Report attempt to confirm age."""
1364                 self._downloader.to_screen(u'[metacafe] Confirming age')
1365
1366         def report_download_webpage(self, video_id):
1367                 """Report webpage download."""
1368                 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
1369
1370         def report_extraction(self, video_id):
1371                 """Report information extraction."""
1372                 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
1373
1374         def _real_initialize(self):
1375                 # Retrieve disclaimer
1376                 request = urllib2.Request(self._DISCLAIMER)
1377                 try:
1378                         self.report_disclaimer()
1379                         disclaimer = urllib2.urlopen(request).read()
1380                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1381                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1382                         return
1383
1384                 # Confirm age
1385                 disclaimer_form = {
1386                         'filters': '0',
1387                         'submit': "Continue - I'm over 18",
1388                         }
1389                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
1390                 try:
1391                         self.report_age_confirmation()
1392                         disclaimer = urllib2.urlopen(request).read()
1393                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1394                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1395                         return
1396
1397         def _real_extract(self, url):
1398                 # Extract id and simplified title from URL
1399                 mobj = re.match(self._VALID_URL, url)
1400                 if mobj is None:
1401                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1402                         return
1403
1404                 video_id = mobj.group(1)
1405
1406                 # Check if video comes from YouTube
1407                 mobj2 = re.match(r'^yt-(.*)$', video_id)
1408                 if mobj2 is not None:
1409                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1410                         return
1411
1412                 # At this point we have a new video
1413                 self._downloader.increment_downloads()
1414
1415                 simple_title = mobj.group(2).decode('utf-8')
1416
1417                 # Retrieve video webpage to extract further information
1418                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1419                 try:
1420                         self.report_download_webpage(video_id)
1421                         webpage = urllib2.urlopen(request).read()
1422                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1423                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1424                         return
1425
1426                 # Extract URL, uploader and title from webpage
1427                 self.report_extraction(video_id)
1428                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1429                 if mobj is not None:
1430                         mediaURL = urllib.unquote(mobj.group(1))
1431                         video_extension = mediaURL[-3:]
1432
1433                         # Extract gdaKey if available
1434                         mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1435                         if mobj is None:
1436                                 video_url = mediaURL
1437                         else:
1438                                 gdaKey = mobj.group(1)
1439                                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1440                 else:
1441                         mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
1442                         if mobj is None:
1443                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1444                                 return
1445                         vardict = parse_qs(mobj.group(1))
1446                         if 'mediaData' not in vardict:
1447                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1448                                 return
1449                         mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
1450                         if mobj is None:
1451                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1452                                 return
1453                         mediaURL = mobj.group(1).replace('\\/', '/')
1454                         video_extension = mediaURL[-3:]
1455                         video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
1456
1457                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1458                 if mobj is None:
1459                         self._downloader.trouble(u'ERROR: unable to extract title')
1460                         return
1461                 video_title = mobj.group(1).decode('utf-8')
1462                 video_title = sanitize_title(video_title)
1463
1464                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1465                 if mobj is None:
1466                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1467                         return
1468                 video_uploader = mobj.group(1)
1469
1470                 try:
1471                         # Process video information
1472                         self._downloader.process_info({
1473                                 'id':           video_id.decode('utf-8'),
1474                                 'url':          video_url.decode('utf-8'),
1475                                 'uploader':     video_uploader.decode('utf-8'),
1476                                 'upload_date':  u'NA',
1477                                 'title':        video_title,
1478                                 'stitle':       simple_title,
1479                                 'ext':          video_extension.decode('utf-8'),
1480                                 'format':       u'NA',
1481                                 'player_url':   None,
1482                         })
1483                 except UnavailableVideoError:
1484                         self._downloader.trouble(u'\nERROR: unable to download video')
1485
1486
1487 class DailymotionIE(InfoExtractor):
1488         """Information Extractor for Dailymotion"""
1489
1490         _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1491
1492         def __init__(self, downloader=None):
1493                 InfoExtractor.__init__(self, downloader)
1494
1495         @staticmethod
1496         def suitable(url):
1497                 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1498
1499         def report_download_webpage(self, video_id):
1500                 """Report webpage download."""
1501                 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
1502
1503         def report_extraction(self, video_id):
1504                 """Report information extraction."""
1505                 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
1506
1507         def _real_initialize(self):
1508                 return
1509
1510         def _real_extract(self, url):
1511                 # Extract id and simplified title from URL
1512                 mobj = re.match(self._VALID_URL, url)
1513                 if mobj is None:
1514                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1515                         return
1516
1517                 # At this point we have a new video
1518                 self._downloader.increment_downloads()
1519                 video_id = mobj.group(1)
1520
1521                 simple_title = mobj.group(2).decode('utf-8')
1522                 video_extension = 'flv'
1523
1524                 # Retrieve video webpage to extract further information
1525                 request = urllib2.Request(url)
1526                 try:
1527                         self.report_download_webpage(video_id)
1528                         webpage = urllib2.urlopen(request).read()
1529                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1530                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1531                         return
1532
1533                 # Extract URL, uploader and title from webpage
1534                 self.report_extraction(video_id)
1535                 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1536                 if mobj is None:
1537                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1538                         return
1539                 mediaURL = urllib.unquote(mobj.group(1))
1540
1541                 # if needed add http://www.dailymotion.com/ if relative URL
1542
1543                 video_url = mediaURL
1544
1545                 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1546                 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1547                 if mobj is None:
1548                         self._downloader.trouble(u'ERROR: unable to extract title')
1549                         return
1550                 video_title = mobj.group(1).decode('utf-8')
1551                 video_title = sanitize_title(video_title)
1552
1553                 mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
1554                 if mobj is None:
1555                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1556                         return
1557                 video_uploader = mobj.group(1)
1558
1559                 try:
1560                         # Process video information
1561                         self._downloader.process_info({
1562                                 'id':           video_id.decode('utf-8'),
1563                                 'url':          video_url.decode('utf-8'),
1564                                 'uploader':     video_uploader.decode('utf-8'),
1565                                 'upload_date':  u'NA',
1566                                 'title':        video_title,
1567                                 'stitle':       simple_title,
1568                                 'ext':          video_extension.decode('utf-8'),
1569                                 'format':       u'NA',
1570                                 'player_url':   None,
1571                         })
1572                 except UnavailableVideoError:
1573                         self._downloader.trouble(u'\nERROR: unable to download video')
1574
1575
1576 class GoogleIE(InfoExtractor):
1577         """Information extractor for video.google.com."""
1578
1579         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1580
1581         def __init__(self, downloader=None):
1582                 InfoExtractor.__init__(self, downloader)
1583
1584         @staticmethod
1585         def suitable(url):
1586                 return (re.match(GoogleIE._VALID_URL, url) is not None)
1587
1588         def report_download_webpage(self, video_id):
1589                 """Report webpage download."""
1590                 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
1591
1592         def report_extraction(self, video_id):
1593                 """Report information extraction."""
1594                 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
1595
1596         def _real_initialize(self):
1597                 return
1598
1599         def _real_extract(self, url):
1600                 # Extract id from URL
1601                 mobj = re.match(self._VALID_URL, url)
1602                 if mobj is None:
1603                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1604                         return
1605
1606                 # At this point we have a new video
1607                 self._downloader.increment_downloads()
1608                 video_id = mobj.group(1)
1609
1610                 video_extension = 'mp4'
1611
1612                 # Retrieve video webpage to extract further information
1613                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1614                 try:
1615                         self.report_download_webpage(video_id)
1616                         webpage = urllib2.urlopen(request).read()
1617                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1618                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1619                         return
1620
1621                 # Extract URL, uploader, and title from webpage
1622                 self.report_extraction(video_id)
1623                 mobj = re.search(r"download_url:'([^']+)'", webpage)
1624                 if mobj is None:
1625                         video_extension = 'flv'
1626                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1627                 if mobj is None:
1628                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1629                         return
1630                 mediaURL = urllib.unquote(mobj.group(1))
1631                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1632                 mediaURL = mediaURL.replace('\\x26', '\x26')
1633
1634                 video_url = mediaURL
1635
1636                 mobj = re.search(r'<title>(.*)</title>', webpage)
1637                 if mobj is None:
1638                         self._downloader.trouble(u'ERROR: unable to extract title')
1639                         return
1640                 video_title = mobj.group(1).decode('utf-8')
1641                 video_title = sanitize_title(video_title)
1642                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1643
1644                 # Extract video description
1645                 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1646                 if mobj is None:
1647                         self._downloader.trouble(u'ERROR: unable to extract video description')
1648                         return
1649                 video_description = mobj.group(1).decode('utf-8')
1650                 if not video_description:
1651                         video_description = 'No description available.'
1652
1653                 # Extract video thumbnail
1654                 if self._downloader.params.get('forcethumbnail', False):
1655                         request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1656                         try:
1657                                 webpage = urllib2.urlopen(request).read()
1658                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1659                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1660                                 return
1661                         mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1662                         if mobj is None:
1663                                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1664                                 return
1665                         video_thumbnail = mobj.group(1)
1666                 else:   # we need something to pass to process_info
1667                         video_thumbnail = ''
1668
1669                 try:
1670                         # Process video information
1671                         self._downloader.process_info({
1672                                 'id':           video_id.decode('utf-8'),
1673                                 'url':          video_url.decode('utf-8'),
1674                                 'uploader':     u'NA',
1675                                 'upload_date':  u'NA',
1676                                 'title':        video_title,
1677                                 'stitle':       simple_title,
1678                                 'ext':          video_extension.decode('utf-8'),
1679                                 'format':       u'NA',
1680                                 'player_url':   None,
1681                         })
1682                 except UnavailableVideoError:
1683                         self._downloader.trouble(u'\nERROR: unable to download video')
1684
1685
1686 class PhotobucketIE(InfoExtractor):
1687         """Information extractor for photobucket.com."""
1688
1689         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1690
1691         def __init__(self, downloader=None):
1692                 InfoExtractor.__init__(self, downloader)
1693
1694         @staticmethod
1695         def suitable(url):
1696                 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1697
1698         def report_download_webpage(self, video_id):
1699                 """Report webpage download."""
1700                 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
1701
1702         def report_extraction(self, video_id):
1703                 """Report information extraction."""
1704                 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
1705
1706         def _real_initialize(self):
1707                 return
1708
1709         def _real_extract(self, url):
1710                 # Extract id from URL
1711                 mobj = re.match(self._VALID_URL, url)
1712                 if mobj is None:
1713                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1714                         return
1715
1716                 # At this point we have a new video
1717                 self._downloader.increment_downloads()
1718                 video_id = mobj.group(1)
1719
1720                 video_extension = 'flv'
1721
1722                 # Retrieve video webpage to extract further information
1723                 request = urllib2.Request(url)
1724                 try:
1725                         self.report_download_webpage(video_id)
1726                         webpage = urllib2.urlopen(request).read()
1727                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1728                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1729                         return
1730
1731                 # Extract URL, uploader, and title from webpage
1732                 self.report_extraction(video_id)
1733                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1734                 if mobj is None:
1735                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1736                         return
1737                 mediaURL = urllib.unquote(mobj.group(1))
1738
1739                 video_url = mediaURL
1740
1741                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1742                 if mobj is None:
1743                         self._downloader.trouble(u'ERROR: unable to extract title')
1744                         return
1745                 video_title = mobj.group(1).decode('utf-8')
1746                 video_title = sanitize_title(video_title)
1747                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1748
1749                 video_uploader = mobj.group(2).decode('utf-8')
1750
1751                 try:
1752                         # Process video information
1753                         self._downloader.process_info({
1754                                 'id':           video_id.decode('utf-8'),
1755                                 'url':          video_url.decode('utf-8'),
1756                                 'uploader':     video_uploader,
1757                                 'upload_date':  u'NA',
1758                                 'title':        video_title,
1759                                 'stitle':       simple_title,
1760                                 'ext':          video_extension.decode('utf-8'),
1761                                 'format':       u'NA',
1762                                 'player_url':   None,
1763                         })
1764                 except UnavailableVideoError:
1765                         self._downloader.trouble(u'\nERROR: unable to download video')
1766
1767
1768 class YahooIE(InfoExtractor):
1769         """Information extractor for video.yahoo.com."""
1770
1771         # _VALID_URL matches all Yahoo! Video URLs
1772         # _VPAGE_URL matches only the extractable '/watch/' URLs
1773         _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1774         _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1775
1776         def __init__(self, downloader=None):
1777                 InfoExtractor.__init__(self, downloader)
1778
1779         @staticmethod
1780         def suitable(url):
1781                 return (re.match(YahooIE._VALID_URL, url) is not None)
1782
1783         def report_download_webpage(self, video_id):
1784                 """Report webpage download."""
1785                 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
1786
1787         def report_extraction(self, video_id):
1788                 """Report information extraction."""
1789                 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
1790
1791         def _real_initialize(self):
1792                 return
1793
1794         def _real_extract(self, url, new_video=True):
1795                 # Extract ID from URL
1796                 mobj = re.match(self._VALID_URL, url)
1797                 if mobj is None:
1798                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1799                         return
1800
1801                 # At this point we have a new video
1802                 self._downloader.increment_downloads()
1803                 video_id = mobj.group(2)
1804                 video_extension = 'flv'
1805
1806                 # Rewrite valid but non-extractable URLs as
1807                 # extractable English language /watch/ URLs
1808                 if re.match(self._VPAGE_URL, url) is None:
1809                         request = urllib2.Request(url)
1810                         try:
1811                                 webpage = urllib2.urlopen(request).read()
1812                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1813                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1814                                 return
1815
1816                         mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1817                         if mobj is None:
1818                                 self._downloader.trouble(u'ERROR: Unable to extract id field')
1819                                 return
1820                         yahoo_id = mobj.group(1)
1821
1822                         mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1823                         if mobj is None:
1824                                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1825                                 return
1826                         yahoo_vid = mobj.group(1)
1827
1828                         url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1829                         return self._real_extract(url, new_video=False)
1830
1831                 # Retrieve video webpage to extract further information
1832                 request = urllib2.Request(url)
1833                 try:
1834                         self.report_download_webpage(video_id)
1835                         webpage = urllib2.urlopen(request).read()
1836                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1837                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1838                         return
1839
1840                 # Extract uploader and title from webpage
1841                 self.report_extraction(video_id)
1842                 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1843                 if mobj is None:
1844                         self._downloader.trouble(u'ERROR: unable to extract video title')
1845                         return
1846                 video_title = mobj.group(1).decode('utf-8')
1847                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1848
1849                 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1850                 if mobj is None:
1851                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
1852                         return
1853                 video_uploader = mobj.group(1).decode('utf-8')
1854
1855                 # Extract video thumbnail
1856                 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1857                 if mobj is None:
1858                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1859                         return
1860                 video_thumbnail = mobj.group(1).decode('utf-8')
1861
1862                 # Extract video description
1863                 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1864                 if mobj is None:
1865                         self._downloader.trouble(u'ERROR: unable to extract video description')
1866                         return
1867                 video_description = mobj.group(1).decode('utf-8')
1868                 if not video_description:
1869                         video_description = 'No description available.'
1870
1871                 # Extract video height and width
1872                 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1873                 if mobj is None:
1874                         self._downloader.trouble(u'ERROR: unable to extract video height')
1875                         return
1876                 yv_video_height = mobj.group(1)
1877
1878                 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1879                 if mobj is None:
1880                         self._downloader.trouble(u'ERROR: unable to extract video width')
1881                         return
1882                 yv_video_width = mobj.group(1)
1883
1884                 # Retrieve video playlist to extract media URL
1885                 # I'm not completely sure what all these options are, but we
1886                 # seem to need most of them, otherwise the server sends a 401.
1887                 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
1888                 yv_bitrate = '700'  # according to Wikipedia this is hard-coded
1889                 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1890                                 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1891                                 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1892                 try:
1893                         self.report_download_webpage(video_id)
1894                         webpage = urllib2.urlopen(request).read()
1895                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1896                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1897                         return
1898
1899                 # Extract media URL from playlist XML
1900                 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1901                 if mobj is None:
1902                         self._downloader.trouble(u'ERROR: Unable to extract media URL')
1903                         return
1904                 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1905                 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1906
1907                 try:
1908                         # Process video information
1909                         self._downloader.process_info({
1910                                 'id':           video_id.decode('utf-8'),
1911                                 'url':          video_url,
1912                                 'uploader':     video_uploader,
1913                                 'upload_date':  u'NA',
1914                                 'title':        video_title,
1915                                 'stitle':       simple_title,
1916                                 'ext':          video_extension.decode('utf-8'),
1917                                 'thumbnail':    video_thumbnail.decode('utf-8'),
1918                                 'description':  video_description,
1919                                 'thumbnail':    video_thumbnail,
1920                                 'player_url':   None,
1921                         })
1922                 except UnavailableVideoError:
1923                         self._downloader.trouble(u'\nERROR: unable to download video')
1924
1925
1926 class VimeoIE(InfoExtractor):
1927         """Information extractor for vimeo.com."""
1928
1929         # _VALID_URL matches Vimeo URLs
1930         _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)'
1931
1932         def __init__(self, downloader=None):
1933                 InfoExtractor.__init__(self, downloader)
1934
1935         @staticmethod
1936         def suitable(url):
1937                 return (re.match(VimeoIE._VALID_URL, url) is not None)
1938
1939         def report_download_webpage(self, video_id):
1940                 """Report webpage download."""
1941                 self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1942
1943         def report_extraction(self, video_id):
1944                 """Report information extraction."""
1945                 self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1946
1947         def _real_initialize(self):
1948                 return
1949
1950         def _real_extract(self, url, new_video=True):
1951                 # Extract ID from URL
1952                 mobj = re.match(self._VALID_URL, url)
1953                 if mobj is None:
1954                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1955                         return
1956
1957                 # At this point we have a new video
1958                 self._downloader.increment_downloads()
1959                 video_id = mobj.group(1)
1960
1961                 # Retrieve video webpage to extract further information
1962                 request = urllib2.Request("http://vimeo.com/moogaloop/load/clip:%s" % video_id, None, std_headers)
1963                 try:
1964                         self.report_download_webpage(video_id)
1965                         webpage = urllib2.urlopen(request).read()
1966                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1967                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1968                         return
1969
1970                 # Now we begin extracting as much information as we can from what we
1971                 # retrieved. First we extract the information common to all extractors,
1972                 # and latter we extract those that are Vimeo specific.
1973                 self.report_extraction(video_id)
1974
1975                 # Extract title
1976                 mobj = re.search(r'<caption>(.*?)</caption>', webpage)
1977                 if mobj is None:
1978                         self._downloader.trouble(u'ERROR: unable to extract video title')
1979                         return
1980                 video_title = mobj.group(1).decode('utf-8')
1981                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1982
1983                 # Extract uploader
1984                 mobj = re.search(r'<uploader_url>http://vimeo.com/(.*?)</uploader_url>', webpage)
1985                 if mobj is None:
1986                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
1987                         return
1988                 video_uploader = mobj.group(1).decode('utf-8')
1989
1990                 # Extract video thumbnail
1991                 mobj = re.search(r'<thumbnail>(.*?)</thumbnail>', webpage)
1992                 if mobj is None:
1993                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1994                         return
1995                 video_thumbnail = mobj.group(1).decode('utf-8')
1996
1997                 # # Extract video description
1998                 # mobj = re.search(r'<meta property="og:description" content="(.*)" />', webpage)
1999                 # if mobj is None:
2000                 #       self._downloader.trouble(u'ERROR: unable to extract video description')
2001                 #       return
2002                 # video_description = mobj.group(1).decode('utf-8')
2003                 # if not video_description: video_description = 'No description available.'
2004                 video_description = 'Foo.'
2005
2006                 # Vimeo specific: extract request signature
2007                 mobj = re.search(r'<request_signature>(.*?)</request_signature>', webpage)
2008                 if mobj is None:
2009                         self._downloader.trouble(u'ERROR: unable to extract request signature')
2010                         return
2011                 sig = mobj.group(1).decode('utf-8')
2012
2013                 # Vimeo specific: Extract request signature expiration
2014                 mobj = re.search(r'<request_signature_expires>(.*?)</request_signature_expires>', webpage)
2015                 if mobj is None:
2016                         self._downloader.trouble(u'ERROR: unable to extract request signature expiration')
2017                         return
2018                 sig_exp = mobj.group(1).decode('utf-8')
2019
2020                 video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s" % (video_id, sig, sig_exp)
2021
2022                 try:
2023                         # Process video information
2024                         self._downloader.process_info({
2025                                 'id':           video_id.decode('utf-8'),
2026                                 'url':          video_url,
2027                                 'uploader':     video_uploader,
2028                                 'upload_date':  u'NA',
2029                                 'title':        video_title,
2030                                 'stitle':       simple_title,
2031                                 'ext':          u'mp4',
2032                                 'thumbnail':    video_thumbnail.decode('utf-8'),
2033                                 'description':  video_description,
2034                                 'thumbnail':    video_thumbnail,
2035                                 'description':  video_description,
2036                                 'player_url':   None,
2037                         })
2038                 except UnavailableVideoError:
2039                         self._downloader.trouble(u'ERROR: unable to download video')
2040
2041
2042 class GenericIE(InfoExtractor):
2043         """Generic last-resort information extractor."""
2044
2045         def __init__(self, downloader=None):
2046                 InfoExtractor.__init__(self, downloader)
2047
2048         @staticmethod
2049         def suitable(url):
2050                 return True
2051
2052         def report_download_webpage(self, video_id):
2053                 """Report webpage download."""
2054                 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
2055                 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
2056
2057         def report_extraction(self, video_id):
2058                 """Report information extraction."""
2059                 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
2060
2061         def _real_initialize(self):
2062                 return
2063
2064         def _real_extract(self, url):
2065                 # At this point we have a new video
2066                 self._downloader.increment_downloads()
2067
2068                 video_id = url.split('/')[-1]
2069                 request = urllib2.Request(url)
2070                 try:
2071                         self.report_download_webpage(video_id)
2072                         webpage = urllib2.urlopen(request).read()
2073                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2074                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
2075                         return
2076                 except ValueError, err:
2077                         # since this is the last-resort InfoExtractor, if
2078                         # this error is thrown, it'll be thrown here
2079                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2080                         return
2081
2082                 self.report_extraction(video_id)
2083                 # Start with something easy: JW Player in SWFObject
2084                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
2085                 if mobj is None:
2086                         # Broaden the search a little bit
2087                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
2088                 if mobj is None:
2089                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2090                         return
2091
2092                 # It's possible that one of the regexes
2093                 # matched, but returned an empty group:
2094                 if mobj.group(1) is None:
2095                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2096                         return
2097
2098                 video_url = urllib.unquote(mobj.group(1))
2099                 video_id = os.path.basename(video_url)
2100
2101                 # here's a fun little line of code for you:
2102                 video_extension = os.path.splitext(video_id)[1][1:]
2103                 video_id = os.path.splitext(video_id)[0]
2104
2105                 # it's tempting to parse this further, but you would
2106                 # have to take into account all the variations like
2107                 #   Video Title - Site Name
2108                 #   Site Name | Video Title
2109                 #   Video Title - Tagline | Site Name
2110                 # and so on and so forth; it's just not practical
2111                 mobj = re.search(r'<title>(.*)</title>', webpage)
2112                 if mobj is None:
2113                         self._downloader.trouble(u'ERROR: unable to extract title')
2114                         return
2115                 video_title = mobj.group(1).decode('utf-8')
2116                 video_title = sanitize_title(video_title)
2117                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
2118
2119                 # video uploader is domain name
2120                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
2121                 if mobj is None:
2122                         self._downloader.trouble(u'ERROR: unable to extract title')
2123                         return
2124                 video_uploader = mobj.group(1).decode('utf-8')
2125
2126                 try:
2127                         # Process video information
2128                         self._downloader.process_info({
2129                                 'id':           video_id.decode('utf-8'),
2130                                 'url':          video_url.decode('utf-8'),
2131                                 'uploader':     video_uploader,
2132                                 'upload_date':  u'NA',
2133                                 'title':        video_title,
2134                                 'stitle':       simple_title,
2135                                 'ext':          video_extension.decode('utf-8'),
2136                                 'format':       u'NA',
2137                                 'player_url':   None,
2138                         })
2139                 except UnavailableVideoError, err:
2140                         self._downloader.trouble(u'\nERROR: unable to download video')
2141
2142
2143 class YoutubeSearchIE(InfoExtractor):
2144         """Information Extractor for YouTube search queries."""
2145         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
2146         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
2147         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
2148         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
2149         _youtube_ie = None
2150         _max_youtube_results = 1000
2151
2152         def __init__(self, youtube_ie, downloader=None):
2153                 InfoExtractor.__init__(self, downloader)
2154                 self._youtube_ie = youtube_ie
2155
2156         @staticmethod
2157         def suitable(url):
2158                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
2159
2160         def report_download_page(self, query, pagenum):
2161                 """Report attempt to download playlist page with given number."""
2162                 query = query.decode(preferredencoding())
2163                 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
2164
2165         def _real_initialize(self):
2166                 self._youtube_ie.initialize()
2167
2168         def _real_extract(self, query):
2169                 mobj = re.match(self._VALID_QUERY, query)
2170                 if mobj is None:
2171                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2172                         return
2173
2174                 prefix, query = query.split(':')
2175                 prefix = prefix[8:]
2176                 query = query.encode('utf-8')
2177                 if prefix == '':
2178                         self._download_n_results(query, 1)
2179                         return
2180                 elif prefix == 'all':
2181                         self._download_n_results(query, self._max_youtube_results)
2182                         return
2183                 else:
2184                         try:
2185                                 n = long(prefix)
2186                                 if n <= 0:
2187                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2188                                         return
2189                                 elif n > self._max_youtube_results:
2190                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
2191                                         n = self._max_youtube_results
2192                                 self._download_n_results(query, n)
2193                                 return
2194                         except ValueError: # parsing prefix as integer fails
2195                                 self._download_n_results(query, 1)
2196                                 return
2197
2198         def _download_n_results(self, query, n):
2199                 """Downloads a specified number of results for a query"""
2200
2201                 video_ids = []
2202                 already_seen = set()
2203                 pagenum = 1
2204
2205                 while True:
2206                         self.report_download_page(query, pagenum)
2207                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2208                         request = urllib2.Request(result_url)
2209                         try:
2210                                 page = urllib2.urlopen(request).read()
2211                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2212                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2213                                 return
2214
2215                         # Extract video identifiers
2216                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2217                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
2218                                 if video_id not in already_seen:
2219                                         video_ids.append(video_id)
2220                                         already_seen.add(video_id)
2221                                         if len(video_ids) == n:
2222                                                 # Specified n videos reached
2223                                                 for id in video_ids:
2224                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2225                                                 return
2226
2227                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2228                                 for id in video_ids:
2229                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2230                                 return
2231
2232                         pagenum = pagenum + 1
2233
2234
2235 class GoogleSearchIE(InfoExtractor):
2236         """Information Extractor for Google Video search queries."""
2237         _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
2238         _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
2239         _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
2240         _MORE_PAGES_INDICATOR = r'<span>Next</span>'
2241         _google_ie = None
2242         _max_google_results = 1000
2243
2244         def __init__(self, google_ie, downloader=None):
2245                 InfoExtractor.__init__(self, downloader)
2246                 self._google_ie = google_ie
2247
2248         @staticmethod
2249         def suitable(url):
2250                 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
2251
2252         def report_download_page(self, query, pagenum):
2253                 """Report attempt to download playlist page with given number."""
2254                 query = query.decode(preferredencoding())
2255                 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
2256
2257         def _real_initialize(self):
2258                 self._google_ie.initialize()
2259
2260         def _real_extract(self, query):
2261                 mobj = re.match(self._VALID_QUERY, query)
2262                 if mobj is None:
2263                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2264                         return
2265
2266                 prefix, query = query.split(':')
2267                 prefix = prefix[8:]
2268                 query = query.encode('utf-8')
2269                 if prefix == '':
2270                         self._download_n_results(query, 1)
2271                         return
2272                 elif prefix == 'all':
2273                         self._download_n_results(query, self._max_google_results)
2274                         return
2275                 else:
2276                         try:
2277                                 n = long(prefix)
2278                                 if n <= 0:
2279                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2280                                         return
2281                                 elif n > self._max_google_results:
2282                                         self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
2283                                         n = self._max_google_results
2284                                 self._download_n_results(query, n)
2285                                 return
2286                         except ValueError: # parsing prefix as integer fails
2287                                 self._download_n_results(query, 1)
2288                                 return
2289
2290         def _download_n_results(self, query, n):
2291                 """Downloads a specified number of results for a query"""
2292
2293                 video_ids = []
2294                 already_seen = set()
2295                 pagenum = 1
2296
2297                 while True:
2298                         self.report_download_page(query, pagenum)
2299                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2300                         request = urllib2.Request(result_url)
2301                         try:
2302                                 page = urllib2.urlopen(request).read()
2303                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2304                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2305                                 return
2306
2307                         # Extract video identifiers
2308                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2309                                 video_id = mobj.group(1)
2310                                 if video_id not in already_seen:
2311                                         video_ids.append(video_id)
2312                                         already_seen.add(video_id)
2313                                         if len(video_ids) == n:
2314                                                 # Specified n videos reached
2315                                                 for id in video_ids:
2316                                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
2317                                                 return
2318
2319                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2320                                 for id in video_ids:
2321                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
2322                                 return
2323
2324                         pagenum = pagenum + 1
2325
2326
2327 class YahooSearchIE(InfoExtractor):
2328         """Information Extractor for Yahoo! Video search queries."""
2329         _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
2330         _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
2331         _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
2332         _MORE_PAGES_INDICATOR = r'\s*Next'
2333         _yahoo_ie = None
2334         _max_yahoo_results = 1000
2335
2336         def __init__(self, yahoo_ie, downloader=None):
2337                 InfoExtractor.__init__(self, downloader)
2338                 self._yahoo_ie = yahoo_ie
2339
2340         @staticmethod
2341         def suitable(url):
2342                 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
2343
2344         def report_download_page(self, query, pagenum):
2345                 """Report attempt to download playlist page with given number."""
2346                 query = query.decode(preferredencoding())
2347                 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
2348
2349         def _real_initialize(self):
2350                 self._yahoo_ie.initialize()
2351
2352         def _real_extract(self, query):
2353                 mobj = re.match(self._VALID_QUERY, query)
2354                 if mobj is None:
2355                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2356                         return
2357
2358                 prefix, query = query.split(':')
2359                 prefix = prefix[8:]
2360                 query = query.encode('utf-8')
2361                 if prefix == '':
2362                         self._download_n_results(query, 1)
2363                         return
2364                 elif prefix == 'all':
2365                         self._download_n_results(query, self._max_yahoo_results)
2366                         return
2367                 else:
2368                         try:
2369                                 n = long(prefix)
2370                                 if n <= 0:
2371                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2372                                         return
2373                                 elif n > self._max_yahoo_results:
2374                                         self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
2375                                         n = self._max_yahoo_results
2376                                 self._download_n_results(query, n)
2377                                 return
2378                         except ValueError: # parsing prefix as integer fails
2379                                 self._download_n_results(query, 1)
2380                                 return
2381
2382         def _download_n_results(self, query, n):
2383                 """Downloads a specified number of results for a query"""
2384
2385                 video_ids = []
2386                 already_seen = set()
2387                 pagenum = 1
2388
2389                 while True:
2390                         self.report_download_page(query, pagenum)
2391                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2392                         request = urllib2.Request(result_url)
2393                         try:
2394                                 page = urllib2.urlopen(request).read()
2395                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2396                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2397                                 return
2398
2399                         # Extract video identifiers
2400                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2401                                 video_id = mobj.group(1)
2402                                 if video_id not in already_seen:
2403                                         video_ids.append(video_id)
2404                                         already_seen.add(video_id)
2405                                         if len(video_ids) == n:
2406                                                 # Specified n videos reached
2407                                                 for id in video_ids:
2408                                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
2409                                                 return
2410
2411                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2412                                 for id in video_ids:
2413                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
2414                                 return
2415
2416                         pagenum = pagenum + 1
2417
2418
2419 class YoutubePlaylistIE(InfoExtractor):
2420         """Information Extractor for YouTube playlists."""
2421
2422         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
2423         _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
2424         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
2425         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
2426         _youtube_ie = None
2427
2428         def __init__(self, youtube_ie, downloader=None):
2429                 InfoExtractor.__init__(self, downloader)
2430                 self._youtube_ie = youtube_ie
2431
2432         @staticmethod
2433         def suitable(url):
2434                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
2435
2436         def report_download_page(self, playlist_id, pagenum):
2437                 """Report attempt to download playlist page with given number."""
2438                 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
2439
2440         def _real_initialize(self):
2441                 self._youtube_ie.initialize()
2442
2443         def _real_extract(self, url):
2444                 # Extract playlist id
2445                 mobj = re.match(self._VALID_URL, url)
2446                 if mobj is None:
2447                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2448                         return
2449
2450                 # Single video case
2451                 if mobj.group(3) is not None:
2452                         self._youtube_ie.extract(mobj.group(3))
2453                         return
2454
2455                 # Download playlist pages
2456                 # prefix is 'p' as default for playlists but there are other types that need extra care
2457                 playlist_prefix = mobj.group(1)
2458                 if playlist_prefix == 'a':
2459                         playlist_access = 'artist'
2460                 else:
2461                         playlist_prefix = 'p'
2462                         playlist_access = 'view_play_list'
2463                 playlist_id = mobj.group(2)
2464                 video_ids = []
2465                 pagenum = 1
2466
2467                 while True:
2468                         self.report_download_page(playlist_id, pagenum)
2469                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum))
2470                         try:
2471                                 page = urllib2.urlopen(request).read()
2472                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2473                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2474                                 return
2475
2476                         # Extract video identifiers
2477                         ids_in_page = []
2478                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2479                                 if mobj.group(1) not in ids_in_page:
2480                                         ids_in_page.append(mobj.group(1))
2481                         video_ids.extend(ids_in_page)
2482
2483                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2484                                 break
2485                         pagenum = pagenum + 1
2486
2487                 playliststart = self._downloader.params.get('playliststart', 1) - 1
2488                 playlistend = self._downloader.params.get('playlistend', -1)
2489                 video_ids = video_ids[playliststart:playlistend]
2490
2491                 for id in video_ids:
2492                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2493                 return
2494
2495
2496 class YoutubeUserIE(InfoExtractor):
2497         """Information Extractor for YouTube users."""
2498
2499         _VALID_URL = r'(?:(?:(?:http://)?(?:\w+\.)?youtube.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
2500         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
2501         _GDATA_PAGE_SIZE = 50
2502         _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
2503         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
2504         _youtube_ie = None
2505
2506         def __init__(self, youtube_ie, downloader=None):
2507                 InfoExtractor.__init__(self, downloader)
2508                 self._youtube_ie = youtube_ie
2509
2510         @staticmethod
2511         def suitable(url):
2512                 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
2513
2514         def report_download_page(self, username, start_index):
2515                 """Report attempt to download user page."""
2516                 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
2517                                 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
2518
2519         def _real_initialize(self):
2520                 self._youtube_ie.initialize()
2521
2522         def _real_extract(self, url):
2523                 # Extract username
2524                 mobj = re.match(self._VALID_URL, url)
2525                 if mobj is None:
2526                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2527                         return
2528
2529                 username = mobj.group(1)
2530
2531                 # Download video ids using YouTube Data API. Result size per
2532                 # query is limited (currently to 50 videos) so we need to query
2533                 # page by page until there are no video ids - it means we got
2534                 # all of them.
2535
2536                 video_ids = []
2537                 pagenum = 0
2538
2539                 while True:
2540                         start_index = pagenum * self._GDATA_PAGE_SIZE + 1
2541                         self.report_download_page(username, start_index)
2542
2543                         request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
2544
2545                         try:
2546                                 page = urllib2.urlopen(request).read()
2547                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2548                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2549                                 return
2550
2551                         # Extract video identifiers
2552                         ids_in_page = []
2553
2554                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2555                                 if mobj.group(1) not in ids_in_page:
2556                                         ids_in_page.append(mobj.group(1))
2557
2558                         video_ids.extend(ids_in_page)
2559
2560                         # A little optimization - if current page is not
2561                         # "full", ie. does not contain PAGE_SIZE video ids then
2562                         # we can assume that this page is the last one - there
2563                         # are no more ids on further pages - no need to query
2564                         # again.
2565
2566                         if len(ids_in_page) < self._GDATA_PAGE_SIZE:
2567                                 break
2568
2569                         pagenum += 1
2570
2571                 all_ids_count = len(video_ids)
2572                 playliststart = self._downloader.params.get('playliststart', 1) - 1
2573                 playlistend = self._downloader.params.get('playlistend', -1)
2574
2575                 if playlistend == -1:
2576                         video_ids = video_ids[playliststart:]
2577                 else:
2578                         video_ids = video_ids[playliststart:playlistend]
2579
2580                 self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" %
2581                                 (username, all_ids_count, len(video_ids)))
2582
2583                 for video_id in video_ids:
2584                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id)
2585
2586
2587 class DepositFilesIE(InfoExtractor):
2588         """Information extractor for depositfiles.com"""
2589
2590         _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)'
2591
2592         def __init__(self, downloader=None):
2593                 InfoExtractor.__init__(self, downloader)
2594
2595         @staticmethod
2596         def suitable(url):
2597                 return (re.match(DepositFilesIE._VALID_URL, url) is not None)
2598
2599         def report_download_webpage(self, file_id):
2600                 """Report webpage download."""
2601                 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
2602
2603         def report_extraction(self, file_id):
2604                 """Report information extraction."""
2605                 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
2606
2607         def _real_initialize(self):
2608                 return
2609
2610         def _real_extract(self, url):
2611                 # At this point we have a new file
2612                 self._downloader.increment_downloads()
2613
2614                 file_id = url.split('/')[-1]
2615                 # Rebuild url in english locale
2616                 url = 'http://depositfiles.com/en/files/' + file_id
2617
2618                 # Retrieve file webpage with 'Free download' button pressed
2619                 free_download_indication = { 'gateway_result' : '1' }
2620                 request = urllib2.Request(url, urllib.urlencode(free_download_indication))
2621                 try:
2622                         self.report_download_webpage(file_id)
2623                         webpage = urllib2.urlopen(request).read()
2624                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2625                         self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
2626                         return
2627
2628                 # Search for the real file URL
2629                 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
2630                 if (mobj is None) or (mobj.group(1) is None):
2631                         # Try to figure out reason of the error.
2632                         mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
2633                         if (mobj is not None) and (mobj.group(1) is not None):
2634                                 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
2635                                 self._downloader.trouble(u'ERROR: %s' % restriction_message)
2636                         else:
2637                                 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
2638                         return
2639
2640                 file_url = mobj.group(1)
2641                 file_extension = os.path.splitext(file_url)[1][1:]
2642
2643                 # Search for file title
2644                 mobj = re.search(r'<b title="(.*?)">', webpage)
2645                 if mobj is None:
2646                         self._downloader.trouble(u'ERROR: unable to extract title')
2647                         return
2648                 file_title = mobj.group(1).decode('utf-8')
2649
2650                 try:
2651                         # Process file information
2652                         self._downloader.process_info({
2653                                 'id':           file_id.decode('utf-8'),
2654                                 'url':          file_url.decode('utf-8'),
2655                                 'uploader':     u'NA',
2656                                 'upload_date':  u'NA',
2657                                 'title':        file_title,
2658                                 'stitle':       file_title,
2659                                 'ext':          file_extension.decode('utf-8'),
2660                                 'format':       u'NA',
2661                                 'player_url':   None,
2662                         })
2663                 except UnavailableVideoError, err:
2664                         self._downloader.trouble(u'ERROR: unable to download file')
2665
2666
2667 class FacebookIE(InfoExtractor):
2668         """Information Extractor for Facebook"""
2669
2670         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook.com/video/video.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
2671         _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
2672         _NETRC_MACHINE = 'facebook'
2673         _available_formats = ['highqual', 'lowqual']
2674         _video_extensions = {
2675                 'highqual': 'mp4',
2676                 'lowqual': 'mp4',
2677         }
2678
2679         def __init__(self, downloader=None):
2680                 InfoExtractor.__init__(self, downloader)
2681
2682         @staticmethod
2683         def suitable(url):
2684                 return (re.match(FacebookIE._VALID_URL, url) is not None)
2685
2686         def _reporter(self, message):
2687                 """Add header and report message."""
2688                 self._downloader.to_screen(u'[facebook] %s' % message)
2689
2690         def report_login(self):
2691                 """Report attempt to log in."""
2692                 self._reporter(u'Logging in')
2693
2694         def report_video_webpage_download(self, video_id):
2695                 """Report attempt to download video webpage."""
2696                 self._reporter(u'%s: Downloading video webpage' % video_id)
2697
2698         def report_information_extraction(self, video_id):
2699                 """Report attempt to extract video information."""
2700                 self._reporter(u'%s: Extracting video information' % video_id)
2701
2702         def _parse_page(self, video_webpage):
2703                 """Extract video information from page"""
2704                 # General data
2705                 data = {'title': r'class="video_title datawrap">(.*?)</',
2706                         'description': r'<div class="datawrap">(.*?)</div>',
2707                         'owner': r'\("video_owner_name", "(.*?)"\)',
2708                         'upload_date': r'data-date="(.*?)"',
2709                         'thumbnail':  r'\("thumb_url", "(?P<THUMB>.*?)"\)',
2710                         }
2711                 video_info = {}
2712                 for piece in data.keys():
2713                         mobj = re.search(data[piece], video_webpage)
2714                         if mobj is not None:
2715                                 video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
2716
2717                 # Video urls
2718                 video_urls = {}
2719                 for fmt in self._available_formats:
2720                         mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
2721                         if mobj is not None:
2722                                 # URL is in a Javascript segment inside an escaped Unicode format within
2723                                 # the generally utf-8 page
2724                                 video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
2725                 video_info['video_urls'] = video_urls
2726
2727                 return video_info
2728
2729         def _real_initialize(self):
2730                 if self._downloader is None:
2731                         return
2732
2733                 useremail = None
2734                 password = None
2735                 downloader_params = self._downloader.params
2736
2737                 # Attempt to use provided username and password or .netrc data
2738                 if downloader_params.get('username', None) is not None:
2739                         useremail = downloader_params['username']
2740                         password = downloader_params['password']
2741                 elif downloader_params.get('usenetrc', False):
2742                         try:
2743                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
2744                                 if info is not None:
2745                                         useremail = info[0]
2746                                         password = info[2]
2747                                 else:
2748                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
2749                         except (IOError, netrc.NetrcParseError), err:
2750                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
2751                                 return
2752
2753                 if useremail is None:
2754                         return
2755
2756                 # Log in
2757                 login_form = {
2758                         'email': useremail,
2759                         'pass': password,
2760                         'login': 'Log+In'
2761                         }
2762                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
2763                 try:
2764                         self.report_login()
2765                         login_results = urllib2.urlopen(request).read()
2766                         if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
2767                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
2768                                 return
2769                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2770                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
2771                         return
2772
2773         def _real_extract(self, url):
2774                 mobj = re.match(self._VALID_URL, url)
2775                 if mobj is None:
2776                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2777                         return
2778                 video_id = mobj.group('ID')
2779
2780                 # Get video webpage
2781                 self.report_video_webpage_download(video_id)
2782                 request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
2783                 try:
2784                         page = urllib2.urlopen(request)
2785                         video_webpage = page.read()
2786                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2787                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2788                         return
2789
2790                 # Start extracting information
2791                 self.report_information_extraction(video_id)
2792
2793                 # Extract information
2794                 video_info = self._parse_page(video_webpage)
2795
2796                 # uploader
2797                 if 'owner' not in video_info:
2798                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
2799                         return
2800                 video_uploader = video_info['owner']
2801
2802                 # title
2803                 if 'title' not in video_info:
2804                         self._downloader.trouble(u'ERROR: unable to extract video title')
2805                         return
2806                 video_title = video_info['title']
2807                 video_title = video_title.decode('utf-8')
2808                 video_title = sanitize_title(video_title)
2809
2810                 # simplified title
2811                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
2812                 simple_title = simple_title.strip(ur'_')
2813
2814                 # thumbnail image
2815                 if 'thumbnail' not in video_info:
2816                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
2817                         video_thumbnail = ''
2818                 else:
2819                         video_thumbnail = video_info['thumbnail']
2820
2821                 # upload date
2822                 upload_date = u'NA'
2823                 if 'upload_date' in video_info:
2824                         upload_time = video_info['upload_date']
2825                         timetuple = email.utils.parsedate_tz(upload_time)
2826                         if timetuple is not None:
2827                                 try:
2828                                         upload_date = time.strftime('%Y%m%d', timetuple[0:9])
2829                                 except:
2830                                         pass
2831
2832                 # description
2833                 video_description = video_info.get('description', 'No description available.')
2834
2835                 url_map = video_info['video_urls']
2836                 if len(url_map.keys()) > 0:
2837                         # Decide which formats to download
2838                         req_format = self._downloader.params.get('format', None)
2839                         format_limit = self._downloader.params.get('format_limit', None)
2840
2841                         if format_limit is not None and format_limit in self._available_formats:
2842                                 format_list = self._available_formats[self._available_formats.index(format_limit):]
2843                         else:
2844                                 format_list = self._available_formats
2845                         existing_formats = [x for x in format_list if x in url_map]
2846                         if len(existing_formats) == 0:
2847                                 self._downloader.trouble(u'ERROR: no known formats available for video')
2848                                 return
2849                         if req_format is None:
2850                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2851                         elif req_format == '-1':
2852                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2853                         else:
2854                                 # Specific format
2855                                 if req_format not in url_map:
2856                                         self._downloader.trouble(u'ERROR: requested format not available')
2857                                         return
2858                                 video_url_list = [(req_format, url_map[req_format])] # Specific format
2859
2860                 for format_param, video_real_url in video_url_list:
2861
2862                         # At this point we have a new video
2863                         self._downloader.increment_downloads()
2864
2865                         # Extension
2866                         video_extension = self._video_extensions.get(format_param, 'mp4')
2867
2868                         try:
2869                                 # Process video information
2870                                 self._downloader.process_info({
2871                                         'id':           video_id.decode('utf-8'),
2872                                         'url':          video_real_url.decode('utf-8'),
2873                                         'uploader':     video_uploader.decode('utf-8'),
2874                                         'upload_date':  upload_date,
2875                                         'title':        video_title,
2876                                         'stitle':       simple_title,
2877                                         'ext':          video_extension.decode('utf-8'),
2878                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
2879                                         'thumbnail':    video_thumbnail.decode('utf-8'),
2880                                         'description':  video_description.decode('utf-8'),
2881                                         'player_url':   None,
2882                                 })
2883                         except UnavailableVideoError, err:
2884                                 self._downloader.trouble(u'\nERROR: unable to download video')
2885
2886 class BlipTVIE(InfoExtractor):
2887         """Information extractor for blip.tv"""
2888
2889         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2890         _URL_EXT = r'^.*\.([a-z0-9]+)$'
2891
2892         @staticmethod
2893         def suitable(url):
2894                 return (re.match(BlipTVIE._VALID_URL, url) is not None)
2895
2896         def report_extraction(self, file_id):
2897                 """Report information extraction."""
2898                 self._downloader.to_screen(u'[blip.tv] %s: Extracting information' % file_id)
2899
2900         def _simplify_title(self, title):
2901                 res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
2902                 res = res.strip(ur'_')
2903                 return res
2904
2905         def _real_extract(self, url):
2906                 mobj = re.match(self._VALID_URL, url)
2907                 if mobj is None:
2908                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2909                         return
2910
2911                 if '?' in url:
2912                         cchar = '&'
2913                 else:
2914                         cchar = '?'
2915                 json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2916                 request = urllib2.Request(json_url)
2917                 self.report_extraction(mobj.group(1))
2918                 try:
2919                         json_code = urllib2.urlopen(request).read()
2920                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2921                         self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
2922                         return
2923                 try:
2924                         json_data = json.loads(json_code)
2925                         if 'Post' in json_data:
2926                                 data = json_data['Post']
2927                         else:
2928                                 data = json_data
2929
2930                         upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2931                         video_url = data['media']['url']
2932                         umobj = re.match(self._URL_EXT, video_url)
2933                         if umobj is None:
2934                                 raise ValueError('Can not determine filename extension')
2935                         ext = umobj.group(1)
2936
2937                         self._downloader.increment_downloads()
2938
2939                         info = {
2940                                 'id': data['item_id'],
2941                                 'url': video_url,
2942                                 'uploader': data['display_name'],
2943                                 'upload_date': upload_date,
2944                                 'title': data['title'],
2945                                 'stitle': self._simplify_title(data['title']),
2946                                 'ext': ext,
2947                                 'format': data['media']['mimeType'],
2948                                 'thumbnail': data['thumbnailUrl'],
2949                                 'description': data['description'],
2950                                 'player_url': data['embedUrl']
2951                         }
2952                 except (ValueError,KeyError), err:
2953                         self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2954                         return
2955
2956                 try:
2957                         self._downloader.process_info(info)
2958                 except UnavailableVideoError, err:
2959                         self._downloader.trouble(u'\nERROR: unable to download video')
2960
2961
2962 class MyVideoIE(InfoExtractor):
2963         """Information Extractor for myvideo.de."""
2964
2965         _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
2966
2967         def __init__(self, downloader=None):
2968                 InfoExtractor.__init__(self, downloader)
2969
2970         @staticmethod
2971         def suitable(url):
2972                 return (re.match(MyVideoIE._VALID_URL, url) is not None)
2973
2974         def report_download_webpage(self, video_id):
2975                 """Report webpage download."""
2976                 self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
2977
2978         def report_extraction(self, video_id):
2979                 """Report information extraction."""
2980                 self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
2981
2982         def _real_initialize(self):
2983                 return
2984
2985         def _real_extract(self,url):
2986                 mobj = re.match(self._VALID_URL, url)
2987                 if mobj is None:
2988                         self._download.trouble(u'ERROR: invalid URL: %s' % url)
2989                         return
2990
2991                 video_id = mobj.group(1)
2992                 simple_title = mobj.group(2).decode('utf-8')
2993                 # should actually not be necessary
2994                 simple_title = sanitize_title(simple_title)
2995                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', simple_title)
2996
2997                 # Get video webpage
2998                 request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
2999                 try:
3000                         self.report_download_webpage(video_id)
3001                         webpage = urllib2.urlopen(request).read()
3002                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3003                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
3004                         return
3005
3006                 self.report_extraction(video_id)
3007                 mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
3008                                  webpage)
3009                 if mobj is None:
3010                         self._downloader.trouble(u'ERROR: unable to extract media URL')
3011                         return
3012                 video_url = mobj.group(1) + ('/%s.flv' % video_id)
3013
3014                 mobj = re.search('<title>([^<]+)</title>', webpage)
3015                 if mobj is None:
3016                         self._downloader.trouble(u'ERROR: unable to extract title')
3017                         return
3018
3019                 video_title = mobj.group(1)
3020                 video_title = sanitize_title(video_title)
3021
3022                 try:
3023                         print(video_url)
3024                         self._downloader.process_info({
3025                                 'id':           video_id,
3026                                 'url':          video_url,
3027                                 'uploader':     u'NA',
3028                                 'upload_date':  u'NA',
3029                                 'title':        video_title,
3030                                 'stitle':       simple_title,
3031                                 'ext':          u'flv',
3032                                 'format':       u'NA',
3033                                 'player_url':   None,
3034                         })
3035                 except UnavailableVideoError:
3036                         self._downloader.trouble(u'\nERROR: Unable to download video')
3037
3038 class ComedyCentralIE(InfoExtractor):
3039         """Information extractor for The Daily Show and Colbert Report """
3040
3041         _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
3042
3043         @staticmethod
3044         def suitable(url):
3045                 return (re.match(ComedyCentralIE._VALID_URL, url) is not None)
3046
3047         def report_extraction(self, episode_id):
3048                 self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
3049
3050         def report_config_download(self, episode_id):
3051                 self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
3052
3053         def report_player_url(self, episode_id):
3054                 self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
3055
3056         def _simplify_title(self, title):
3057                 res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
3058                 res = res.strip(ur'_')
3059                 return res
3060
3061         def _real_extract(self, url):
3062                 mobj = re.match(self._VALID_URL, url)
3063                 if mobj is None:
3064                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3065                         return
3066
3067                 if mobj.group('shortname'):
3068                         if mobj.group('shortname') in ('tds', 'thedailyshow'):
3069                                 url = 'http://www.thedailyshow.com/full-episodes/'
3070                         else:
3071                                 url = 'http://www.colbertnation.com/full-episodes/'
3072                         mobj = re.match(self._VALID_URL, url)
3073                         assert mobj is not None
3074
3075                 dlNewest = not mobj.group('episode')
3076                 if dlNewest:
3077                         epTitle = mobj.group('showname')
3078                 else:
3079                         epTitle = mobj.group('episode')
3080
3081                 req = urllib2.Request(url)
3082                 self.report_extraction(epTitle)
3083                 try:
3084                         htmlHandle = urllib2.urlopen(req)
3085                         html = htmlHandle.read()
3086                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3087                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
3088                         return
3089                 if dlNewest:
3090                         url = htmlHandle.geturl()
3091                         mobj = re.match(self._VALID_URL, url)
3092                         if mobj is None:
3093                                 self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
3094                                 return
3095                         if mobj.group('episode') == '':
3096                                 self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
3097                                 return
3098                         epTitle = mobj.group('episode')
3099
3100                 mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/(.*?:episode:([^:]*):)(.*?))"/>', html)
3101                 if len(mMovieParams) == 0:
3102                         self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
3103                         return
3104                 show_id = mMovieParams[0][2]
3105                 ACT_COUNT = { # TODO: Detect this dynamically
3106                         'thedailyshow.com': 4,
3107                         'colbertnation.com': 3,
3108                 }.get(show_id, 4)
3109                 OFFSET = {
3110                         'thedailyshow.com': 1,
3111                         'colbertnation.com': 1,
3112                 }.get(show_id, 1)
3113
3114                 first_player_url = mMovieParams[0][0]
3115                 startMediaNum = int(mMovieParams[0][3]) + OFFSET
3116                 movieId = mMovieParams[0][1]
3117
3118                 playerReq = urllib2.Request(first_player_url)
3119                 self.report_player_url(epTitle)
3120                 try:
3121                         playerResponse = urllib2.urlopen(playerReq)
3122                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3123                         self._downloader.trouble(u'ERROR: unable to download player: %s' % unicode(err))
3124                         return
3125                 player_url = playerResponse.geturl()
3126
3127                 for actNum in range(ACT_COUNT):
3128                         mediaNum = startMediaNum + actNum
3129                         mediaId = movieId + str(mediaNum)
3130                         configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
3131                                                 urllib.urlencode({'uri': mediaId}))
3132                         configReq = urllib2.Request(configUrl)
3133                         self.report_config_download(epTitle)
3134                         try:
3135                                 configXml = urllib2.urlopen(configReq).read()
3136                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3137                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
3138                                 return
3139
3140                         cdoc = xml.etree.ElementTree.fromstring(configXml)
3141                         turls = []
3142                         for rendition in cdoc.findall('.//rendition'):
3143                                 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
3144                                 turls.append(finfo)
3145
3146                         if len(turls) == 0:
3147                                 self._downloader.trouble(u'\nERROR: unable to download ' + str(mediaNum) + ': No videos found')
3148                                 continue
3149
3150                         # For now, just pick the highest bitrate
3151                         format,video_url = turls[-1]
3152
3153                         self._downloader.increment_downloads()
3154
3155                         effTitle = show_id.replace('.com', '') + '-' + epTitle
3156                         info = {
3157                                 'id': str(mediaNum),
3158                                 'url': video_url,
3159                                 'uploader': show_id,
3160                                 'upload_date': 'NA',
3161                                 'title': effTitle,
3162                                 'stitle': self._simplify_title(effTitle),
3163                                 'ext': 'mp4',
3164                                 'format': format,
3165                                 'thumbnail': None,
3166                                 'description': 'TODO: Not yet supported',
3167                                 'player_url': player_url
3168                         }
3169
3170                         try:
3171                                 self._downloader.process_info(info)
3172                         except UnavailableVideoError, err:
3173                                 self._downloader.trouble(u'\nERROR: unable to download ' + str(mediaNum))
3174                                 continue
3175
3176
3177 class PostProcessor(object):
3178         """Post Processor class.
3179
3180         PostProcessor objects can be added to downloaders with their
3181         add_post_processor() method. When the downloader has finished a
3182         successful download, it will take its internal chain of PostProcessors
3183         and start calling the run() method on each one of them, first with
3184         an initial argument and then with the returned value of the previous
3185         PostProcessor.
3186
3187         The chain will be stopped if one of them ever returns None or the end
3188         of the chain is reached.
3189
3190         PostProcessor objects follow a "mutual registration" process similar
3191         to InfoExtractor objects.
3192         """
3193
3194         _downloader = None
3195
3196         def __init__(self, downloader=None):
3197                 self._downloader = downloader
3198
3199         def set_downloader(self, downloader):
3200                 """Sets the downloader for this PP."""
3201                 self._downloader = downloader
3202
3203         def run(self, information):
3204                 """Run the PostProcessor.
3205
3206                 The "information" argument is a dictionary like the ones
3207                 composed by InfoExtractors. The only difference is that this
3208                 one has an extra field called "filepath" that points to the
3209                 downloaded file.
3210
3211                 When this method returns None, the postprocessing chain is
3212                 stopped. However, this method may return an information
3213                 dictionary that will be passed to the next postprocessing
3214                 object in the chain. It can be the one it received after
3215                 changing some fields.
3216
3217                 In addition, this method may raise a PostProcessingError
3218                 exception that will be taken into account by the downloader
3219                 it was called from.
3220                 """
3221                 return information # by default, do nothing
3222
3223
3224 class FFmpegExtractAudioPP(PostProcessor):
3225
3226         def __init__(self, downloader=None, preferredcodec=None):
3227                 PostProcessor.__init__(self, downloader)
3228                 if preferredcodec is None:
3229                         preferredcodec = 'best'
3230                 self._preferredcodec = preferredcodec
3231
3232         @staticmethod
3233         def get_audio_codec(path):
3234                 try:
3235                         cmd = ['ffprobe', '-show_streams', '--', path]
3236                         handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
3237                         output = handle.communicate()[0]
3238                         if handle.wait() != 0:
3239                                 return None
3240                 except (IOError, OSError):
3241                         return None
3242                 audio_codec = None
3243                 for line in output.split('\n'):
3244                         if line.startswith('codec_name='):
3245                                 audio_codec = line.split('=')[1].strip()
3246                         elif line.strip() == 'codec_type=audio' and audio_codec is not None:
3247                                 return audio_codec
3248                 return None
3249
3250         @staticmethod
3251         def run_ffmpeg(path, out_path, codec, more_opts):
3252                 try:
3253                         cmd = ['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + ['--', out_path]
3254                         ret = subprocess.call(cmd, stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
3255                         return (ret == 0)
3256                 except (IOError, OSError):
3257                         return False
3258
3259         def run(self, information):
3260                 path = information['filepath']
3261
3262                 filecodec = self.get_audio_codec(path)
3263                 if filecodec is None:
3264                         self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
3265                         return None
3266
3267                 more_opts = []
3268                 if self._preferredcodec == 'best' or self._preferredcodec == filecodec:
3269                         if filecodec == 'aac' or filecodec == 'mp3':
3270                                 # Lossless if possible
3271                                 acodec = 'copy'
3272                                 extension = filecodec
3273                                 if filecodec == 'aac':
3274                                         more_opts = ['-f', 'adts']
3275                         else:
3276                                 # MP3 otherwise.
3277                                 acodec = 'libmp3lame'
3278                                 extension = 'mp3'
3279                                 more_opts = ['-ab', '128k']
3280                 else:
3281                         # We convert the audio (lossy)
3282                         acodec = {'mp3': 'libmp3lame', 'aac': 'aac'}[self._preferredcodec]
3283                         extension = self._preferredcodec
3284                         more_opts = ['-ab', '128k']
3285                         if self._preferredcodec == 'aac':
3286                                 more_opts += ['-f', 'adts']
3287
3288                 (prefix, ext) = os.path.splitext(path)
3289                 new_path = prefix + '.' + extension
3290                 self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path)
3291                 status = self.run_ffmpeg(path, new_path, acodec, more_opts)
3292
3293                 if not status:
3294                         self._downloader.to_stderr(u'WARNING: error running ffmpeg')
3295                         return None
3296
3297                 try:
3298                         os.remove(path)
3299                 except (IOError, OSError):
3300                         self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
3301                         return None
3302
3303                 information['filepath'] = new_path
3304                 return information
3305
3306
3307 def updateSelf(downloader, filename):
3308         ''' Update the program file with the latest version from the repository '''
3309         # Note: downloader only used for options
3310         if not os.access(filename, os.W_OK):
3311                 sys.exit('ERROR: no write permissions on %s' % filename)
3312
3313         downloader.to_screen('Updating to latest version...')
3314
3315         try:
3316                 try:
3317                         urlh = urllib.urlopen(UPDATE_URL)
3318                         newcontent = urlh.read()
3319                 finally:
3320                         urlh.close()
3321         except (IOError, OSError), err:
3322                 sys.exit('ERROR: unable to download latest version')
3323
3324         try:
3325                 outf = open(filename, 'wb')
3326                 try:
3327                         outf.write(newcontent)
3328                 finally:
3329                         outf.close()
3330         except (IOError, OSError), err:
3331                 sys.exit('ERROR: unable to overwrite current version')
3332
3333         downloader.to_screen('Updated youtube-dl. Restart to use the new version.')
3334
3335 def parseOpts():
3336         # Deferred imports
3337         import getpass
3338         import optparse
3339
3340         def _format_option_string(option):
3341                 ''' ('-o', '--option') -> -o, --format METAVAR'''
3342
3343                 opts = []
3344
3345                 if option._short_opts: opts.append(option._short_opts[0])
3346                 if option._long_opts: opts.append(option._long_opts[0])
3347                 if len(opts) > 1: opts.insert(1, ', ')
3348
3349                 if option.takes_value(): opts.append(' %s' % option.metavar)
3350
3351                 return "".join(opts)
3352
3353         def _find_term_columns():
3354                 columns = os.environ.get('COLUMNS', None)
3355                 if columns:
3356                         return int(columns)
3357
3358                 try:
3359                         sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
3360                         out,err = sp.communicate()
3361                         return int(out.split()[1])
3362                 except:
3363                         pass
3364                 return None
3365
3366         max_width = 80
3367         max_help_position = 80
3368
3369         # No need to wrap help messages if we're on a wide console
3370         columns = _find_term_columns()
3371         if columns: max_width = columns
3372
3373         fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
3374         fmt.format_option_strings = _format_option_string
3375
3376         kw = {
3377                 'version'   : __version__,
3378                 'formatter' : fmt,
3379                 'usage' : '%prog [options] url...',
3380                 'conflict_handler' : 'resolve',
3381         }
3382
3383         parser = optparse.OptionParser(**kw)
3384
3385         # option groups
3386         general        = optparse.OptionGroup(parser, 'General Options')
3387         authentication = optparse.OptionGroup(parser, 'Authentication Options')
3388         video_format   = optparse.OptionGroup(parser, 'Video Format Options')
3389         postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
3390         filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
3391         verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
3392
3393         general.add_option('-h', '--help',
3394                         action='help', help='print this help text and exit')
3395         general.add_option('-v', '--version',
3396                         action='version', help='print program version and exit')
3397         general.add_option('-U', '--update',
3398                         action='store_true', dest='update_self', help='update this program to latest version')
3399         general.add_option('-i', '--ignore-errors',
3400                         action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
3401         general.add_option('-r', '--rate-limit',
3402                         dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
3403         general.add_option('-R', '--retries',
3404                         dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
3405         general.add_option('--playlist-start',
3406                         dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
3407         general.add_option('--playlist-end',
3408                         dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
3409         general.add_option('--dump-user-agent',
3410                         action='store_true', dest='dump_user_agent',
3411                         help='display the current browser identification', default=False)
3412
3413         authentication.add_option('-u', '--username',
3414                         dest='username', metavar='USERNAME', help='account username')
3415         authentication.add_option('-p', '--password',
3416                         dest='password', metavar='PASSWORD', help='account password')
3417         authentication.add_option('-n', '--netrc',
3418                         action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
3419
3420
3421         video_format.add_option('-f', '--format',
3422                         action='store', dest='format', metavar='FORMAT', help='video format code')
3423         video_format.add_option('--all-formats',
3424                         action='store_const', dest='format', help='download all available video formats', const='-1')
3425         video_format.add_option('--max-quality',
3426                         action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
3427
3428
3429         verbosity.add_option('-q', '--quiet',
3430                         action='store_true', dest='quiet', help='activates quiet mode', default=False)
3431         verbosity.add_option('-s', '--simulate',
3432                         action='store_true', dest='simulate', help='do not download video', default=False)
3433         verbosity.add_option('-g', '--get-url',
3434                         action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
3435         verbosity.add_option('-e', '--get-title',
3436                         action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
3437         verbosity.add_option('--get-thumbnail',
3438                         action='store_true', dest='getthumbnail',
3439                         help='simulate, quiet but print thumbnail URL', default=False)
3440         verbosity.add_option('--get-description',
3441                         action='store_true', dest='getdescription',
3442                         help='simulate, quiet but print video description', default=False)
3443         verbosity.add_option('--get-filename',
3444                         action='store_true', dest='getfilename',
3445                         help='simulate, quiet but print output filename', default=False)
3446         verbosity.add_option('--no-progress',
3447                         action='store_true', dest='noprogress', help='do not print progress bar', default=False)
3448         verbosity.add_option('--console-title',
3449                         action='store_true', dest='consoletitle',
3450                         help='display progress in console titlebar', default=False)
3451
3452
3453         filesystem.add_option('-t', '--title',
3454                         action='store_true', dest='usetitle', help='use title in file name', default=False)
3455         filesystem.add_option('-l', '--literal',
3456                         action='store_true', dest='useliteral', help='use literal title in file name', default=False)
3457         filesystem.add_option('-A', '--auto-number',
3458                         action='store_true', dest='autonumber',
3459                         help='number downloaded files starting from 00000', default=False)
3460         filesystem.add_option('-o', '--output',
3461                         dest='outtmpl', metavar='TEMPLATE', help='output filename template')
3462         filesystem.add_option('-a', '--batch-file',
3463                         dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
3464         filesystem.add_option('-w', '--no-overwrites',
3465                         action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
3466         filesystem.add_option('-c', '--continue',
3467                         action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
3468         filesystem.add_option('--cookies',
3469                         dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
3470         filesystem.add_option('--no-part',
3471                         action='store_true', dest='nopart', help='do not use .part files', default=False)
3472         filesystem.add_option('--no-mtime',
3473                         action='store_false', dest='updatetime',
3474                         help='do not use the Last-modified header to set the file modification time', default=True)
3475         filesystem.add_option('--write-description',
3476                         action='store_true', dest='writedescription',
3477                         help='write video description to a .description file', default=False)
3478         filesystem.add_option('--write-info-json',
3479                         action='store_true', dest='writeinfojson',
3480                         help='write video metadata to a .info.json file', default=False)
3481
3482
3483         postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
3484                         help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
3485         postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
3486                         help='"best", "aac" or "mp3"; best by default')
3487
3488
3489         parser.add_option_group(general)
3490         parser.add_option_group(filesystem)
3491         parser.add_option_group(verbosity)
3492         parser.add_option_group(video_format)
3493         parser.add_option_group(authentication)
3494         parser.add_option_group(postproc)
3495
3496         opts, args = parser.parse_args()
3497
3498         return parser, opts, args
3499
3500 def main():
3501         parser, opts, args = parseOpts()
3502
3503         # Open appropriate CookieJar
3504         if opts.cookiefile is None:
3505                 jar = cookielib.CookieJar()
3506         else:
3507                 try:
3508                         jar = cookielib.MozillaCookieJar(opts.cookiefile)
3509                         if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
3510                                 jar.load()
3511                 except (IOError, OSError), err:
3512                         sys.exit(u'ERROR: unable to open cookie file')
3513
3514         # Dump user agent
3515         if opts.dump_user_agent:
3516                 print std_headers['User-Agent']
3517                 sys.exit(0)
3518
3519         # General configuration
3520         cookie_processor = urllib2.HTTPCookieProcessor(jar)
3521         opener = urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())
3522         urllib2.install_opener(opener)
3523         socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
3524
3525         # Batch file verification
3526         batchurls = []
3527         if opts.batchfile is not None:
3528                 try:
3529                         if opts.batchfile == '-':
3530                                 batchfd = sys.stdin
3531                         else:
3532                                 batchfd = open(opts.batchfile, 'r')
3533                         batchurls = batchfd.readlines()
3534                         batchurls = [x.strip() for x in batchurls]
3535                         batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
3536                 except IOError:
3537                         sys.exit(u'ERROR: batch file could not be read')
3538         all_urls = batchurls + args
3539
3540         # Conflicting, missing and erroneous options
3541         if opts.usenetrc and (opts.username is not None or opts.password is not None):
3542                 parser.error(u'using .netrc conflicts with giving username/password')
3543         if opts.password is not None and opts.username is None:
3544                 parser.error(u'account username missing')
3545         if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber):
3546                 parser.error(u'using output template conflicts with using title, literal title or auto number')
3547         if opts.usetitle and opts.useliteral:
3548                 parser.error(u'using title conflicts with using literal title')
3549         if opts.username is not None and opts.password is None:
3550                 opts.password = getpass.getpass(u'Type account password and press return:')
3551         if opts.ratelimit is not None:
3552                 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
3553                 if numeric_limit is None:
3554                         parser.error(u'invalid rate limit specified')
3555                 opts.ratelimit = numeric_limit
3556         if opts.retries is not None:
3557                 try:
3558                         opts.retries = long(opts.retries)
3559                 except (TypeError, ValueError), err:
3560                         parser.error(u'invalid retry count specified')
3561         try:
3562                 opts.playliststart = int(opts.playliststart)
3563                 if opts.playliststart <= 0:
3564                         raise ValueError(u'Playlist start must be positive')
3565         except (TypeError, ValueError), err:
3566                 parser.error(u'invalid playlist start number specified')
3567         try:
3568                 opts.playlistend = int(opts.playlistend)
3569                 if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
3570                         raise ValueError(u'Playlist end must be greater than playlist start')
3571         except (TypeError, ValueError), err:
3572                 parser.error(u'invalid playlist end number specified')
3573         if opts.extractaudio:
3574                 if opts.audioformat not in ['best', 'aac', 'mp3']:
3575                         parser.error(u'invalid audio format specified')
3576
3577         # Information extractors
3578         youtube_ie = YoutubeIE()
3579         metacafe_ie = MetacafeIE(youtube_ie)
3580         dailymotion_ie = DailymotionIE()
3581         youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
3582         youtube_user_ie = YoutubeUserIE(youtube_ie)
3583         youtube_search_ie = YoutubeSearchIE(youtube_ie)
3584         google_ie = GoogleIE()
3585         google_search_ie = GoogleSearchIE(google_ie)
3586         photobucket_ie = PhotobucketIE()
3587         yahoo_ie = YahooIE()
3588         yahoo_search_ie = YahooSearchIE(yahoo_ie)
3589         deposit_files_ie = DepositFilesIE()
3590         facebook_ie = FacebookIE()
3591         bliptv_ie = BlipTVIE()
3592         vimeo_ie = VimeoIE()
3593         myvideo_ie = MyVideoIE()
3594         comedycentral_ie = ComedyCentralIE()
3595
3596         generic_ie = GenericIE()
3597
3598         # File downloader
3599         fd = FileDownloader({
3600                 'usenetrc': opts.usenetrc,
3601                 'username': opts.username,
3602                 'password': opts.password,
3603                 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
3604                 'forceurl': opts.geturl,
3605                 'forcetitle': opts.gettitle,
3606                 'forcethumbnail': opts.getthumbnail,
3607                 'forcedescription': opts.getdescription,
3608                 'forcefilename': opts.getfilename,
3609                 'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
3610                 'format': opts.format,
3611                 'format_limit': opts.format_limit,
3612                 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
3613                         or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
3614                         or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
3615                         or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
3616                         or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
3617                         or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
3618                         or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
3619                         or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
3620                         or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
3621                         or u'%(id)s.%(ext)s'),
3622                 'ignoreerrors': opts.ignoreerrors,
3623                 'ratelimit': opts.ratelimit,
3624                 'nooverwrites': opts.nooverwrites,
3625                 'retries': opts.retries,
3626                 'continuedl': opts.continue_dl,
3627                 'noprogress': opts.noprogress,
3628                 'playliststart': opts.playliststart,
3629                 'playlistend': opts.playlistend,
3630                 'logtostderr': opts.outtmpl == '-',
3631                 'consoletitle': opts.consoletitle,
3632                 'nopart': opts.nopart,
3633                 'updatetime': opts.updatetime,
3634                 'writedescription': opts.writedescription,
3635                 'writeinfojson': opts.writeinfojson,
3636                 })
3637         fd.add_info_extractor(youtube_search_ie)
3638         fd.add_info_extractor(youtube_pl_ie)
3639         fd.add_info_extractor(youtube_user_ie)
3640         fd.add_info_extractor(metacafe_ie)
3641         fd.add_info_extractor(dailymotion_ie)
3642         fd.add_info_extractor(youtube_ie)
3643         fd.add_info_extractor(google_ie)
3644         fd.add_info_extractor(google_search_ie)
3645         fd.add_info_extractor(photobucket_ie)
3646         fd.add_info_extractor(yahoo_ie)
3647         fd.add_info_extractor(yahoo_search_ie)
3648         fd.add_info_extractor(deposit_files_ie)
3649         fd.add_info_extractor(facebook_ie)
3650         fd.add_info_extractor(bliptv_ie)
3651         fd.add_info_extractor(vimeo_ie)
3652         fd.add_info_extractor(myvideo_ie)
3653         fd.add_info_extractor(comedycentral_ie)
3654
3655         # This must come last since it's the
3656         # fallback if none of the others work
3657         fd.add_info_extractor(generic_ie)
3658
3659         # PostProcessors
3660         if opts.extractaudio:
3661                 fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat))
3662
3663         # Update version
3664         if opts.update_self:
3665                 updateSelf(fd, sys.argv[0])
3666
3667         # Maybe do nothing
3668         if len(all_urls) < 1:
3669                 if not opts.update_self:
3670                         parser.error(u'you must provide at least one URL')
3671                 else:
3672                         sys.exit()
3673         retcode = fd.download(all_urls)
3674
3675         # Dump cookie jar if requested
3676         if opts.cookiefile is not None:
3677                 try:
3678                         jar.save()
3679                 except (IOError, OSError), err:
3680                         sys.exit(u'ERROR: unable to save cookie jar')
3681
3682         sys.exit(retcode)
3683
3684
3685 if __name__ == '__main__':
3686         try:
3687                 main()
3688         except DownloadError:
3689                 sys.exit(1)
3690         except SameFileError:
3691                 sys.exit(u'ERROR: fixed output name but more than one file to download')
3692         except KeyboardInterrupt:
3693                 sys.exit(u'\nERROR: Interrupted by user')
3694
3695 # vim: set ts=4 sw=4 sts=4 noet ai si filetype=python: