youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __author__  = (
   5         'Ricardo Garcia Gonzalez',
   6         'Danny Colligan',
   7         'Benjamin Johnson',
   8         'Vasyl\' Vavrychuk',
   9         'Witold Baryluk',
  10         'Paweł Paprota',
  11         'Gergely Imreh',
  12         'Rogério Brito',
  13         'Philipp Hagemeister',
  14         'Sören Schulze',
  15         )
  16
  17 __license__ = 'Public Domain'
  18 __version__ = '2011.09.06-phihag'
  19
  20 UPDATE_URL = 'https://raw.github.com/phihag/youtube-dl/master/youtube-dl'
  21
  22 import cookielib
  23 import datetime
  24 import gzip
  25 import htmlentitydefs
  26 import httplib
  27 import locale
  28 import math
  29 import netrc
  30 import os
  31 import os.path
  32 import re
  33 import socket
  34 import string
  35 import subprocess
  36 import sys
  37 import time
  38 import urllib
  39 import urllib2
  40 import warnings
  41 import zlib
  42
  43 if os.name == 'nt':
  44         import ctypes
  45
  46 try:
  47         import email.utils
  48 except ImportError: # Python 2.4
  49         import email.Utils
  50 try:
  51         import cStringIO as StringIO
  52 except ImportError:
  53         import StringIO
  54
  55 # parse_qs was moved from the cgi module to the urlparse module recently.
  56 try:
  57         from urlparse import parse_qs
  58 except ImportError:
  59         from cgi import parse_qs
  60
  61 try:
  62         import lxml.etree
  63 except ImportError:
  64         pass # Handled below
  65
  66 try:
  67         import xml.etree.ElementTree
  68 except ImportError: # Python<2.5
  69         pass # Not officially supported, but let it slip
  70
  71 std_headers = {
  72         'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
  73         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  74         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  75         'Accept-Encoding': 'gzip, deflate',
  76         'Accept-Language': 'en-us,en;q=0.5',
  77 }
  78
  79 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  80
  81 try:
  82         import json
  83 except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
  84         import re
  85         class json(object):
  86                 @staticmethod
  87                 def loads(s):
  88                         s = s.decode('UTF-8')
  89                         def raiseError(msg, i):
  90                                 raise ValueError(msg + ' at position ' + str(i) + ' of ' + repr(s) + ': ' + repr(s[i:]))
  91                         def skipSpace(i, expectMore=True):
  92                                 while i < len(s) and s[i] in ' \t\r\n':
  93                                         i += 1
  94                                 if expectMore:
  95                                         if i >= len(s):
  96                                                 raiseError('Premature end', i)
  97                                 return i
  98                         def decodeEscape(match):
  99                                 esc = match.group(1)
 100                                 _STATIC = {
 101                                         '"': '"',
 102                                         '\\': '\\',
 103                                         '/': '/',
 104                                         'b': unichr(0x8),
 105                                         'f': unichr(0xc),
 106                                         'n': '\n',
 107                                         'r': '\r',
 108                                         't': '\t',
 109                                 }
 110                                 if esc in _STATIC:
 111                                         return _STATIC[esc]
 112                                 if esc[0] == 'u':
 113                                         if len(esc) == 1+4:
 114                                                 return unichr(int(esc[1:5], 16))
 115                                         if len(esc) == 5+6 and esc[5:7] == '\\u':
 116                                                 hi = int(esc[1:5], 16)
 117                                                 low = int(esc[7:11], 16)
 118                                                 return unichr((hi - 0xd800) * 0x400 + low - 0xdc00 + 0x10000)
 119                                 raise ValueError('Unknown escape ' + str(esc))
 120                         def parseString(i):
 121                                 i += 1
 122                                 e = i
 123                                 while True:
 124                                         e = s.index('"', e)
 125                                         bslashes = 0
 126                                         while s[e-bslashes-1] == '\\':
 127                                                 bslashes += 1
 128                                         if bslashes % 2 == 1:
 129                                                 e += 1
 130                                                 continue
 131                                         break
 132                                 rexp = re.compile(r'\\(u[dD][89aAbB][0-9a-fA-F]{2}\\u[0-9a-fA-F]{4}|u[0-9a-fA-F]{4}|.|$)')
 133                                 stri = rexp.sub(decodeEscape, s[i:e])
 134                                 return (e+1,stri)
 135                         def parseObj(i):
 136                                 i += 1
 137                                 res = {}
 138                                 i = skipSpace(i)
 139                                 if s[i] == '}': # Empty dictionary
 140                                         return (i+1,res)
 141                                 while True:
 142                                         if s[i] != '"':
 143                                                 raiseError('Expected a string object key', i)
 144                                         i,key = parseString(i)
 145                                         i = skipSpace(i)
 146                                         if i >= len(s) or s[i] != ':':
 147                                                 raiseError('Expected a colon', i)
 148                                         i,val = parse(i+1)
 149                                         res[key] = val
 150                                         i = skipSpace(i)
 151                                         if s[i] == '}':
 152                                                 return (i+1, res)
 153                                         if s[i] != ',':
 154                                                 raiseError('Expected comma or closing curly brace', i)
 155                                         i = skipSpace(i+1)
 156                         def parseArray(i):
 157                                 res = []
 158                                 i = skipSpace(i+1)
 159                                 if s[i] == ']': # Empty array
 160                                         return (i+1,res)
 161                                 while True:
 162                                         i,val = parse(i)
 163                                         res.append(val)
 164                                         i = skipSpace(i) # Raise exception if premature end
 165                                         if s[i] == ']':
 166                                                 return (i+1, res)
 167                                         if s[i] != ',':
 168                                                 raiseError('Expected a comma or closing bracket', i)
 169                                         i = skipSpace(i+1)
 170                         def parseDiscrete(i):
 171                                 for k,v in {'true': True, 'false': False, 'null': None}.items():
 172                                         if s.startswith(k, i):
 173                                                 return (i+len(k), v)
 174                                 raiseError('Not a boolean (or null)', i)
 175                         def parseNumber(i):
 176                                 mobj = re.match('^(-?(0|[1-9][0-9]*)(\.[0-9]*)?([eE][+-]?[0-9]+)?)', s[i:])
 177                                 if mobj is None:
 178                                         raiseError('Not a number', i)
 179                                 nums = mobj.group(1)
 180                                 if '.' in nums or 'e' in nums or 'E' in nums:
 181                                         return (i+len(nums), float(nums))
 182                                 return (i+len(nums), int(nums))
 183                         CHARMAP = {'{': parseObj, '[': parseArray, '"': parseString, 't': parseDiscrete, 'f': parseDiscrete, 'n': parseDiscrete}
 184                         def parse(i):
 185                                 i = skipSpace(i)
 186                                 i,res = CHARMAP.get(s[i], parseNumber)(i)
 187                                 i = skipSpace(i, False)
 188                                 return (i,res)
 189                         i,res = parse(0)
 190                         if i < len(s):
 191                                 raise ValueError('Extra data at end of input (index ' + str(i) + ' of ' + repr(s) + ': ' + repr(s[i:]) + ')')
 192                         return res
 193
 194 def preferredencoding():
 195         """Get preferred encoding.
 196
 197         Returns the best encoding scheme for the system, based on
 198         locale.getpreferredencoding() and some further tweaks.
 199         """
 200         def yield_preferredencoding():
 201                 try:
 202                         pref = locale.getpreferredencoding()
 203                         u'TEST'.encode(pref)
 204                 except:
 205                         pref = 'UTF-8'
 206                 while True:
 207                         yield pref
 208         return yield_preferredencoding().next()
 209
 210
 211 def htmlentity_transform(matchobj):
 212         """Transforms an HTML entity to a Unicode character.
 213
 214         This function receives a match object and is intended to be used with
 215         the re.sub() function.
 216         """
 217         entity = matchobj.group(1)
 218
 219         # Known non-numeric HTML entity
 220         if entity in htmlentitydefs.name2codepoint:
 221                 return unichr(htmlentitydefs.name2codepoint[entity])
 222
 223         # Unicode character
 224         mobj = re.match(ur'(?u)#(x?\d+)', entity)
 225         if mobj is not None:
 226                 numstr = mobj.group(1)
 227                 if numstr.startswith(u'x'):
 228                         base = 16
 229                         numstr = u'0%s' % numstr
 230                 else:
 231                         base = 10
 232                 return unichr(long(numstr, base))
 233
 234         # Unknown entity in name, return its literal representation
 235         return (u'&%s;' % entity)
 236
 237
 238 def sanitize_title(utitle):
 239         """Sanitizes a video title so it could be used as part of a filename."""
 240         utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
 241         return utitle.replace(unicode(os.sep), u'%')
 242
 243
 244 def sanitize_open(filename, open_mode):
 245         """Try to open the given filename, and slightly tweak it if this fails.
 246
 247         Attempts to open the given filename. If this fails, it tries to change
 248         the filename slightly, step by step, until it's either able to open it
 249         or it fails and raises a final exception, like the standard open()
 250         function.
 251
 252         It returns the tuple (stream, definitive_file_name).
 253         """
 254         try:
 255                 if filename == u'-':
 256                         if sys.platform == 'win32':
 257                                 import msvcrt
 258                                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 259                         return (sys.stdout, filename)
 260                 stream = open(filename, open_mode)
 261                 return (stream, filename)
 262         except (IOError, OSError), err:
 263                 # In case of error, try to remove win32 forbidden chars
 264                 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 265
 266                 # An exception here should be caught in the caller
 267                 stream = open(filename, open_mode)
 268                 return (stream, filename)
 269
 270
 271 def timeconvert(timestr):
 272         """Convert RFC 2822 defined time string into system timestamp"""
 273         timestamp = None
 274         timetuple = email.utils.parsedate_tz(timestr)
 275         if timetuple is not None:
 276                 timestamp = email.utils.mktime_tz(timetuple)
 277         return timestamp
 278
 279
 280 class DownloadError(Exception):
 281         """Download Error exception.
 282
 283         This exception may be thrown by FileDownloader objects if they are not
 284         configured to continue on errors. They will contain the appropriate
 285         error message.
 286         """
 287         pass
 288
 289
 290 class SameFileError(Exception):
 291         """Same File exception.
 292
 293         This exception will be thrown by FileDownloader objects if they detect
 294         multiple files would have to be downloaded to the same file on disk.
 295         """
 296         pass
 297
 298
 299 class PostProcessingError(Exception):
 300         """Post Processing exception.
 301
 302         This exception may be raised by PostProcessor's .run() method to
 303         indicate an error in the postprocessing task.
 304         """
 305         pass
 306
 307
 308 class UnavailableVideoError(Exception):
 309         """Unavailable Format exception.
 310
 311         This exception will be thrown when a video is requested
 312         in a format that is not available for that video.
 313         """
 314         pass
 315
 316
 317 class ContentTooShortError(Exception):
 318         """Content Too Short exception.
 319
 320         This exception may be raised by FileDownloader objects when a file they
 321         download is too small for what the server announced first, indicating
 322         the connection was probably interrupted.
 323         """
 324         # Both in bytes
 325         downloaded = None
 326         expected = None
 327
 328         def __init__(self, downloaded, expected):
 329                 self.downloaded = downloaded
 330                 self.expected = expected
 331
 332
 333 class YoutubeDLHandler(urllib2.HTTPHandler):
 334         """Handler for HTTP requests and responses.
 335
 336         This class, when installed with an OpenerDirector, automatically adds
 337         the standard headers to every HTTP request and handles gzipped and
 338         deflated responses from web servers. If compression is to be avoided in
 339         a particular request, the original request in the program code only has
 340         to include the HTTP header "Youtubedl-No-Compression", which will be
 341         removed before making the real request.
 342
 343         Part of this code was copied from:
 344
 345         http://techknack.net/python-urllib2-handlers/
 346
 347         Andrew Rowls, the author of that code, agreed to release it to the
 348         public domain.
 349         """
 350
 351         @staticmethod
 352         def deflate(data):
 353                 try:
 354                         return zlib.decompress(data, -zlib.MAX_WBITS)
 355                 except zlib.error:
 356                         return zlib.decompress(data)
 357
 358         @staticmethod
 359         def addinfourl_wrapper(stream, headers, url, code):
 360                 if hasattr(urllib2.addinfourl, 'getcode'):
 361                         return urllib2.addinfourl(stream, headers, url, code)
 362                 ret = urllib2.addinfourl(stream, headers, url)
 363                 ret.code = code
 364                 return ret
 365
 366         def http_request(self, req):
 367                 for h in std_headers:
 368                         if h in req.headers:
 369                                 del req.headers[h]
 370                         req.add_header(h, std_headers[h])
 371                 if 'Youtubedl-no-compression' in req.headers:
 372                         if 'Accept-encoding' in req.headers:
 373                                 del req.headers['Accept-encoding']
 374                         del req.headers['Youtubedl-no-compression']
 375                 return req
 376
 377         def http_response(self, req, resp):
 378                 old_resp = resp
 379                 # gzip
 380                 if resp.headers.get('Content-encoding', '') == 'gzip':
 381                         gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
 382                         resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 383                         resp.msg = old_resp.msg
 384                 # deflate
 385                 if resp.headers.get('Content-encoding', '') == 'deflate':
 386                         gz = StringIO.StringIO(self.deflate(resp.read()))
 387                         resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 388                         resp.msg = old_resp.msg
 389                 return resp
 390
 391
 392 class FileDownloader(object):
 393         """File Downloader class.
 394
 395         File downloader objects are the ones responsible of downloading the
 396         actual video file and writing it to disk if the user has requested
 397         it, among some other tasks. In most cases there should be one per
 398         program. As, given a video URL, the downloader doesn't know how to
 399         extract all the needed information, task that InfoExtractors do, it
 400         has to pass the URL to one of them.
 401
 402         For this, file downloader objects have a method that allows
 403         InfoExtractors to be registered in a given order. When it is passed
 404         a URL, the file downloader handles it to the first InfoExtractor it
 405         finds that reports being able to handle it. The InfoExtractor extracts
 406         all the information about the video or videos the URL refers to, and
 407         asks the FileDownloader to process the video information, possibly
 408         downloading the video.
 409
 410         File downloaders accept a lot of parameters. In order not to saturate
 411         the object constructor with arguments, it receives a dictionary of
 412         options instead. These options are available through the params
 413         attribute for the InfoExtractors to use. The FileDownloader also
 414         registers itself as the downloader in charge for the InfoExtractors
 415         that are added to it, so this is a "mutual registration".
 416
 417         Available options:
 418
 419         username:         Username for authentication purposes.
 420         password:         Password for authentication purposes.
 421         usenetrc:         Use netrc for authentication instead.
 422         quiet:            Do not print messages to stdout.
 423         forceurl:         Force printing final URL.
 424         forcetitle:       Force printing title.
 425         forcethumbnail:   Force printing thumbnail URL.
 426         forcedescription: Force printing description.
 427         forcefilename:    Force printing final filename.
 428         simulate:         Do not download the video files.
 429         format:           Video format code.
 430         format_limit:     Highest quality format to try.
 431         outtmpl:          Template for output names.
 432         ignoreerrors:     Do not stop on download errors.
 433         ratelimit:        Download speed limit, in bytes/sec.
 434         nooverwrites:     Prevent overwriting files.
 435         retries:          Number of times to retry for HTTP error 5xx
 436         continuedl:       Try to continue downloads if possible.
 437         noprogress:       Do not print the progress bar.
 438         playliststart:    Playlist item to start at.
 439         playlistend:      Playlist item to end at.
 440         logtostderr:      Log messages to stderr instead of stdout.
 441         consoletitle:     Display progress in console window's titlebar.
 442         nopart:           Do not use temporary .part files.
 443         updatetime:       Use the Last-modified header to set output file timestamps.
 444         writedescription: Write the video description to a .description file
 445         writeinfojson:    Write the video description to a .info.json file
 446         """
 447
 448         params = None
 449         _ies = []
 450         _pps = []
 451         _download_retcode = None
 452         _num_downloads = None
 453         _screen_file = None
 454
 455         def __init__(self, params):
 456                 """Create a FileDownloader object with the given options."""
 457                 self._ies = []
 458                 self._pps = []
 459                 self._download_retcode = 0
 460                 self._num_downloads = 0
 461                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 462                 self.params = params
 463
 464         @staticmethod
 465         def format_bytes(bytes):
 466                 if bytes is None:
 467                         return 'N/A'
 468                 if type(bytes) is str:
 469                         bytes = float(bytes)
 470                 if bytes == 0.0:
 471                         exponent = 0
 472                 else:
 473                         exponent = long(math.log(bytes, 1024.0))
 474                 suffix = 'bkMGTPEZY'[exponent]
 475                 converted = float(bytes) / float(1024 ** exponent)
 476                 return '%.2f%s' % (converted, suffix)
 477
 478         @staticmethod
 479         def calc_percent(byte_counter, data_len):
 480                 if data_len is None:
 481                         return '---.-%'
 482                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 483
 484         @staticmethod
 485         def calc_eta(start, now, total, current):
 486                 if total is None:
 487                         return '--:--'
 488                 dif = now - start
 489                 if current == 0 or dif < 0.001: # One millisecond
 490                         return '--:--'
 491                 rate = float(current) / dif
 492                 eta = long((float(total) - float(current)) / rate)
 493                 (eta_mins, eta_secs) = divmod(eta, 60)
 494                 if eta_mins > 99:
 495                         return '--:--'
 496                 return '%02d:%02d' % (eta_mins, eta_secs)
 497
 498         @staticmethod
 499         def calc_speed(start, now, bytes):
 500                 dif = now - start
 501                 if bytes == 0 or dif < 0.001: # One millisecond
 502                         return '%10s' % '---b/s'
 503                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 504
 505         @staticmethod
 506         def best_block_size(elapsed_time, bytes):
 507                 new_min = max(bytes / 2.0, 1.0)
 508                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 509                 if elapsed_time < 0.001:
 510                         return long(new_max)
 511                 rate = bytes / elapsed_time
 512                 if rate > new_max:
 513                         return long(new_max)
 514                 if rate < new_min:
 515                         return long(new_min)
 516                 return long(rate)
 517
 518         @staticmethod
 519         def parse_bytes(bytestr):
 520                 """Parse a string indicating a byte quantity into a long integer."""
 521                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 522                 if matchobj is None:
 523                         return None
 524                 number = float(matchobj.group(1))
 525                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 526                 return long(round(number * multiplier))
 527
 528         def add_info_extractor(self, ie):
 529                 """Add an InfoExtractor object to the end of the list."""
 530                 self._ies.append(ie)
 531                 ie.set_downloader(self)
 532
 533         def add_post_processor(self, pp):
 534                 """Add a PostProcessor object to the end of the chain."""
 535                 self._pps.append(pp)
 536                 pp.set_downloader(self)
 537
 538         def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
 539                 """Print message to stdout if not in quiet mode."""
 540                 try:
 541                         if not self.params.get('quiet', False):
 542                                 terminator = [u'\n', u''][skip_eol]
 543                                 print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
 544                         self._screen_file.flush()
 545                 except (UnicodeEncodeError), err:
 546                         if not ignore_encoding_errors:
 547                                 raise
 548
 549         def to_stderr(self, message):
 550                 """Print message to stderr."""
 551                 print >>sys.stderr, message.encode(preferredencoding())
 552
 553         def to_cons_title(self, message):
 554                 """Set console/terminal window title to message."""
 555                 if not self.params.get('consoletitle', False):
 556                         return
 557                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 558                         # c_wchar_p() might not be necessary if `message` is
 559                         # already of type unicode()
 560                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 561                 elif 'TERM' in os.environ:
 562                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 563
 564         def fixed_template(self):
 565                 """Checks if the output template is fixed."""
 566                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 567
 568         def trouble(self, message=None):
 569                 """Determine action to take when a download problem appears.
 570
 571                 Depending on if the downloader has been configured to ignore
 572                 download errors or not, this method may throw an exception or
 573                 not when errors are found, after printing the message.
 574                 """
 575                 if message is not None:
 576                         self.to_stderr(message)
 577                 if not self.params.get('ignoreerrors', False):
 578                         raise DownloadError(message)
 579                 self._download_retcode = 1
 580
 581         def slow_down(self, start_time, byte_counter):
 582                 """Sleep if the download speed is over the rate limit."""
 583                 rate_limit = self.params.get('ratelimit', None)
 584                 if rate_limit is None or byte_counter == 0:
 585                         return
 586                 now = time.time()
 587                 elapsed = now - start_time
 588                 if elapsed <= 0.0:
 589                         return
 590                 speed = float(byte_counter) / elapsed
 591                 if speed > rate_limit:
 592                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 593
 594         def temp_name(self, filename):
 595                 """Returns a temporary filename for the given filename."""
 596                 if self.params.get('nopart', False) or filename == u'-' or \
 597                                 (os.path.exists(filename) and not os.path.isfile(filename)):
 598                         return filename
 599                 return filename + u'.part'
 600
 601         def undo_temp_name(self, filename):
 602                 if filename.endswith(u'.part'):
 603                         return filename[:-len(u'.part')]
 604                 return filename
 605
 606         def try_rename(self, old_filename, new_filename):
 607                 try:
 608                         if old_filename == new_filename:
 609                                 return
 610                         os.rename(old_filename, new_filename)
 611                 except (IOError, OSError), err:
 612                         self.trouble(u'ERROR: unable to rename file')
 613
 614         def try_utime(self, filename, last_modified_hdr):
 615                 """Try to set the last-modified time of the given file."""
 616                 if last_modified_hdr is None:
 617                         return
 618                 if not os.path.isfile(filename):
 619                         return
 620                 timestr = last_modified_hdr
 621                 if timestr is None:
 622                         return
 623                 filetime = timeconvert(timestr)
 624                 if filetime is None:
 625                         return
 626                 try:
 627                         os.utime(filename, (time.time(), filetime))
 628                 except:
 629                         pass
 630
 631         def report_writedescription(self, descfn):
 632                 """ Report that the description file is being written """
 633                 self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True)
 634
 635         def report_writeinfojson(self, infofn):
 636                 """ Report that the metadata file has been written """
 637                 self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True)
 638
 639         def report_destination(self, filename):
 640                 """Report destination filename."""
 641                 self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
 642
 643         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 644                 """Report download progress."""
 645                 if self.params.get('noprogress', False):
 646                         return
 647                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 648                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 649                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 650                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 651
 652         def report_resuming_byte(self, resume_len):
 653                 """Report attempt to resume at given byte."""
 654                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 655
 656         def report_retry(self, count, retries):
 657                 """Report retry in case of HTTP error 5xx"""
 658                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 659
 660         def report_file_already_downloaded(self, file_name):
 661                 """Report file has already been fully downloaded."""
 662                 try:
 663                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 664                 except (UnicodeEncodeError), err:
 665                         self.to_screen(u'[download] The file has already been downloaded')
 666
 667         def report_unable_to_resume(self):
 668                 """Report it was impossible to resume download."""
 669                 self.to_screen(u'[download] Unable to resume')
 670
 671         def report_finish(self):
 672                 """Report download finished."""
 673                 if self.params.get('noprogress', False):
 674                         self.to_screen(u'[download] Download completed')
 675                 else:
 676                         self.to_screen(u'')
 677
 678         def increment_downloads(self):
 679                 """Increment the ordinal that assigns a number to each file."""
 680                 self._num_downloads += 1
 681
 682         def prepare_filename(self, info_dict):
 683                 """Generate the output filename."""
 684                 try:
 685                         template_dict = dict(info_dict)
 686                         template_dict['epoch'] = unicode(long(time.time()))
 687                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 688                         filename = self.params['outtmpl'] % template_dict
 689                         return filename
 690                 except (ValueError, KeyError), err:
 691                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 692                         return None
 693
 694         def process_info(self, info_dict):
 695                 """Process a single dictionary returned by an InfoExtractor."""
 696                 filename = self.prepare_filename(info_dict)
 697                 # Do nothing else if in simulate mode
 698                 if self.params.get('simulate', False):
 699                         # Forced printings
 700                         if self.params.get('forcetitle', False):
 701                                 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
 702                         if self.params.get('forceurl', False):
 703                                 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
 704                         if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 705                                 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
 706                         if self.params.get('forcedescription', False) and 'description' in info_dict:
 707                                 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
 708                         if self.params.get('forcefilename', False) and filename is not None:
 709                                 print filename.encode(preferredencoding(), 'xmlcharrefreplace')
 710
 711                         return
 712
 713                 if filename is None:
 714                         return
 715                 if self.params.get('nooverwrites', False) and os.path.exists(filename):
 716                         self.to_stderr(u'WARNING: file exists and will be skipped')
 717                         return
 718
 719                 try:
 720                         dn = os.path.dirname(filename)
 721                         if dn != '' and not os.path.exists(dn):
 722                                 os.makedirs(dn)
 723                 except (OSError, IOError), err:
 724                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 725                         return
 726
 727                 if self.params.get('writedescription', False):
 728                         try:
 729                                 descfn = filename + '.description'
 730                                 self.report_writedescription(descfn)
 731                                 descfile = open(descfn, 'wb')
 732                                 try:
 733                                         descfile.write(info_dict['description'].encode('utf-8'))
 734                                 finally:
 735                                         descfile.close()
 736                         except (OSError, IOError):
 737                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 738                                 return
 739
 740                 if self.params.get('writeinfojson', False):
 741                         infofn = filename + '.info.json'
 742                         self.report_writeinfojson(infofn)
 743                         try:
 744                                 json.dump
 745                         except (NameError,AttributeError):
 746                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 747                                 return
 748                         try:
 749                                 infof = open(infofn, 'wb')
 750                                 try:
 751                                         json.dump(info_dict, infof)
 752                                 finally:
 753                                         infof.close()
 754                         except (OSError, IOError):
 755                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 756                                 return
 757
 758                 try:
 759                         success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
 760                 except (OSError, IOError), err:
 761                         raise UnavailableVideoError
 762                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 763                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 764                         return
 765                 except (ContentTooShortError, ), err:
 766                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 767                         return
 768
 769                 if success:
 770                         try:
 771                                 self.post_process(filename, info_dict)
 772                         except (PostProcessingError), err:
 773                                 self.trouble(u'ERROR: postprocessing: %s' % str(err))
 774                                 return
 775
 776         def download(self, url_list):
 777                 """Download a given list of URLs."""
 778                 if len(url_list) > 1 and self.fixed_template():
 779                         raise SameFileError(self.params['outtmpl'])
 780
 781                 for url in url_list:
 782                         suitable_found = False
 783                         for ie in self._ies:
 784                                 # Go to next InfoExtractor if not suitable
 785                                 if not ie.suitable(url):
 786                                         continue
 787
 788                                 # Suitable InfoExtractor found
 789                                 suitable_found = True
 790
 791                                 # Extract information from URL and process it
 792                                 ie.extract(url)
 793
 794                                 # Suitable InfoExtractor had been found; go to next URL
 795                                 break
 796
 797                         if not suitable_found:
 798                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 799
 800                 return self._download_retcode
 801
 802         def post_process(self, filename, ie_info):
 803                 """Run the postprocessing chain on the given file."""
 804                 info = dict(ie_info)
 805                 info['filepath'] = filename
 806                 for pp in self._pps:
 807                         info = pp.run(info)
 808                         if info is None:
 809                                 break
 810
 811         def _download_with_rtmpdump(self, filename, url, player_url):
 812                 self.report_destination(filename)
 813                 tmpfilename = self.temp_name(filename)
 814
 815                 # Check for rtmpdump first
 816                 try:
 817                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 818                 except (OSError, IOError):
 819                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 820                         return False
 821
 822                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 823                 # the connection was interrumpted and resuming appears to be
 824                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 825                 basic_args = ['rtmpdump'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 826                 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
 827                 while retval == 2 or retval == 1:
 828                         prevsize = os.path.getsize(tmpfilename)
 829                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 830                         time.sleep(5.0) # This seems to be needed
 831                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 832                         cursize = os.path.getsize(tmpfilename)
 833                         if prevsize == cursize and retval == 1:
 834                                 break
 835                 if retval == 0:
 836                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
 837                         self.try_rename(tmpfilename, filename)
 838                         return True
 839                 else:
 840                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 841                         return False
 842
 843         def _do_download(self, filename, url, player_url):
 844                 # Check file already present
 845                 if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
 846                         self.report_file_already_downloaded(filename)
 847                         return True
 848
 849                 # Attempt to download using rtmpdump
 850                 if url.startswith('rtmp'):
 851                         return self._download_with_rtmpdump(filename, url, player_url)
 852
 853                 tmpfilename = self.temp_name(filename)
 854                 stream = None
 855                 open_mode = 'wb'
 856
 857                 # Do not include the Accept-Encoding header
 858                 headers = {'Youtubedl-no-compression': 'True'}
 859                 basic_request = urllib2.Request(url, None, headers)
 860                 request = urllib2.Request(url, None, headers)
 861
 862                 # Establish possible resume length
 863                 if os.path.isfile(tmpfilename):
 864                         resume_len = os.path.getsize(tmpfilename)
 865                 else:
 866                         resume_len = 0
 867
 868                 # Request parameters in case of being able to resume
 869                 if self.params.get('continuedl', False) and resume_len != 0:
 870                         self.report_resuming_byte(resume_len)
 871                         request.add_header('Range', 'bytes=%d-' % resume_len)
 872                         open_mode = 'ab'
 873
 874                 count = 0
 875                 retries = self.params.get('retries', 0)
 876                 while count <= retries:
 877                         # Establish connection
 878                         try:
 879                                 data = urllib2.urlopen(request)
 880                                 break
 881                         except (urllib2.HTTPError, ), err:
 882                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 883                                         # Unexpected HTTP error
 884                                         raise
 885                                 elif err.code == 416:
 886                                         # Unable to resume (requested range not satisfiable)
 887                                         try:
 888                                                 # Open the connection again without the range header
 889                                                 data = urllib2.urlopen(basic_request)
 890                                                 content_length = data.info()['Content-Length']
 891                                         except (urllib2.HTTPError, ), err:
 892                                                 if err.code < 500 or err.code >= 600:
 893                                                         raise
 894                                         else:
 895                                                 # Examine the reported length
 896                                                 if (content_length is not None and
 897                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 898                                                         # The file had already been fully downloaded.
 899                                                         # Explanation to the above condition: in issue #175 it was revealed that
 900                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 901                                                         # changing the file size slightly and causing problems for some users. So
 902                                                         # I decided to implement a suggested change and consider the file
 903                                                         # completely downloaded if the file size differs less than 100 bytes from
 904                                                         # the one in the hard drive.
 905                                                         self.report_file_already_downloaded(filename)
 906                                                         self.try_rename(tmpfilename, filename)
 907                                                         return True
 908                                                 else:
 909                                                         # The length does not match, we start the download over
 910                                                         self.report_unable_to_resume()
 911                                                         open_mode = 'wb'
 912                                                         break
 913                         # Retry
 914                         count += 1
 915                         if count <= retries:
 916                                 self.report_retry(count, retries)
 917
 918                 if count > retries:
 919                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 920                         return False
 921
 922                 data_len = data.info().get('Content-length', None)
 923                 if data_len is not None:
 924                         data_len = long(data_len) + resume_len
 925                 data_len_str = self.format_bytes(data_len)
 926                 byte_counter = 0 + resume_len
 927                 block_size = 1024
 928                 start = time.time()
 929                 while True:
 930                         # Download and write
 931                         before = time.time()
 932                         data_block = data.read(block_size)
 933                         after = time.time()
 934                         if len(data_block) == 0:
 935                                 break
 936                         byte_counter += len(data_block)
 937
 938                         # Open file just in time
 939                         if stream is None:
 940                                 try:
 941                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 942                                         assert stream is not None
 943                                         filename = self.undo_temp_name(tmpfilename)
 944                                         self.report_destination(filename)
 945                                 except (OSError, IOError), err:
 946                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 947                                         return False
 948                         try:
 949                                 stream.write(data_block)
 950                         except (IOError, OSError), err:
 951                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 952                                 return False
 953                         block_size = self.best_block_size(after - before, len(data_block))
 954
 955                         # Progress message
 956                         percent_str = self.calc_percent(byte_counter, data_len)
 957                         eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 958                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 959                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 960
 961                         # Apply rate limit
 962                         self.slow_down(start, byte_counter - resume_len)
 963
 964                 if stream is None:
 965                         self.trouble(u'\nERROR: Did not get any data blocks')
 966                         return False
 967                 stream.close()
 968                 self.report_finish()
 969                 if data_len is not None and byte_counter != data_len:
 970                         raise ContentTooShortError(byte_counter, long(data_len))
 971                 self.try_rename(tmpfilename, filename)
 972
 973                 # Update file modification time
 974                 if self.params.get('updatetime', True):
 975                         self.try_utime(filename, data.info().get('last-modified', None))
 976
 977                 return True
 978
 979
 980 class InfoExtractor(object):
 981         """Information Extractor class.
 982
 983         Information extractors are the classes that, given a URL, extract
 984         information from the video (or videos) the URL refers to. This
 985         information includes the real video URL, the video title and simplified
 986         title, author and others. The information is stored in a dictionary
 987         which is then passed to the FileDownloader. The FileDownloader
 988         processes this information possibly downloading the video to the file
 989         system, among other possible outcomes. The dictionaries must include
 990         the following fields:
 991
 992         id:             Video identifier.
 993         url:            Final video URL.
 994         uploader:       Nickname of the video uploader.
 995         title:          Literal title.
 996         stitle:         Simplified title.
 997         ext:            Video filename extension.
 998         format:         Video format.
 999         player_url:     SWF Player URL (may be None).
1000
1001         The following fields are optional. Their primary purpose is to allow
1002         youtube-dl to serve as the backend for a video search function, such
1003         as the one in youtube2mp3.  They are only used when their respective
1004         forced printing functions are called:
1005
1006         thumbnail:      Full URL to a video thumbnail image.
1007         description:    One-line video description.
1008
1009         Subclasses of this one should re-define the _real_initialize() and
1010         _real_extract() methods, as well as the suitable() static method.
1011         Probably, they should also be instantiated and added to the main
1012         downloader.
1013         """
1014
1015         _ready = False
1016         _downloader = None
1017
1018         def __init__(self, downloader=None):
1019                 """Constructor. Receives an optional downloader."""
1020                 self._ready = False
1021                 self.set_downloader(downloader)
1022
1023         @staticmethod
1024         def suitable(url):
1025                 """Receives a URL and returns True if suitable for this IE."""
1026                 return False
1027
1028         def initialize(self):
1029                 """Initializes an instance (authentication, etc)."""
1030                 if not self._ready:
1031                         self._real_initialize()
1032                         self._ready = True
1033
1034         def extract(self, url):
1035                 """Extracts URL information and returns it in list of dicts."""
1036                 self.initialize()
1037                 return self._real_extract(url)
1038
1039         def set_downloader(self, downloader):
1040                 """Sets the downloader for this IE."""
1041                 self._downloader = downloader
1042
1043         def _real_initialize(self):
1044                 """Real initialization process. Redefine in subclasses."""
1045                 pass
1046
1047         def _real_extract(self, url):
1048                 """Real extraction process. Redefine in subclasses."""
1049                 pass
1050
1051
1052 class YoutubeIE(InfoExtractor):
1053         """Information extractor for youtube.com."""
1054
1055         _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
1056         _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
1057         _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
1058         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
1059         _NETRC_MACHINE = 'youtube'
1060         # Listed in order of quality
1061         _available_formats = ['38', '37', '45', '22', '43', '35', '34', '18', '6', '5', '17', '13']
1062         _video_extensions = {
1063                 '13': '3gp',
1064                 '17': 'mp4',
1065                 '18': 'mp4',
1066                 '22': 'mp4',
1067                 '37': 'mp4',
1068                 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
1069                 '43': 'webm',
1070                 '45': 'webm',
1071         }
1072
1073         @staticmethod
1074         def suitable(url):
1075                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
1076
1077         def report_lang(self):
1078                 """Report attempt to set language."""
1079                 self._downloader.to_screen(u'[youtube] Setting language')
1080
1081         def report_login(self):
1082                 """Report attempt to log in."""
1083                 self._downloader.to_screen(u'[youtube] Logging in')
1084
1085         def report_age_confirmation(self):
1086                 """Report attempt to confirm age."""
1087                 self._downloader.to_screen(u'[youtube] Confirming age')
1088
1089         def report_video_webpage_download(self, video_id):
1090                 """Report attempt to download video webpage."""
1091                 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
1092
1093         def report_video_info_webpage_download(self, video_id):
1094                 """Report attempt to download video info webpage."""
1095                 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
1096
1097         def report_information_extraction(self, video_id):
1098                 """Report attempt to extract video information."""
1099                 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
1100
1101         def report_unavailable_format(self, video_id, format):
1102                 """Report extracted video URL."""
1103                 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
1104
1105         def report_rtmp_download(self):
1106                 """Indicate the download will use the RTMP protocol."""
1107                 self._downloader.to_screen(u'[youtube] RTMP download detected')
1108
1109         def _real_initialize(self):
1110                 if self._downloader is None:
1111                         return
1112
1113                 username = None
1114                 password = None
1115                 downloader_params = self._downloader.params
1116
1117                 # Attempt to use provided username and password or .netrc data
1118                 if downloader_params.get('username', None) is not None:
1119                         username = downloader_params['username']
1120                         password = downloader_params['password']
1121                 elif downloader_params.get('usenetrc', False):
1122                         try:
1123                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
1124                                 if info is not None:
1125                                         username = info[0]
1126                                         password = info[2]
1127                                 else:
1128                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
1129                         except (IOError, netrc.NetrcParseError), err:
1130                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
1131                                 return
1132
1133                 # Set language
1134                 request = urllib2.Request(self._LANG_URL)
1135                 try:
1136                         self.report_lang()
1137                         urllib2.urlopen(request).read()
1138                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1139                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
1140                         return
1141
1142                 # No authentication to be performed
1143                 if username is None:
1144                         return
1145
1146                 # Log in
1147                 login_form = {
1148                                 'current_form': 'loginForm',
1149                                 'next':         '/',
1150                                 'action_login': 'Log In',
1151                                 'username':     username,
1152                                 'password':     password,
1153                                 }
1154                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
1155                 try:
1156                         self.report_login()
1157                         login_results = urllib2.urlopen(request).read()
1158                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
1159                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
1160                                 return
1161                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1162                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
1163                         return
1164
1165                 # Confirm age
1166                 age_form = {
1167                                 'next_url':             '/',
1168                                 'action_confirm':       'Confirm',
1169                                 }
1170                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
1171                 try:
1172                         self.report_age_confirmation()
1173                         age_results = urllib2.urlopen(request).read()
1174                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1175                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1176                         return
1177
1178         def _real_extract(self, url):
1179                 # Extract video id from URL
1180                 mobj = re.match(self._VALID_URL, url)
1181                 if mobj is None:
1182                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1183                         return
1184                 video_id = mobj.group(2)
1185
1186                 # Get video webpage
1187                 self.report_video_webpage_download(video_id)
1188                 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&amp;has_verified=1' % video_id)
1189                 try:
1190                         video_webpage = urllib2.urlopen(request).read()
1191                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1192                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
1193                         return
1194
1195                 # Attempt to extract SWF player URL
1196                 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1197                 if mobj is not None:
1198                         player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1199                 else:
1200                         player_url = None
1201
1202                 # Get video info
1203                 self.report_video_info_webpage_download(video_id)
1204                 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1205                         video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1206                                         % (video_id, el_type))
1207                         request = urllib2.Request(video_info_url)
1208                         try:
1209                                 video_info_webpage = urllib2.urlopen(request).read()
1210                                 video_info = parse_qs(video_info_webpage)
1211                                 if 'token' in video_info:
1212                                         break
1213                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1214                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
1215                                 return
1216                 if 'token' not in video_info:
1217                         if 'reason' in video_info:
1218                                 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
1219                         else:
1220                                 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
1221                         return
1222
1223                 # Start extracting information
1224                 self.report_information_extraction(video_id)
1225
1226                 # uploader
1227                 if 'author' not in video_info:
1228                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1229                         return
1230                 video_uploader = urllib.unquote_plus(video_info['author'][0])
1231
1232                 # title
1233                 if 'title' not in video_info:
1234                         self._downloader.trouble(u'ERROR: unable to extract video title')
1235                         return
1236                 video_title = urllib.unquote_plus(video_info['title'][0])
1237                 video_title = video_title.decode('utf-8')
1238                 video_title = sanitize_title(video_title)
1239
1240                 # simplified title
1241                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1242                 simple_title = simple_title.strip(ur'_')
1243
1244                 # thumbnail image
1245                 if 'thumbnail_url' not in video_info:
1246                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
1247                         video_thumbnail = ''
1248                 else:   # don't panic if we can't find it
1249                         video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
1250
1251                 # upload date
1252                 upload_date = u'NA'
1253                 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
1254                 if mobj is not None:
1255                         upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1256                         format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
1257                         for expression in format_expressions:
1258                                 try:
1259                                         upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
1260                                 except:
1261                                         pass
1262
1263                 # description
1264                 try:
1265                         lxml.etree
1266                 except NameError:
1267                         video_description = u'No description available.'
1268                         if self._downloader.params.get('forcedescription', False) or self._downloader.params.get('writedescription', False):
1269                                 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
1270                                 if mobj is not None:
1271                                         video_description = mobj.group(1).decode('utf-8')
1272                 else:
1273                         html_parser = lxml.etree.HTMLParser(encoding='utf-8')
1274                         vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
1275                         video_description = u''.join(vwebpage_doc.xpath('id("eow-description")//text()'))
1276                         # TODO use another parser
1277
1278                 # token
1279                 video_token = urllib.unquote_plus(video_info['token'][0])
1280
1281                 # Decide which formats to download
1282                 req_format = self._downloader.params.get('format', None)
1283
1284                 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1285                         self.report_rtmp_download()
1286                         video_url_list = [(None, video_info['conn'][0])]
1287                 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
1288                         url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
1289                         url_data = [parse_qs(uds) for uds in url_data_strs]
1290                         url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
1291                         url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
1292
1293                         format_limit = self._downloader.params.get('format_limit', None)
1294                         if format_limit is not None and format_limit in self._available_formats:
1295                                 format_list = self._available_formats[self._available_formats.index(format_limit):]
1296                         else:
1297                                 format_list = self._available_formats
1298                         existing_formats = [x for x in format_list if x in url_map]
1299                         if len(existing_formats) == 0:
1300                                 self._downloader.trouble(u'ERROR: no known formats available for video')
1301                                 return
1302                         if req_format is None:
1303                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
1304                         elif req_format == '-1':
1305                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1306                         else:
1307                                 # Specific format
1308                                 if req_format not in url_map:
1309                                         self._downloader.trouble(u'ERROR: requested format not available')
1310                                         return
1311                                 video_url_list = [(req_format, url_map[req_format])] # Specific format
1312                 else:
1313                         self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
1314                         return
1315
1316                 for format_param, video_real_url in video_url_list:
1317                         # At this point we have a new video
1318                         self._downloader.increment_downloads()
1319
1320                         # Extension
1321                         video_extension = self._video_extensions.get(format_param, 'flv')
1322
1323                         try:
1324                                 # Process video information
1325                                 self._downloader.process_info({
1326                                         'id':           video_id.decode('utf-8'),
1327                                         'url':          video_real_url.decode('utf-8'),
1328                                         'uploader':     video_uploader.decode('utf-8'),
1329                                         'upload_date':  upload_date,
1330                                         'title':        video_title,
1331                                         'stitle':       simple_title,
1332                                         'ext':          video_extension.decode('utf-8'),
1333                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
1334                                         'thumbnail':    video_thumbnail.decode('utf-8'),
1335                                         'description':  video_description,
1336                                         'player_url':   player_url,
1337                                 })
1338                         except UnavailableVideoError, err:
1339                                 self._downloader.trouble(u'\nERROR: unable to download video')
1340
1341
1342 class MetacafeIE(InfoExtractor):
1343         """Information Extractor for metacafe.com."""
1344
1345         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
1346         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
1347         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
1348         _youtube_ie = None
1349
1350         def __init__(self, youtube_ie, downloader=None):
1351                 InfoExtractor.__init__(self, downloader)
1352                 self._youtube_ie = youtube_ie
1353
1354         @staticmethod
1355         def suitable(url):
1356                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
1357
1358         def report_disclaimer(self):
1359                 """Report disclaimer retrieval."""
1360                 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
1361
1362         def report_age_confirmation(self):
1363                 """Report attempt to confirm age."""
1364                 self._downloader.to_screen(u'[metacafe] Confirming age')
1365
1366         def report_download_webpage(self, video_id):
1367                 """Report webpage download."""
1368                 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
1369
1370         def report_extraction(self, video_id):
1371                 """Report information extraction."""
1372                 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
1373
1374         def _real_initialize(self):
1375                 # Retrieve disclaimer
1376                 request = urllib2.Request(self._DISCLAIMER)
1377                 try:
1378                         self.report_disclaimer()
1379                         disclaimer = urllib2.urlopen(request).read()
1380                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1381                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1382                         return
1383
1384                 # Confirm age
1385                 disclaimer_form = {
1386                         'filters': '0',
1387                         'submit': "Continue - I'm over 18",
1388                         }
1389                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
1390                 try:
1391                         self.report_age_confirmation()
1392                         disclaimer = urllib2.urlopen(request).read()
1393                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1394                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1395                         return
1396
1397         def _real_extract(self, url):
1398                 # Extract id and simplified title from URL
1399                 mobj = re.match(self._VALID_URL, url)
1400                 if mobj is None:
1401                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1402                         return
1403
1404                 video_id = mobj.group(1)
1405
1406                 # Check if video comes from YouTube
1407                 mobj2 = re.match(r'^yt-(.*)$', video_id)
1408                 if mobj2 is not None:
1409                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1410                         return
1411
1412                 # At this point we have a new video
1413                 self._downloader.increment_downloads()
1414
1415                 simple_title = mobj.group(2).decode('utf-8')
1416
1417                 # Retrieve video webpage to extract further information
1418                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1419                 try:
1420                         self.report_download_webpage(video_id)
1421                         webpage = urllib2.urlopen(request).read()
1422                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1423                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1424                         return
1425
1426                 # Extract URL, uploader and title from webpage
1427                 self.report_extraction(video_id)
1428                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1429                 if mobj is not None:
1430                         mediaURL = urllib.unquote(mobj.group(1))
1431                         video_extension = mediaURL[-3:]
1432
1433                         # Extract gdaKey if available
1434                         mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1435                         if mobj is None:
1436                                 video_url = mediaURL
1437                         else:
1438                                 gdaKey = mobj.group(1)
1439                                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1440                 else:
1441                         mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
1442                         if mobj is None:
1443                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1444                                 return
1445                         vardict = parse_qs(mobj.group(1))
1446                         if 'mediaData' not in vardict:
1447                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1448                                 return
1449                         mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
1450                         if mobj is None:
1451                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
1452                                 return
1453                         mediaURL = mobj.group(1).replace('\\/', '/')
1454                         video_extension = mediaURL[-3:]
1455                         video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
1456
1457                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1458                 if mobj is None:
1459                         self._downloader.trouble(u'ERROR: unable to extract title')
1460                         return
1461                 video_title = mobj.group(1).decode('utf-8')
1462                 video_title = sanitize_title(video_title)
1463
1464                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1465                 if mobj is None:
1466                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1467                         return
1468                 video_uploader = mobj.group(1)
1469
1470                 try:
1471                         # Process video information
1472                         self._downloader.process_info({
1473                                 'id':           video_id.decode('utf-8'),
1474                                 'url':          video_url.decode('utf-8'),
1475                                 'uploader':     video_uploader.decode('utf-8'),
1476                                 'upload_date':  u'NA',
1477                                 'title':        video_title,
1478                                 'stitle':       simple_title,
1479                                 'ext':          video_extension.decode('utf-8'),
1480                                 'format':       u'NA',
1481                                 'player_url':   None,
1482                         })
1483                 except UnavailableVideoError:
1484                         self._downloader.trouble(u'\nERROR: unable to download video')
1485
1486
1487 class DailymotionIE(InfoExtractor):
1488         """Information Extractor for Dailymotion"""
1489
1490         _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1491
1492         def __init__(self, downloader=None):
1493                 InfoExtractor.__init__(self, downloader)
1494
1495         @staticmethod
1496         def suitable(url):
1497                 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1498
1499         def report_download_webpage(self, video_id):
1500                 """Report webpage download."""
1501                 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
1502
1503         def report_extraction(self, video_id):
1504                 """Report information extraction."""
1505                 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
1506
1507         def _real_initialize(self):
1508                 return
1509
1510         def _real_extract(self, url):
1511                 # Extract id and simplified title from URL
1512                 mobj = re.match(self._VALID_URL, url)
1513                 if mobj is None:
1514                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1515                         return
1516
1517                 # At this point we have a new video
1518                 self._downloader.increment_downloads()
1519                 video_id = mobj.group(1)
1520
1521                 simple_title = mobj.group(2).decode('utf-8')
1522                 video_extension = 'flv'
1523
1524                 # Retrieve video webpage to extract further information
1525                 request = urllib2.Request(url)
1526                 try:
1527                         self.report_download_webpage(video_id)
1528                         webpage = urllib2.urlopen(request).read()
1529                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1530                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1531                         return
1532
1533                 # Extract URL, uploader and title from webpage
1534                 self.report_extraction(video_id)
1535                 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1536                 if mobj is None:
1537                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1538                         return
1539                 mediaURL = urllib.unquote(mobj.group(1))
1540
1541                 # if needed add http://www.dailymotion.com/ if relative URL
1542
1543                 video_url = mediaURL
1544
1545                 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1546                 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1547                 if mobj is None:
1548                         self._downloader.trouble(u'ERROR: unable to extract title')
1549                         return
1550                 video_title = mobj.group(1).decode('utf-8')
1551                 video_title = sanitize_title(video_title)
1552
1553                 mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
1554                 if mobj is None:
1555                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1556                         return
1557                 video_uploader = mobj.group(1)
1558
1559                 try:
1560                         # Process video information
1561                         self._downloader.process_info({
1562                                 'id':           video_id.decode('utf-8'),
1563                                 'url':          video_url.decode('utf-8'),
1564                                 'uploader':     video_uploader.decode('utf-8'),
1565                                 'upload_date':  u'NA',
1566                                 'title':        video_title,
1567                                 'stitle':       simple_title,
1568                                 'ext':          video_extension.decode('utf-8'),
1569                                 'format':       u'NA',
1570                                 'player_url':   None,
1571                         })
1572                 except UnavailableVideoError:
1573                         self._downloader.trouble(u'\nERROR: unable to download video')
1574
1575
1576 class GoogleIE(InfoExtractor):
1577         """Information extractor for video.google.com."""
1578
1579         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1580
1581         def __init__(self, downloader=None):
1582                 InfoExtractor.__init__(self, downloader)
1583
1584         @staticmethod
1585         def suitable(url):
1586                 return (re.match(GoogleIE._VALID_URL, url) is not None)
1587
1588         def report_download_webpage(self, video_id):
1589                 """Report webpage download."""
1590                 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
1591
1592         def report_extraction(self, video_id):
1593                 """Report information extraction."""
1594                 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
1595
1596         def _real_initialize(self):
1597                 return
1598
1599         def _real_extract(self, url):
1600                 # Extract id from URL
1601                 mobj = re.match(self._VALID_URL, url)
1602                 if mobj is None:
1603                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1604                         return
1605
1606                 # At this point we have a new video
1607                 self._downloader.increment_downloads()
1608                 video_id = mobj.group(1)
1609
1610                 video_extension = 'mp4'
1611
1612                 # Retrieve video webpage to extract further information
1613                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1614                 try:
1615                         self.report_download_webpage(video_id)
1616                         webpage = urllib2.urlopen(request).read()
1617                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1618                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1619                         return
1620
1621                 # Extract URL, uploader, and title from webpage
1622                 self.report_extraction(video_id)
1623                 mobj = re.search(r"download_url:'([^']+)'", webpage)
1624                 if mobj is None:
1625                         video_extension = 'flv'
1626                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1627                 if mobj is None:
1628                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1629                         return
1630                 mediaURL = urllib.unquote(mobj.group(1))
1631                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1632                 mediaURL = mediaURL.replace('\\x26', '\x26')
1633
1634                 video_url = mediaURL
1635
1636                 mobj = re.search(r'<title>(.*)</title>', webpage)
1637                 if mobj is None:
1638                         self._downloader.trouble(u'ERROR: unable to extract title')
1639                         return
1640                 video_title = mobj.group(1).decode('utf-8')
1641                 video_title = sanitize_title(video_title)
1642                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1643
1644                 # Extract video description
1645                 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1646                 if mobj is None:
1647                         self._downloader.trouble(u'ERROR: unable to extract video description')
1648                         return
1649                 video_description = mobj.group(1).decode('utf-8')
1650                 if not video_description:
1651                         video_description = 'No description available.'
1652
1653                 # Extract video thumbnail
1654                 if self._downloader.params.get('forcethumbnail', False):
1655                         request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1656                         try:
1657                                 webpage = urllib2.urlopen(request).read()
1658                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1659                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1660                                 return
1661                         mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1662                         if mobj is None:
1663                                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1664                                 return
1665                         video_thumbnail = mobj.group(1)
1666                 else:   # we need something to pass to process_info
1667                         video_thumbnail = ''
1668
1669                 try:
1670                         # Process video information
1671                         self._downloader.process_info({
1672                                 'id':           video_id.decode('utf-8'),
1673                                 'url':          video_url.decode('utf-8'),
1674                                 'uploader':     u'NA',
1675                                 'upload_date':  u'NA',
1676                                 'title':        video_title,
1677                                 'stitle':       simple_title,
1678                                 'ext':          video_extension.decode('utf-8'),
1679                                 'format':       u'NA',
1680                                 'player_url':   None,
1681                         })
1682                 except UnavailableVideoError:
1683                         self._downloader.trouble(u'\nERROR: unable to download video')
1684
1685
1686 class PhotobucketIE(InfoExtractor):
1687         """Information extractor for photobucket.com."""
1688
1689         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1690
1691         def __init__(self, downloader=None):
1692                 InfoExtractor.__init__(self, downloader)
1693
1694         @staticmethod
1695         def suitable(url):
1696                 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1697
1698         def report_download_webpage(self, video_id):
1699                 """Report webpage download."""
1700                 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
1701
1702         def report_extraction(self, video_id):
1703                 """Report information extraction."""
1704                 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
1705
1706         def _real_initialize(self):
1707                 return
1708
1709         def _real_extract(self, url):
1710                 # Extract id from URL
1711                 mobj = re.match(self._VALID_URL, url)
1712                 if mobj is None:
1713                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1714                         return
1715
1716                 # At this point we have a new video
1717                 self._downloader.increment_downloads()
1718                 video_id = mobj.group(1)
1719
1720                 video_extension = 'flv'
1721
1722                 # Retrieve video webpage to extract further information
1723                 request = urllib2.Request(url)
1724                 try:
1725                         self.report_download_webpage(video_id)
1726                         webpage = urllib2.urlopen(request).read()
1727                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1728                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1729                         return
1730
1731                 # Extract URL, uploader, and title from webpage
1732                 self.report_extraction(video_id)
1733                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1734                 if mobj is None:
1735                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1736                         return
1737                 mediaURL = urllib.unquote(mobj.group(1))
1738
1739                 video_url = mediaURL
1740
1741                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1742                 if mobj is None:
1743                         self._downloader.trouble(u'ERROR: unable to extract title')
1744                         return
1745                 video_title = mobj.group(1).decode('utf-8')
1746                 video_title = sanitize_title(video_title)
1747                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1748
1749                 video_uploader = mobj.group(2).decode('utf-8')
1750
1751                 try:
1752                         # Process video information
1753                         self._downloader.process_info({
1754                                 'id':           video_id.decode('utf-8'),
1755                                 'url':          video_url.decode('utf-8'),
1756                                 'uploader':     video_uploader,
1757                                 'upload_date':  u'NA',
1758                                 'title':        video_title,
1759                                 'stitle':       simple_title,
1760                                 'ext':          video_extension.decode('utf-8'),
1761                                 'format':       u'NA',
1762                                 'player_url':   None,
1763                         })
1764                 except UnavailableVideoError:
1765                         self._downloader.trouble(u'\nERROR: unable to download video')
1766
1767
1768 class YahooIE(InfoExtractor):
1769         """Information extractor for video.yahoo.com."""
1770
1771         # _VALID_URL matches all Yahoo! Video URLs
1772         # _VPAGE_URL matches only the extractable '/watch/' URLs
1773         _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1774         _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1775
1776         def __init__(self, downloader=None):
1777                 InfoExtractor.__init__(self, downloader)
1778
1779         @staticmethod
1780         def suitable(url):
1781                 return (re.match(YahooIE._VALID_URL, url) is not None)
1782
1783         def report_download_webpage(self, video_id):
1784                 """Report webpage download."""
1785                 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
1786
1787         def report_extraction(self, video_id):
1788                 """Report information extraction."""
1789                 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
1790
1791         def _real_initialize(self):
1792                 return
1793
1794         def _real_extract(self, url, new_video=True):
1795                 # Extract ID from URL
1796                 mobj = re.match(self._VALID_URL, url)
1797                 if mobj is None:
1798                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1799                         return
1800
1801                 # At this point we have a new video
1802                 self._downloader.increment_downloads()
1803                 video_id = mobj.group(2)
1804                 video_extension = 'flv'
1805
1806                 # Rewrite valid but non-extractable URLs as
1807                 # extractable English language /watch/ URLs
1808                 if re.match(self._VPAGE_URL, url) is None:
1809                         request = urllib2.Request(url)
1810                         try:
1811                                 webpage = urllib2.urlopen(request).read()
1812                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1813                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1814                                 return
1815
1816                         mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1817                         if mobj is None:
1818                                 self._downloader.trouble(u'ERROR: Unable to extract id field')
1819                                 return
1820                         yahoo_id = mobj.group(1)
1821
1822                         mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1823                         if mobj is None:
1824                                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1825                                 return
1826                         yahoo_vid = mobj.group(1)
1827
1828                         url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1829                         return self._real_extract(url, new_video=False)
1830
1831                 # Retrieve video webpage to extract further information
1832                 request = urllib2.Request(url)
1833                 try:
1834                         self.report_download_webpage(video_id)
1835                         webpage = urllib2.urlopen(request).read()
1836                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1837                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1838                         return
1839
1840                 # Extract uploader and title from webpage
1841                 self.report_extraction(video_id)
1842                 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1843                 if mobj is None:
1844                         self._downloader.trouble(u'ERROR: unable to extract video title')
1845                         return
1846                 video_title = mobj.group(1).decode('utf-8')
1847                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1848
1849                 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1850                 if mobj is None:
1851                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
1852                         return
1853                 video_uploader = mobj.group(1).decode('utf-8')
1854
1855                 # Extract video thumbnail
1856                 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1857                 if mobj is None:
1858                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1859                         return
1860                 video_thumbnail = mobj.group(1).decode('utf-8')
1861
1862                 # Extract video description
1863                 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1864                 if mobj is None:
1865                         self._downloader.trouble(u'ERROR: unable to extract video description')
1866                         return
1867                 video_description = mobj.group(1).decode('utf-8')
1868                 if not video_description:
1869                         video_description = 'No description available.'
1870
1871                 # Extract video height and width
1872                 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1873                 if mobj is None:
1874                         self._downloader.trouble(u'ERROR: unable to extract video height')
1875                         return
1876                 yv_video_height = mobj.group(1)
1877
1878                 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1879                 if mobj is None:
1880                         self._downloader.trouble(u'ERROR: unable to extract video width')
1881                         return
1882                 yv_video_width = mobj.group(1)
1883
1884                 # Retrieve video playlist to extract media URL
1885                 # I'm not completely sure what all these options are, but we
1886                 # seem to need most of them, otherwise the server sends a 401.
1887                 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
1888                 yv_bitrate = '700'  # according to Wikipedia this is hard-coded
1889                 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1890                                 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1891                                 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1892                 try:
1893                         self.report_download_webpage(video_id)
1894                         webpage = urllib2.urlopen(request).read()
1895                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1896                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1897                         return
1898
1899                 # Extract media URL from playlist XML
1900                 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1901                 if mobj is None:
1902                         self._downloader.trouble(u'ERROR: Unable to extract media URL')
1903                         return
1904                 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1905                 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1906
1907                 try:
1908                         # Process video information
1909                         self._downloader.process_info({
1910                                 'id':           video_id.decode('utf-8'),
1911                                 'url':          video_url,
1912                                 'uploader':     video_uploader,
1913                                 'upload_date':  u'NA',
1914                                 'title':        video_title,
1915                                 'stitle':       simple_title,
1916                                 'ext':          video_extension.decode('utf-8'),
1917                                 'thumbnail':    video_thumbnail.decode('utf-8'),
1918                                 'description':  video_description,
1919                                 'thumbnail':    video_thumbnail,
1920                                 'description':  video_description,
1921                                 'player_url':   None,
1922                         })
1923                 except UnavailableVideoError:
1924                         self._downloader.trouble(u'\nERROR: unable to download video')
1925
1926
1927 class VimeoIE(InfoExtractor):
1928         """Information extractor for vimeo.com."""
1929
1930         # _VALID_URL matches Vimeo URLs
1931         _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)'
1932
1933         def __init__(self, downloader=None):
1934                 InfoExtractor.__init__(self, downloader)
1935
1936         @staticmethod
1937         def suitable(url):
1938                 return (re.match(VimeoIE._VALID_URL, url) is not None)
1939
1940         def report_download_webpage(self, video_id):
1941                 """Report webpage download."""
1942                 self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1943
1944         def report_extraction(self, video_id):
1945                 """Report information extraction."""
1946                 self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1947
1948         def _real_initialize(self):
1949                 return
1950
1951         def _real_extract(self, url, new_video=True):
1952                 # Extract ID from URL
1953                 mobj = re.match(self._VALID_URL, url)
1954                 if mobj is None:
1955                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1956                         return
1957
1958                 # At this point we have a new video
1959                 self._downloader.increment_downloads()
1960                 video_id = mobj.group(1)
1961
1962                 # Retrieve video webpage to extract further information
1963                 request = urllib2.Request("http://vimeo.com/moogaloop/load/clip:%s" % video_id, None, std_headers)
1964                 try:
1965                         self.report_download_webpage(video_id)
1966                         webpage = urllib2.urlopen(request).read()
1967                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1968                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1969                         return
1970
1971                 # Now we begin extracting as much information as we can from what we
1972                 # retrieved. First we extract the information common to all extractors,
1973                 # and latter we extract those that are Vimeo specific.
1974                 self.report_extraction(video_id)
1975
1976                 # Extract title
1977                 mobj = re.search(r'<caption>(.*?)</caption>', webpage)
1978                 if mobj is None:
1979                         self._downloader.trouble(u'ERROR: unable to extract video title')
1980                         return
1981                 video_title = mobj.group(1).decode('utf-8')
1982                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1983
1984                 # Extract uploader
1985                 mobj = re.search(r'<uploader_url>http://vimeo.com/(.*?)</uploader_url>', webpage)
1986                 if mobj is None:
1987                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
1988                         return
1989                 video_uploader = mobj.group(1).decode('utf-8')
1990
1991                 # Extract video thumbnail
1992                 mobj = re.search(r'<thumbnail>(.*?)</thumbnail>', webpage)
1993                 if mobj is None:
1994                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1995                         return
1996                 video_thumbnail = mobj.group(1).decode('utf-8')
1997
1998                 # # Extract video description
1999                 # mobj = re.search(r'<meta property="og:description" content="(.*)" />', webpage)
2000                 # if mobj is None:
2001                 #       self._downloader.trouble(u'ERROR: unable to extract video description')
2002                 #       return
2003                 # video_description = mobj.group(1).decode('utf-8')
2004                 # if not video_description: video_description = 'No description available.'
2005                 video_description = 'Foo.'
2006
2007                 # Vimeo specific: extract request signature
2008                 mobj = re.search(r'<request_signature>(.*?)</request_signature>', webpage)
2009                 if mobj is None:
2010                         self._downloader.trouble(u'ERROR: unable to extract request signature')
2011                         return
2012                 sig = mobj.group(1).decode('utf-8')
2013
2014                 # Vimeo specific: Extract request signature expiration
2015                 mobj = re.search(r'<request_signature_expires>(.*?)</request_signature_expires>', webpage)
2016                 if mobj is None:
2017                         self._downloader.trouble(u'ERROR: unable to extract request signature expiration')
2018                         return
2019                 sig_exp = mobj.group(1).decode('utf-8')
2020
2021                 video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s" % (video_id, sig, sig_exp)
2022
2023                 try:
2024                         # Process video information
2025                         self._downloader.process_info({
2026                                 'id':           video_id.decode('utf-8'),
2027                                 'url':          video_url,
2028                                 'uploader':     video_uploader,
2029                                 'upload_date':  u'NA',
2030                                 'title':        video_title,
2031                                 'stitle':       simple_title,
2032                                 'ext':          u'mp4',
2033                                 'thumbnail':    video_thumbnail.decode('utf-8'),
2034                                 'description':  video_description,
2035                                 'thumbnail':    video_thumbnail,
2036                                 'description':  video_description,
2037                                 'player_url':   None,
2038                         })
2039                 except UnavailableVideoError:
2040                         self._downloader.trouble(u'ERROR: unable to download video')
2041
2042
2043 class GenericIE(InfoExtractor):
2044         """Generic last-resort information extractor."""
2045
2046         def __init__(self, downloader=None):
2047                 InfoExtractor.__init__(self, downloader)
2048
2049         @staticmethod
2050         def suitable(url):
2051                 return True
2052
2053         def report_download_webpage(self, video_id):
2054                 """Report webpage download."""
2055                 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
2056                 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
2057
2058         def report_extraction(self, video_id):
2059                 """Report information extraction."""
2060                 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
2061
2062         def _real_initialize(self):
2063                 return
2064
2065         def _real_extract(self, url):
2066                 # At this point we have a new video
2067                 self._downloader.increment_downloads()
2068
2069                 video_id = url.split('/')[-1]
2070                 request = urllib2.Request(url)
2071                 try:
2072                         self.report_download_webpage(video_id)
2073                         webpage = urllib2.urlopen(request).read()
2074                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2075                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
2076                         return
2077                 except ValueError, err:
2078                         # since this is the last-resort InfoExtractor, if
2079                         # this error is thrown, it'll be thrown here
2080                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2081                         return
2082
2083                 self.report_extraction(video_id)
2084                 # Start with something easy: JW Player in SWFObject
2085                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
2086                 if mobj is None:
2087                         # Broaden the search a little bit
2088                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
2089                 if mobj is None:
2090                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2091                         return
2092
2093                 # It's possible that one of the regexes
2094                 # matched, but returned an empty group:
2095                 if mobj.group(1) is None:
2096                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2097                         return
2098
2099                 video_url = urllib.unquote(mobj.group(1))
2100                 video_id = os.path.basename(video_url)
2101
2102                 # here's a fun little line of code for you:
2103                 video_extension = os.path.splitext(video_id)[1][1:]
2104                 video_id = os.path.splitext(video_id)[0]
2105
2106                 # it's tempting to parse this further, but you would
2107                 # have to take into account all the variations like
2108                 #   Video Title - Site Name
2109                 #   Site Name | Video Title
2110                 #   Video Title - Tagline | Site Name
2111                 # and so on and so forth; it's just not practical
2112                 mobj = re.search(r'<title>(.*)</title>', webpage)
2113                 if mobj is None:
2114                         self._downloader.trouble(u'ERROR: unable to extract title')
2115                         return
2116                 video_title = mobj.group(1).decode('utf-8')
2117                 video_title = sanitize_title(video_title)
2118                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
2119
2120                 # video uploader is domain name
2121                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
2122                 if mobj is None:
2123                         self._downloader.trouble(u'ERROR: unable to extract title')
2124                         return
2125                 video_uploader = mobj.group(1).decode('utf-8')
2126
2127                 try:
2128                         # Process video information
2129                         self._downloader.process_info({
2130                                 'id':           video_id.decode('utf-8'),
2131                                 'url':          video_url.decode('utf-8'),
2132                                 'uploader':     video_uploader,
2133                                 'upload_date':  u'NA',
2134                                 'title':        video_title,
2135                                 'stitle':       simple_title,
2136                                 'ext':          video_extension.decode('utf-8'),
2137                                 'format':       u'NA',
2138                                 'player_url':   None,
2139                         })
2140                 except UnavailableVideoError, err:
2141                         self._downloader.trouble(u'\nERROR: unable to download video')
2142
2143
2144 class YoutubeSearchIE(InfoExtractor):
2145         """Information Extractor for YouTube search queries."""
2146         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
2147         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
2148         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
2149         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
2150         _youtube_ie = None
2151         _max_youtube_results = 1000
2152
2153         def __init__(self, youtube_ie, downloader=None):
2154                 InfoExtractor.__init__(self, downloader)
2155                 self._youtube_ie = youtube_ie
2156
2157         @staticmethod
2158         def suitable(url):
2159                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
2160
2161         def report_download_page(self, query, pagenum):
2162                 """Report attempt to download playlist page with given number."""
2163                 query = query.decode(preferredencoding())
2164                 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
2165
2166         def _real_initialize(self):
2167                 self._youtube_ie.initialize()
2168
2169         def _real_extract(self, query):
2170                 mobj = re.match(self._VALID_QUERY, query)
2171                 if mobj is None:
2172                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2173                         return
2174
2175                 prefix, query = query.split(':')
2176                 prefix = prefix[8:]
2177                 query = query.encode('utf-8')
2178                 if prefix == '':
2179                         self._download_n_results(query, 1)
2180                         return
2181                 elif prefix == 'all':
2182                         self._download_n_results(query, self._max_youtube_results)
2183                         return
2184                 else:
2185                         try:
2186                                 n = long(prefix)
2187                                 if n <= 0:
2188                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2189                                         return
2190                                 elif n > self._max_youtube_results:
2191                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
2192                                         n = self._max_youtube_results
2193                                 self._download_n_results(query, n)
2194                                 return
2195                         except ValueError: # parsing prefix as integer fails
2196                                 self._download_n_results(query, 1)
2197                                 return
2198
2199         def _download_n_results(self, query, n):
2200                 """Downloads a specified number of results for a query"""
2201
2202                 video_ids = []
2203                 already_seen = set()
2204                 pagenum = 1
2205
2206                 while True:
2207                         self.report_download_page(query, pagenum)
2208                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2209                         request = urllib2.Request(result_url)
2210                         try:
2211                                 page = urllib2.urlopen(request).read()
2212                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2213                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2214                                 return
2215
2216                         # Extract video identifiers
2217                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2218                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
2219                                 if video_id not in already_seen:
2220                                         video_ids.append(video_id)
2221                                         already_seen.add(video_id)
2222                                         if len(video_ids) == n:
2223                                                 # Specified n videos reached
2224                                                 for id in video_ids:
2225                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2226                                                 return
2227
2228                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2229                                 for id in video_ids:
2230                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2231                                 return
2232
2233                         pagenum = pagenum + 1
2234
2235
2236 class GoogleSearchIE(InfoExtractor):
2237         """Information Extractor for Google Video search queries."""
2238         _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
2239         _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
2240         _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
2241         _MORE_PAGES_INDICATOR = r'<span>Next</span>'
2242         _google_ie = None
2243         _max_google_results = 1000
2244
2245         def __init__(self, google_ie, downloader=None):
2246                 InfoExtractor.__init__(self, downloader)
2247                 self._google_ie = google_ie
2248
2249         @staticmethod
2250         def suitable(url):
2251                 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
2252
2253         def report_download_page(self, query, pagenum):
2254                 """Report attempt to download playlist page with given number."""
2255                 query = query.decode(preferredencoding())
2256                 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
2257
2258         def _real_initialize(self):
2259                 self._google_ie.initialize()
2260
2261         def _real_extract(self, query):
2262                 mobj = re.match(self._VALID_QUERY, query)
2263                 if mobj is None:
2264                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2265                         return
2266
2267                 prefix, query = query.split(':')
2268                 prefix = prefix[8:]
2269                 query = query.encode('utf-8')
2270                 if prefix == '':
2271                         self._download_n_results(query, 1)
2272                         return
2273                 elif prefix == 'all':
2274                         self._download_n_results(query, self._max_google_results)
2275                         return
2276                 else:
2277                         try:
2278                                 n = long(prefix)
2279                                 if n <= 0:
2280                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2281                                         return
2282                                 elif n > self._max_google_results:
2283                                         self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
2284                                         n = self._max_google_results
2285                                 self._download_n_results(query, n)
2286                                 return
2287                         except ValueError: # parsing prefix as integer fails
2288                                 self._download_n_results(query, 1)
2289                                 return
2290
2291         def _download_n_results(self, query, n):
2292                 """Downloads a specified number of results for a query"""
2293
2294                 video_ids = []
2295                 already_seen = set()
2296                 pagenum = 1
2297
2298                 while True:
2299                         self.report_download_page(query, pagenum)
2300                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2301                         request = urllib2.Request(result_url)
2302                         try:
2303                                 page = urllib2.urlopen(request).read()
2304                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2305                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2306                                 return
2307
2308                         # Extract video identifiers
2309                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2310                                 video_id = mobj.group(1)
2311                                 if video_id not in already_seen:
2312                                         video_ids.append(video_id)
2313                                         already_seen.add(video_id)
2314                                         if len(video_ids) == n:
2315                                                 # Specified n videos reached
2316                                                 for id in video_ids:
2317                                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
2318                                                 return
2319
2320                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2321                                 for id in video_ids:
2322                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
2323                                 return
2324
2325                         pagenum = pagenum + 1
2326
2327
2328 class YahooSearchIE(InfoExtractor):
2329         """Information Extractor for Yahoo! Video search queries."""
2330         _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
2331         _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
2332         _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
2333         _MORE_PAGES_INDICATOR = r'\s*Next'
2334         _yahoo_ie = None
2335         _max_yahoo_results = 1000
2336
2337         def __init__(self, yahoo_ie, downloader=None):
2338                 InfoExtractor.__init__(self, downloader)
2339                 self._yahoo_ie = yahoo_ie
2340
2341         @staticmethod
2342         def suitable(url):
2343                 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
2344
2345         def report_download_page(self, query, pagenum):
2346                 """Report attempt to download playlist page with given number."""
2347                 query = query.decode(preferredencoding())
2348                 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
2349
2350         def _real_initialize(self):
2351                 self._yahoo_ie.initialize()
2352
2353         def _real_extract(self, query):
2354                 mobj = re.match(self._VALID_QUERY, query)
2355                 if mobj is None:
2356                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2357                         return
2358
2359                 prefix, query = query.split(':')
2360                 prefix = prefix[8:]
2361                 query = query.encode('utf-8')
2362                 if prefix == '':
2363                         self._download_n_results(query, 1)
2364                         return
2365                 elif prefix == 'all':
2366                         self._download_n_results(query, self._max_yahoo_results)
2367                         return
2368                 else:
2369                         try:
2370                                 n = long(prefix)
2371                                 if n <= 0:
2372                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2373                                         return
2374                                 elif n > self._max_yahoo_results:
2375                                         self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
2376                                         n = self._max_yahoo_results
2377                                 self._download_n_results(query, n)
2378                                 return
2379                         except ValueError: # parsing prefix as integer fails
2380                                 self._download_n_results(query, 1)
2381                                 return
2382
2383         def _download_n_results(self, query, n):
2384                 """Downloads a specified number of results for a query"""
2385
2386                 video_ids = []
2387                 already_seen = set()
2388                 pagenum = 1
2389
2390                 while True:
2391                         self.report_download_page(query, pagenum)
2392                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2393                         request = urllib2.Request(result_url)
2394                         try:
2395                                 page = urllib2.urlopen(request).read()
2396                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2397                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2398                                 return
2399
2400                         # Extract video identifiers
2401                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2402                                 video_id = mobj.group(1)
2403                                 if video_id not in already_seen:
2404                                         video_ids.append(video_id)
2405                                         already_seen.add(video_id)
2406                                         if len(video_ids) == n:
2407                                                 # Specified n videos reached
2408                                                 for id in video_ids:
2409                                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
2410                                                 return
2411
2412                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2413                                 for id in video_ids:
2414                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
2415                                 return
2416
2417                         pagenum = pagenum + 1
2418
2419
2420 class YoutubePlaylistIE(InfoExtractor):
2421         """Information Extractor for YouTube playlists."""
2422
2423         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
2424         _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
2425         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
2426         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
2427         _youtube_ie = None
2428
2429         def __init__(self, youtube_ie, downloader=None):
2430                 InfoExtractor.__init__(self, downloader)
2431                 self._youtube_ie = youtube_ie
2432
2433         @staticmethod
2434         def suitable(url):
2435                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
2436
2437         def report_download_page(self, playlist_id, pagenum):
2438                 """Report attempt to download playlist page with given number."""
2439                 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
2440
2441         def _real_initialize(self):
2442                 self._youtube_ie.initialize()
2443
2444         def _real_extract(self, url):
2445                 # Extract playlist id
2446                 mobj = re.match(self._VALID_URL, url)
2447                 if mobj is None:
2448                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2449                         return
2450
2451                 # Single video case
2452                 if mobj.group(3) is not None:
2453                         self._youtube_ie.extract(mobj.group(3))
2454                         return
2455
2456                 # Download playlist pages
2457                 # prefix is 'p' as default for playlists but there are other types that need extra care
2458                 playlist_prefix = mobj.group(1)
2459                 if playlist_prefix == 'a':
2460                         playlist_access = 'artist'
2461                 else:
2462                         playlist_prefix = 'p'
2463                         playlist_access = 'view_play_list'
2464                 playlist_id = mobj.group(2)
2465                 video_ids = []
2466                 pagenum = 1
2467
2468                 while True:
2469                         self.report_download_page(playlist_id, pagenum)
2470                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum))
2471                         try:
2472                                 page = urllib2.urlopen(request).read()
2473                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2474                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2475                                 return
2476
2477                         # Extract video identifiers
2478                         ids_in_page = []
2479                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2480                                 if mobj.group(1) not in ids_in_page:
2481                                         ids_in_page.append(mobj.group(1))
2482                         video_ids.extend(ids_in_page)
2483
2484                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2485                                 break
2486                         pagenum = pagenum + 1
2487
2488                 playliststart = self._downloader.params.get('playliststart', 1) - 1
2489                 playlistend = self._downloader.params.get('playlistend', -1)
2490                 video_ids = video_ids[playliststart:playlistend]
2491
2492                 for id in video_ids:
2493                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2494                 return
2495
2496
2497 class YoutubeUserIE(InfoExtractor):
2498         """Information Extractor for YouTube users."""
2499
2500         _VALID_URL = r'(?:(?:(?:http://)?(?:\w+\.)?youtube.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
2501         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
2502         _GDATA_PAGE_SIZE = 50
2503         _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
2504         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
2505         _youtube_ie = None
2506
2507         def __init__(self, youtube_ie, downloader=None):
2508                 InfoExtractor.__init__(self, downloader)
2509                 self._youtube_ie = youtube_ie
2510
2511         @staticmethod
2512         def suitable(url):
2513                 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
2514
2515         def report_download_page(self, username, start_index):
2516                 """Report attempt to download user page."""
2517                 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
2518                                 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
2519
2520         def _real_initialize(self):
2521                 self._youtube_ie.initialize()
2522
2523         def _real_extract(self, url):
2524                 # Extract username
2525                 mobj = re.match(self._VALID_URL, url)
2526                 if mobj is None:
2527                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2528                         return
2529
2530                 username = mobj.group(1)
2531
2532                 # Download video ids using YouTube Data API. Result size per
2533                 # query is limited (currently to 50 videos) so we need to query
2534                 # page by page until there are no video ids - it means we got
2535                 # all of them.
2536
2537                 video_ids = []
2538                 pagenum = 0
2539
2540                 while True:
2541                         start_index = pagenum * self._GDATA_PAGE_SIZE + 1
2542                         self.report_download_page(username, start_index)
2543
2544                         request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
2545
2546                         try:
2547                                 page = urllib2.urlopen(request).read()
2548                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2549                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2550                                 return
2551
2552                         # Extract video identifiers
2553                         ids_in_page = []
2554
2555                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2556                                 if mobj.group(1) not in ids_in_page:
2557                                         ids_in_page.append(mobj.group(1))
2558
2559                         video_ids.extend(ids_in_page)
2560
2561                         # A little optimization - if current page is not
2562                         # "full", ie. does not contain PAGE_SIZE video ids then
2563                         # we can assume that this page is the last one - there
2564                         # are no more ids on further pages - no need to query
2565                         # again.
2566
2567                         if len(ids_in_page) < self._GDATA_PAGE_SIZE:
2568                                 break
2569
2570                         pagenum += 1
2571
2572                 all_ids_count = len(video_ids)
2573                 playliststart = self._downloader.params.get('playliststart', 1) - 1
2574                 playlistend = self._downloader.params.get('playlistend', -1)
2575
2576                 if playlistend == -1:
2577                         video_ids = video_ids[playliststart:]
2578                 else:
2579                         video_ids = video_ids[playliststart:playlistend]
2580
2581                 self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" %
2582                                 (username, all_ids_count, len(video_ids)))
2583
2584                 for video_id in video_ids:
2585                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id)
2586
2587
2588 class DepositFilesIE(InfoExtractor):
2589         """Information extractor for depositfiles.com"""
2590
2591         _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)'
2592
2593         def __init__(self, downloader=None):
2594                 InfoExtractor.__init__(self, downloader)
2595
2596         @staticmethod
2597         def suitable(url):
2598                 return (re.match(DepositFilesIE._VALID_URL, url) is not None)
2599
2600         def report_download_webpage(self, file_id):
2601                 """Report webpage download."""
2602                 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
2603
2604         def report_extraction(self, file_id):
2605                 """Report information extraction."""
2606                 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
2607
2608         def _real_initialize(self):
2609                 return
2610
2611         def _real_extract(self, url):
2612                 # At this point we have a new file
2613                 self._downloader.increment_downloads()
2614
2615                 file_id = url.split('/')[-1]
2616                 # Rebuild url in english locale
2617                 url = 'http://depositfiles.com/en/files/' + file_id
2618
2619                 # Retrieve file webpage with 'Free download' button pressed
2620                 free_download_indication = { 'gateway_result' : '1' }
2621                 request = urllib2.Request(url, urllib.urlencode(free_download_indication))
2622                 try:
2623                         self.report_download_webpage(file_id)
2624                         webpage = urllib2.urlopen(request).read()
2625                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2626                         self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
2627                         return
2628
2629                 # Search for the real file URL
2630                 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
2631                 if (mobj is None) or (mobj.group(1) is None):
2632                         # Try to figure out reason of the error.
2633                         mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
2634                         if (mobj is not None) and (mobj.group(1) is not None):
2635                                 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
2636                                 self._downloader.trouble(u'ERROR: %s' % restriction_message)
2637                         else:
2638                                 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
2639                         return
2640
2641                 file_url = mobj.group(1)
2642                 file_extension = os.path.splitext(file_url)[1][1:]
2643
2644                 # Search for file title
2645                 mobj = re.search(r'<b title="(.*?)">', webpage)
2646                 if mobj is None:
2647                         self._downloader.trouble(u'ERROR: unable to extract title')
2648                         return
2649                 file_title = mobj.group(1).decode('utf-8')
2650
2651                 try:
2652                         # Process file information
2653                         self._downloader.process_info({
2654                                 'id':           file_id.decode('utf-8'),
2655                                 'url':          file_url.decode('utf-8'),
2656                                 'uploader':     u'NA',
2657                                 'upload_date':  u'NA',
2658                                 'title':        file_title,
2659                                 'stitle':       file_title,
2660                                 'ext':          file_extension.decode('utf-8'),
2661                                 'format':       u'NA',
2662                                 'player_url':   None,
2663                         })
2664                 except UnavailableVideoError, err:
2665                         self._downloader.trouble(u'ERROR: unable to download file')
2666
2667
2668 class FacebookIE(InfoExtractor):
2669         """Information Extractor for Facebook"""
2670
2671         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook.com/video/video.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
2672         _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
2673         _NETRC_MACHINE = 'facebook'
2674         _available_formats = ['highqual', 'lowqual']
2675         _video_extensions = {
2676                 'highqual': 'mp4',
2677                 'lowqual': 'mp4',
2678         }
2679
2680         def __init__(self, downloader=None):
2681                 InfoExtractor.__init__(self, downloader)
2682
2683         @staticmethod
2684         def suitable(url):
2685                 return (re.match(FacebookIE._VALID_URL, url) is not None)
2686
2687         def _reporter(self, message):
2688                 """Add header and report message."""
2689                 self._downloader.to_screen(u'[facebook] %s' % message)
2690
2691         def report_login(self):
2692                 """Report attempt to log in."""
2693                 self._reporter(u'Logging in')
2694
2695         def report_video_webpage_download(self, video_id):
2696                 """Report attempt to download video webpage."""
2697                 self._reporter(u'%s: Downloading video webpage' % video_id)
2698
2699         def report_information_extraction(self, video_id):
2700                 """Report attempt to extract video information."""
2701                 self._reporter(u'%s: Extracting video information' % video_id)
2702
2703         def _parse_page(self, video_webpage):
2704                 """Extract video information from page"""
2705                 # General data
2706                 data = {'title': r'class="video_title datawrap">(.*?)</',
2707                         'description': r'<div class="datawrap">(.*?)</div>',
2708                         'owner': r'\("video_owner_name", "(.*?)"\)',
2709                         'upload_date': r'data-date="(.*?)"',
2710                         'thumbnail':  r'\("thumb_url", "(?P<THUMB>.*?)"\)',
2711                         }
2712                 video_info = {}
2713                 for piece in data.keys():
2714                         mobj = re.search(data[piece], video_webpage)
2715                         if mobj is not None:
2716                                 video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
2717
2718                 # Video urls
2719                 video_urls = {}
2720                 for fmt in self._available_formats:
2721                         mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
2722                         if mobj is not None:
2723                                 # URL is in a Javascript segment inside an escaped Unicode format within
2724                                 # the generally utf-8 page
2725                                 video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
2726                 video_info['video_urls'] = video_urls
2727
2728                 return video_info
2729
2730         def _real_initialize(self):
2731                 if self._downloader is None:
2732                         return
2733
2734                 useremail = None
2735                 password = None
2736                 downloader_params = self._downloader.params
2737
2738                 # Attempt to use provided username and password or .netrc data
2739                 if downloader_params.get('username', None) is not None:
2740                         useremail = downloader_params['username']
2741                         password = downloader_params['password']
2742                 elif downloader_params.get('usenetrc', False):
2743                         try:
2744                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
2745                                 if info is not None:
2746                                         useremail = info[0]
2747                                         password = info[2]
2748                                 else:
2749                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
2750                         except (IOError, netrc.NetrcParseError), err:
2751                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
2752                                 return
2753
2754                 if useremail is None:
2755                         return
2756
2757                 # Log in
2758                 login_form = {
2759                         'email': useremail,
2760                         'pass': password,
2761                         'login': 'Log+In'
2762                         }
2763                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
2764                 try:
2765                         self.report_login()
2766                         login_results = urllib2.urlopen(request).read()
2767                         if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
2768                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
2769                                 return
2770                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2771                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
2772                         return
2773
2774         def _real_extract(self, url):
2775                 mobj = re.match(self._VALID_URL, url)
2776                 if mobj is None:
2777                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2778                         return
2779                 video_id = mobj.group('ID')
2780
2781                 # Get video webpage
2782                 self.report_video_webpage_download(video_id)
2783                 request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
2784                 try:
2785                         page = urllib2.urlopen(request)
2786                         video_webpage = page.read()
2787                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2788                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2789                         return
2790
2791                 # Start extracting information
2792                 self.report_information_extraction(video_id)
2793
2794                 # Extract information
2795                 video_info = self._parse_page(video_webpage)
2796
2797                 # uploader
2798                 if 'owner' not in video_info:
2799                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
2800                         return
2801                 video_uploader = video_info['owner']
2802
2803                 # title
2804                 if 'title' not in video_info:
2805                         self._downloader.trouble(u'ERROR: unable to extract video title')
2806                         return
2807                 video_title = video_info['title']
2808                 video_title = video_title.decode('utf-8')
2809                 video_title = sanitize_title(video_title)
2810
2811                 # simplified title
2812                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
2813                 simple_title = simple_title.strip(ur'_')
2814
2815                 # thumbnail image
2816                 if 'thumbnail' not in video_info:
2817                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
2818                         video_thumbnail = ''
2819                 else:
2820                         video_thumbnail = video_info['thumbnail']
2821
2822                 # upload date
2823                 upload_date = u'NA'
2824                 if 'upload_date' in video_info:
2825                         upload_time = video_info['upload_date']
2826                         timetuple = email.utils.parsedate_tz(upload_time)
2827                         if timetuple is not None:
2828                                 try:
2829                                         upload_date = time.strftime('%Y%m%d', timetuple[0:9])
2830                                 except:
2831                                         pass
2832
2833                 # description
2834                 video_description = video_info.get('description', 'No description available.')
2835
2836                 url_map = video_info['video_urls']
2837                 if len(url_map.keys()) > 0:
2838                         # Decide which formats to download
2839                         req_format = self._downloader.params.get('format', None)
2840                         format_limit = self._downloader.params.get('format_limit', None)
2841
2842                         if format_limit is not None and format_limit in self._available_formats:
2843                                 format_list = self._available_formats[self._available_formats.index(format_limit):]
2844                         else:
2845                                 format_list = self._available_formats
2846                         existing_formats = [x for x in format_list if x in url_map]
2847                         if len(existing_formats) == 0:
2848                                 self._downloader.trouble(u'ERROR: no known formats available for video')
2849                                 return
2850                         if req_format is None:
2851                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2852                         elif req_format == '-1':
2853                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2854                         else:
2855                                 # Specific format
2856                                 if req_format not in url_map:
2857                                         self._downloader.trouble(u'ERROR: requested format not available')
2858                                         return
2859                                 video_url_list = [(req_format, url_map[req_format])] # Specific format
2860
2861                 for format_param, video_real_url in video_url_list:
2862
2863                         # At this point we have a new video
2864                         self._downloader.increment_downloads()
2865
2866                         # Extension
2867                         video_extension = self._video_extensions.get(format_param, 'mp4')
2868
2869                         try:
2870                                 # Process video information
2871                                 self._downloader.process_info({
2872                                         'id':           video_id.decode('utf-8'),
2873                                         'url':          video_real_url.decode('utf-8'),
2874                                         'uploader':     video_uploader.decode('utf-8'),
2875                                         'upload_date':  upload_date,
2876                                         'title':        video_title,
2877                                         'stitle':       simple_title,
2878                                         'ext':          video_extension.decode('utf-8'),
2879                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
2880                                         'thumbnail':    video_thumbnail.decode('utf-8'),
2881                                         'description':  video_description.decode('utf-8'),
2882                                         'player_url':   None,
2883                                 })
2884                         except UnavailableVideoError, err:
2885                                 self._downloader.trouble(u'\nERROR: unable to download video')
2886
2887 class BlipTVIE(InfoExtractor):
2888         """Information extractor for blip.tv"""
2889
2890         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2891         _URL_EXT = r'^.*\.([a-z0-9]+)$'
2892
2893         @staticmethod
2894         def suitable(url):
2895                 return (re.match(BlipTVIE._VALID_URL, url) is not None)
2896
2897         def report_extraction(self, file_id):
2898                 """Report information extraction."""
2899                 self._downloader.to_screen(u'[blip.tv] %s: Extracting information' % file_id)
2900
2901         def _simplify_title(self, title):
2902                 res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
2903                 res = res.strip(ur'_')
2904                 return res
2905
2906         def _real_extract(self, url):
2907                 mobj = re.match(self._VALID_URL, url)
2908                 if mobj is None:
2909                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2910                         return
2911
2912                 if '?' in url:
2913                         cchar = '&'
2914                 else:
2915                         cchar = '?'
2916                 json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2917                 request = urllib2.Request(json_url)
2918                 self.report_extraction(mobj.group(1))
2919                 try:
2920                         json_code = urllib2.urlopen(request).read()
2921                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2922                         self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
2923                         return
2924                 try:
2925                         json_data = json.loads(json_code)
2926                         if 'Post' in json_data:
2927                                 data = json_data['Post']
2928                         else:
2929                                 data = json_data
2930
2931                         upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2932                         video_url = data['media']['url']
2933                         umobj = re.match(self._URL_EXT, video_url)
2934                         if umobj is None:
2935                                 raise ValueError('Can not determine filename extension')
2936                         ext = umobj.group(1)
2937
2938                         self._downloader.increment_downloads()
2939
2940                         info = {
2941                                 'id': data['item_id'],
2942                                 'url': video_url,
2943                                 'uploader': data['display_name'],
2944                                 'upload_date': upload_date,
2945                                 'title': data['title'],
2946                                 'stitle': self._simplify_title(data['title']),
2947                                 'ext': ext,
2948                                 'format': data['media']['mimeType'],
2949                                 'thumbnail': data['thumbnailUrl'],
2950                                 'description': data['description'],
2951                                 'player_url': data['embedUrl']
2952                         }
2953                 except (ValueError,KeyError), err:
2954                         self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2955                         return
2956
2957                 try:
2958                         self._downloader.process_info(info)
2959                 except UnavailableVideoError, err:
2960                         self._downloader.trouble(u'\nERROR: unable to download video')
2961
2962
2963 class MyVideoIE(InfoExtractor):
2964         """Information Extractor for myvideo.de."""
2965
2966         _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
2967
2968         def __init__(self, downloader=None):
2969                 InfoExtractor.__init__(self, downloader)
2970
2971         @staticmethod
2972         def suitable(url):
2973                 return (re.match(MyVideoIE._VALID_URL, url) is not None)
2974
2975         def report_download_webpage(self, video_id):
2976                 """Report webpage download."""
2977                 self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
2978
2979         def report_extraction(self, video_id):
2980                 """Report information extraction."""
2981                 self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
2982
2983         def _real_initialize(self):
2984                 return
2985
2986         def _real_extract(self,url):
2987                 mobj = re.match(self._VALID_URL, url)
2988                 if mobj is None:
2989                         self._download.trouble(u'ERROR: invalid URL: %s' % url)
2990                         return
2991
2992                 video_id = mobj.group(1)
2993                 simple_title = mobj.group(2).decode('utf-8')
2994                 # should actually not be necessary
2995                 simple_title = sanitize_title(simple_title)
2996                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', simple_title)
2997
2998                 # Get video webpage
2999                 request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
3000                 try:
3001                         self.report_download_webpage(video_id)
3002                         webpage = urllib2.urlopen(request).read()
3003                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3004                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
3005                         return
3006
3007                 self.report_extraction(video_id)
3008                 mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
3009                                  webpage)
3010                 if mobj is None:
3011                         self._downloader.trouble(u'ERROR: unable to extract media URL')
3012                         return
3013                 video_url = mobj.group(1) + ('/%s.flv' % video_id)
3014
3015                 mobj = re.search('<title>([^<]+)</title>', webpage)
3016                 if mobj is None:
3017                         self._downloader.trouble(u'ERROR: unable to extract title')
3018                         return
3019
3020                 video_title = mobj.group(1)
3021                 video_title = sanitize_title(video_title)
3022
3023                 try:
3024                         print(video_url)
3025                         self._downloader.process_info({
3026                                 'id':           video_id,
3027                                 'url':          video_url,
3028                                 'uploader':     u'NA',
3029                                 'upload_date':  u'NA',
3030                                 'title':        video_title,
3031                                 'stitle':       simple_title,
3032                                 'ext':          u'flv',
3033                                 'format':       u'NA',
3034                                 'player_url':   None,
3035                         })
3036                 except UnavailableVideoError:
3037                         self._downloader.trouble(u'\nERROR: Unable to download video')
3038
3039 class ComedyCentralIE(InfoExtractor):
3040         """Information extractor for blip.tv"""
3041
3042         _VALID_URL = r'^(?:https?://)?(www\.)?(thedailyshow|colbertnation)\.com/full-episodes/(.*)$'
3043
3044         @staticmethod
3045         def suitable(url):
3046                 return (re.match(ComedyCentralIE._VALID_URL, url) is not None)
3047
3048         def report_extraction(self, episode_id):
3049                 self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
3050
3051         def report_config_download(self, episode_id):
3052                 self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
3053
3054         def report_player_url(self, episode_id):
3055                 self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
3056
3057         def _simplify_title(self, title):
3058                 res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
3059                 res = res.strip(ur'_')
3060                 return res
3061
3062         def _real_extract(self, url):
3063                 mobj = re.match(self._VALID_URL, url)
3064                 if mobj is None:
3065                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3066                         return
3067                 epTitle = mobj.group(3)
3068
3069                 req = urllib2.Request(url)
3070                 self.report_extraction(epTitle)
3071                 try:
3072                         html = urllib2.urlopen(req).read()
3073                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3074                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
3075                         return
3076
3077                 mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/(.*?:episode:([^:]*):)(.*?))"/>', html)
3078                 if len(mMovieParams) == 0:
3079                         self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
3080                         return
3081                 show_id = mMovieParams[0][2]
3082                 ACT_COUNT = { # TODO: Detect this dynamically
3083                         'thedailyshow.com': 4,
3084                         'colbertnation.com': 3,
3085                 }.get(show_id, 4)
3086                 OFFSET = {
3087                         'thedailyshow.com': -ACT_COUNT,
3088                         'colbertnation.com': 1,
3089                 }.get(show_id, -ACT_COUNT)
3090
3091                 first_player_url = mMovieParams[0][0]
3092                 mediaNum = int(mMovieParams[0][3]) + OFFSET
3093                 movieId = mMovieParams[0][1]
3094
3095                 playerReq = urllib2.Request(first_player_url)
3096                 self.report_player_url(epTitle)
3097                 try:
3098                         playerResponse = urllib2.urlopen(playerReq)
3099                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3100                         self._downloader.trouble(u'ERROR: unable to download player: %s' % unicode(err))
3101                         return
3102                 player_url = playerResponse.geturl()
3103
3104                 for actNum in range(ACT_COUNT):
3105                         actTitle = 'act' + str(actNum+1)
3106                         mediaId = movieId + str(mediaNum + actNum)
3107                         configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
3108                                                 urllib.urlencode({'uri': mediaId}))
3109                         configReq = urllib2.Request(configUrl)
3110                         self.report_config_download(epTitle)
3111                         try:
3112                                 configXml = urllib2.urlopen(configReq).read()
3113                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3114                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
3115                                 return
3116
3117                         cdoc = xml.etree.ElementTree.fromstring(configXml)
3118                         turls = []
3119                         for rendition in cdoc.findall('.//rendition'):
3120                                 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
3121                                 turls.append(finfo)
3122
3123                         if len(turls) == 0:
3124                                 self._downloader.trouble(u'\nERROR: unable to download ' + actTitle + ': No videos found')
3125                                 continue
3126
3127                         # For now, just pick the highest bitrate
3128                         format,video_url = turls[-1]
3129
3130                         self._downloader.increment_downloads()
3131
3132                         effTitle = show_id.replace('.com', '') + '-' + epTitle
3133                         info = {
3134                                 'id': actTitle,
3135                                 'url': video_url,
3136                                 'uploader': show_id,
3137                                 'upload_date': 'NA',
3138                                 'title': effTitle,
3139                                 'stitle': self._simplify_title(effTitle),
3140                                 'ext': 'mp4',
3141                                 'format': format,
3142                                 'thumbnail': None,
3143                                 'description': 'TODO: Not yet supported',
3144                                 'player_url': player_url
3145                         }
3146
3147                         try:
3148                                 self._downloader.process_info(info)
3149                         except UnavailableVideoError, err:
3150                                 self._downloader.trouble(u'\nERROR: unable to download ' + actTitle)
3151                                 continue
3152
3153
3154 class PostProcessor(object):
3155         """Post Processor class.
3156
3157         PostProcessor objects can be added to downloaders with their
3158         add_post_processor() method. When the downloader has finished a
3159         successful download, it will take its internal chain of PostProcessors
3160         and start calling the run() method on each one of them, first with
3161         an initial argument and then with the returned value of the previous
3162         PostProcessor.
3163
3164         The chain will be stopped if one of them ever returns None or the end
3165         of the chain is reached.
3166
3167         PostProcessor objects follow a "mutual registration" process similar
3168         to InfoExtractor objects.
3169         """
3170
3171         _downloader = None
3172
3173         def __init__(self, downloader=None):
3174                 self._downloader = downloader
3175
3176         def set_downloader(self, downloader):
3177                 """Sets the downloader for this PP."""
3178                 self._downloader = downloader
3179
3180         def run(self, information):
3181                 """Run the PostProcessor.
3182
3183                 The "information" argument is a dictionary like the ones
3184                 composed by InfoExtractors. The only difference is that this
3185                 one has an extra field called "filepath" that points to the
3186                 downloaded file.
3187
3188                 When this method returns None, the postprocessing chain is
3189                 stopped. However, this method may return an information
3190                 dictionary that will be passed to the next postprocessing
3191                 object in the chain. It can be the one it received after
3192                 changing some fields.
3193
3194                 In addition, this method may raise a PostProcessingError
3195                 exception that will be taken into account by the downloader
3196                 it was called from.
3197                 """
3198                 return information # by default, do nothing
3199
3200
3201 class FFmpegExtractAudioPP(PostProcessor):
3202
3203         def __init__(self, downloader=None, preferredcodec=None):
3204                 PostProcessor.__init__(self, downloader)
3205                 if preferredcodec is None:
3206                         preferredcodec = 'best'
3207                 self._preferredcodec = preferredcodec
3208
3209         @staticmethod
3210         def get_audio_codec(path):
3211                 try:
3212                         cmd = ['ffprobe', '-show_streams', '--', path]
3213                         handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
3214                         output = handle.communicate()[0]
3215                         if handle.wait() != 0:
3216                                 return None
3217                 except (IOError, OSError):
3218                         return None
3219                 audio_codec = None
3220                 for line in output.split('\n'):
3221                         if line.startswith('codec_name='):
3222                                 audio_codec = line.split('=')[1].strip()
3223                         elif line.strip() == 'codec_type=audio' and audio_codec is not None:
3224                                 return audio_codec
3225                 return None
3226
3227         @staticmethod
3228         def run_ffmpeg(path, out_path, codec, more_opts):
3229                 try:
3230                         cmd = ['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + ['--', out_path]
3231                         ret = subprocess.call(cmd, stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
3232                         return (ret == 0)
3233                 except (IOError, OSError):
3234                         return False
3235
3236         def run(self, information):
3237                 path = information['filepath']
3238
3239                 filecodec = self.get_audio_codec(path)
3240                 if filecodec is None:
3241                         self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
3242                         return None
3243
3244                 more_opts = []
3245                 if self._preferredcodec == 'best' or self._preferredcodec == filecodec:
3246                         if filecodec == 'aac' or filecodec == 'mp3':
3247                                 # Lossless if possible
3248                                 acodec = 'copy'
3249                                 extension = filecodec
3250                                 if filecodec == 'aac':
3251                                         more_opts = ['-f', 'adts']
3252                         else:
3253                                 # MP3 otherwise.
3254                                 acodec = 'libmp3lame'
3255                                 extension = 'mp3'
3256                                 more_opts = ['-ab', '128k']
3257                 else:
3258                         # We convert the audio (lossy)
3259                         acodec = {'mp3': 'libmp3lame', 'aac': 'aac'}[self._preferredcodec]
3260                         extension = self._preferredcodec
3261                         more_opts = ['-ab', '128k']
3262                         if self._preferredcodec == 'aac':
3263                                 more_opts += ['-f', 'adts']
3264
3265                 (prefix, ext) = os.path.splitext(path)
3266                 new_path = prefix + '.' + extension
3267                 self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path)
3268                 status = self.run_ffmpeg(path, new_path, acodec, more_opts)
3269
3270                 if not status:
3271                         self._downloader.to_stderr(u'WARNING: error running ffmpeg')
3272                         return None
3273
3274                 try:
3275                         os.remove(path)
3276                 except (IOError, OSError):
3277                         self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
3278                         return None
3279
3280                 information['filepath'] = new_path
3281                 return information
3282
3283
3284 def updateSelf(downloader, filename):
3285         ''' Update the program file with the latest version from the repository '''
3286         # Note: downloader only used for options
3287         if not os.access(filename, os.W_OK):
3288                 sys.exit('ERROR: no write permissions on %s' % filename)
3289
3290         downloader.to_screen('Updating to latest version...')
3291
3292         try:
3293                 try:
3294                         urlh = urllib.urlopen(UPDATE_URL)
3295                         newcontent = urlh.read()
3296                 finally:
3297                         urlh.close()
3298         except (IOError, OSError), err:
3299                 sys.exit('ERROR: unable to download latest version')
3300
3301         try:
3302                 outf = open(filename, 'wb')
3303                 try:
3304                         outf.write(newcontent)
3305                 finally:
3306                         outf.close()
3307         except (IOError, OSError), err:
3308                 sys.exit('ERROR: unable to overwrite current version')
3309
3310         downloader.to_screen('Updated youtube-dl. Restart to use the new version.')
3311
3312 def parseOpts():
3313         # Deferred imports
3314         import getpass
3315         import optparse
3316
3317         def _format_option_string(option):
3318                 ''' ('-o', '--option') -> -o, --format METAVAR'''
3319
3320                 opts = []
3321
3322                 if option._short_opts: opts.append(option._short_opts[0])
3323                 if option._long_opts: opts.append(option._long_opts[0])
3324                 if len(opts) > 1: opts.insert(1, ', ')
3325
3326                 if option.takes_value(): opts.append(' %s' % option.metavar)
3327
3328                 return "".join(opts)
3329
3330         def _find_term_columns():
3331                 columns = os.environ.get('COLUMNS', None)
3332                 if columns:
3333                         return int(columns)
3334
3335                 try:
3336                         sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
3337                         out,err = sp.communicate()
3338                         return int(out.split()[1])
3339                 except:
3340                         pass
3341                 return None
3342
3343         max_width = 80
3344         max_help_position = 80
3345
3346         # No need to wrap help messages if we're on a wide console
3347         columns = _find_term_columns()
3348         if columns: max_width = columns
3349
3350         fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
3351         fmt.format_option_strings = _format_option_string
3352
3353         kw = {
3354                 'version'   : __version__,
3355                 'formatter' : fmt,
3356                 'usage' : '%prog [options] url...',
3357                 'conflict_handler' : 'resolve',
3358         }
3359
3360         parser = optparse.OptionParser(**kw)
3361
3362         # option groups
3363         general        = optparse.OptionGroup(parser, 'General Options')
3364         authentication = optparse.OptionGroup(parser, 'Authentication Options')
3365         video_format   = optparse.OptionGroup(parser, 'Video Format Options')
3366         postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
3367         filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
3368         verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
3369
3370         general.add_option('-h', '--help',
3371                         action='help', help='print this help text and exit')
3372         general.add_option('-v', '--version',
3373                         action='version', help='print program version and exit')
3374         general.add_option('-U', '--update',
3375                         action='store_true', dest='update_self', help='update this program to latest version')
3376         general.add_option('-i', '--ignore-errors',
3377                         action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
3378         general.add_option('-r', '--rate-limit',
3379                         dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
3380         general.add_option('-R', '--retries',
3381                         dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
3382         general.add_option('--playlist-start',
3383                         dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
3384         general.add_option('--playlist-end',
3385                         dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
3386         general.add_option('--dump-user-agent',
3387                         action='store_true', dest='dump_user_agent',
3388                         help='display the current browser identification', default=False)
3389
3390         authentication.add_option('-u', '--username',
3391                         dest='username', metavar='USERNAME', help='account username')
3392         authentication.add_option('-p', '--password',
3393                         dest='password', metavar='PASSWORD', help='account password')
3394         authentication.add_option('-n', '--netrc',
3395                         action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
3396
3397
3398         video_format.add_option('-f', '--format',
3399                         action='store', dest='format', metavar='FORMAT', help='video format code')
3400         video_format.add_option('--all-formats',
3401                         action='store_const', dest='format', help='download all available video formats', const='-1')
3402         video_format.add_option('--max-quality',
3403                         action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
3404
3405
3406         verbosity.add_option('-q', '--quiet',
3407                         action='store_true', dest='quiet', help='activates quiet mode', default=False)
3408         verbosity.add_option('-s', '--simulate',
3409                         action='store_true', dest='simulate', help='do not download video', default=False)
3410         verbosity.add_option('-g', '--get-url',
3411                         action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
3412         verbosity.add_option('-e', '--get-title',
3413                         action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
3414         verbosity.add_option('--get-thumbnail',
3415                         action='store_true', dest='getthumbnail',
3416                         help='simulate, quiet but print thumbnail URL', default=False)
3417         verbosity.add_option('--get-description',
3418                         action='store_true', dest='getdescription',
3419                         help='simulate, quiet but print video description', default=False)
3420         verbosity.add_option('--get-filename',
3421                         action='store_true', dest='getfilename',
3422                         help='simulate, quiet but print output filename', default=False)
3423         verbosity.add_option('--no-progress',
3424                         action='store_true', dest='noprogress', help='do not print progress bar', default=False)
3425         verbosity.add_option('--console-title',
3426                         action='store_true', dest='consoletitle',
3427                         help='display progress in console titlebar', default=False)
3428
3429
3430         filesystem.add_option('-t', '--title',
3431                         action='store_true', dest='usetitle', help='use title in file name', default=False)
3432         filesystem.add_option('-l', '--literal',
3433                         action='store_true', dest='useliteral', help='use literal title in file name', default=False)
3434         filesystem.add_option('-A', '--auto-number',
3435                         action='store_true', dest='autonumber',
3436                         help='number downloaded files starting from 00000', default=False)
3437         filesystem.add_option('-o', '--output',
3438                         dest='outtmpl', metavar='TEMPLATE', help='output filename template')
3439         filesystem.add_option('-a', '--batch-file',
3440                         dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
3441         filesystem.add_option('-w', '--no-overwrites',
3442                         action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
3443         filesystem.add_option('-c', '--continue',
3444                         action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
3445         filesystem.add_option('--cookies',
3446                         dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
3447         filesystem.add_option('--no-part',
3448                         action='store_true', dest='nopart', help='do not use .part files', default=False)
3449         filesystem.add_option('--no-mtime',
3450                         action='store_false', dest='updatetime',
3451                         help='do not use the Last-modified header to set the file modification time', default=True)
3452         filesystem.add_option('--write-description',
3453                         action='store_true', dest='writedescription',
3454                         help='write video description to a .description file', default=False)
3455         filesystem.add_option('--write-info-json',
3456                         action='store_true', dest='writeinfojson',
3457                         help='write video metadata to a .info.json file', default=False)
3458
3459
3460         postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
3461                         help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
3462         postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
3463                         help='"best", "aac" or "mp3"; best by default')
3464
3465
3466         parser.add_option_group(general)
3467         parser.add_option_group(filesystem)
3468         parser.add_option_group(verbosity)
3469         parser.add_option_group(video_format)
3470         parser.add_option_group(authentication)
3471         parser.add_option_group(postproc)
3472
3473         opts, args = parser.parse_args()
3474
3475         return parser, opts, args
3476
3477 def main():
3478         parser, opts, args = parseOpts()
3479
3480         # Open appropriate CookieJar
3481         if opts.cookiefile is None:
3482                 jar = cookielib.CookieJar()
3483         else:
3484                 try:
3485                         jar = cookielib.MozillaCookieJar(opts.cookiefile)
3486                         if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
3487                                 jar.load()
3488                 except (IOError, OSError), err:
3489                         sys.exit(u'ERROR: unable to open cookie file')
3490
3491         # Dump user agent
3492         if opts.dump_user_agent:
3493                 print std_headers['User-Agent']
3494                 sys.exit(0)
3495
3496         # General configuration
3497         cookie_processor = urllib2.HTTPCookieProcessor(jar)
3498         opener = urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())
3499         urllib2.install_opener(opener)
3500         socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
3501
3502         # Batch file verification
3503         batchurls = []
3504         if opts.batchfile is not None:
3505                 try:
3506                         if opts.batchfile == '-':
3507                                 batchfd = sys.stdin
3508                         else:
3509                                 batchfd = open(opts.batchfile, 'r')
3510                         batchurls = batchfd.readlines()
3511                         batchurls = [x.strip() for x in batchurls]
3512                         batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
3513                 except IOError:
3514                         sys.exit(u'ERROR: batch file could not be read')
3515         all_urls = batchurls + args
3516
3517         # Conflicting, missing and erroneous options
3518         if opts.usenetrc and (opts.username is not None or opts.password is not None):
3519                 parser.error(u'using .netrc conflicts with giving username/password')
3520         if opts.password is not None and opts.username is None:
3521                 parser.error(u'account username missing')
3522         if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber):
3523                 parser.error(u'using output template conflicts with using title, literal title or auto number')
3524         if opts.usetitle and opts.useliteral:
3525                 parser.error(u'using title conflicts with using literal title')
3526         if opts.username is not None and opts.password is None:
3527                 opts.password = getpass.getpass(u'Type account password and press return:')
3528         if opts.ratelimit is not None:
3529                 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
3530                 if numeric_limit is None:
3531                         parser.error(u'invalid rate limit specified')
3532                 opts.ratelimit = numeric_limit
3533         if opts.retries is not None:
3534                 try:
3535                         opts.retries = long(opts.retries)
3536                 except (TypeError, ValueError), err:
3537                         parser.error(u'invalid retry count specified')
3538         try:
3539                 opts.playliststart = int(opts.playliststart)
3540                 if opts.playliststart <= 0:
3541                         raise ValueError(u'Playlist start must be positive')
3542         except (TypeError, ValueError), err:
3543                 parser.error(u'invalid playlist start number specified')
3544         try:
3545                 opts.playlistend = int(opts.playlistend)
3546                 if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
3547                         raise ValueError(u'Playlist end must be greater than playlist start')
3548         except (TypeError, ValueError), err:
3549                 parser.error(u'invalid playlist end number specified')
3550         if opts.extractaudio:
3551                 if opts.audioformat not in ['best', 'aac', 'mp3']:
3552                         parser.error(u'invalid audio format specified')
3553
3554         # Information extractors
3555         youtube_ie = YoutubeIE()
3556         metacafe_ie = MetacafeIE(youtube_ie)
3557         dailymotion_ie = DailymotionIE()
3558         youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
3559         youtube_user_ie = YoutubeUserIE(youtube_ie)
3560         youtube_search_ie = YoutubeSearchIE(youtube_ie)
3561         google_ie = GoogleIE()
3562         google_search_ie = GoogleSearchIE(google_ie)
3563         photobucket_ie = PhotobucketIE()
3564         yahoo_ie = YahooIE()
3565         yahoo_search_ie = YahooSearchIE(yahoo_ie)
3566         deposit_files_ie = DepositFilesIE()
3567         facebook_ie = FacebookIE()
3568         bliptv_ie = BlipTVIE()
3569         vimeo_ie = VimeoIE()
3570         myvideo_ie = MyVideoIE()
3571         comedycentral_ie = ComedyCentralIE()
3572
3573         generic_ie = GenericIE()
3574
3575         # File downloader
3576         fd = FileDownloader({
3577                 'usenetrc': opts.usenetrc,
3578                 'username': opts.username,
3579                 'password': opts.password,
3580                 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
3581                 'forceurl': opts.geturl,
3582                 'forcetitle': opts.gettitle,
3583                 'forcethumbnail': opts.getthumbnail,
3584                 'forcedescription': opts.getdescription,
3585                 'forcefilename': opts.getfilename,
3586                 'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
3587                 'format': opts.format,
3588                 'format_limit': opts.format_limit,
3589                 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
3590                         or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
3591                         or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
3592                         or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
3593                         or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
3594                         or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
3595                         or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
3596                         or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
3597                         or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
3598                         or u'%(id)s.%(ext)s'),
3599                 'ignoreerrors': opts.ignoreerrors,
3600                 'ratelimit': opts.ratelimit,
3601                 'nooverwrites': opts.nooverwrites,
3602                 'retries': opts.retries,
3603                 'continuedl': opts.continue_dl,
3604                 'noprogress': opts.noprogress,
3605                 'playliststart': opts.playliststart,
3606                 'playlistend': opts.playlistend,
3607                 'logtostderr': opts.outtmpl == '-',
3608                 'consoletitle': opts.consoletitle,
3609                 'nopart': opts.nopart,
3610                 'updatetime': opts.updatetime,
3611                 'writedescription': opts.writedescription,
3612                 'writeinfojson': opts.writeinfojson,
3613                 })
3614         fd.add_info_extractor(youtube_search_ie)
3615         fd.add_info_extractor(youtube_pl_ie)
3616         fd.add_info_extractor(youtube_user_ie)
3617         fd.add_info_extractor(metacafe_ie)
3618         fd.add_info_extractor(dailymotion_ie)
3619         fd.add_info_extractor(youtube_ie)
3620         fd.add_info_extractor(google_ie)
3621         fd.add_info_extractor(google_search_ie)
3622         fd.add_info_extractor(photobucket_ie)
3623         fd.add_info_extractor(yahoo_ie)
3624         fd.add_info_extractor(yahoo_search_ie)
3625         fd.add_info_extractor(deposit_files_ie)
3626         fd.add_info_extractor(facebook_ie)
3627         fd.add_info_extractor(bliptv_ie)
3628         fd.add_info_extractor(vimeo_ie)
3629         fd.add_info_extractor(myvideo_ie)
3630         fd.add_info_extractor(comedycentral_ie)
3631
3632         # This must come last since it's the
3633         # fallback if none of the others work
3634         fd.add_info_extractor(generic_ie)
3635
3636         # PostProcessors
3637         if opts.extractaudio:
3638                 fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat))
3639
3640         # Update version
3641         if opts.update_self:
3642                 updateSelf(fd, sys.argv[0])
3643
3644         # Maybe do nothing
3645         if len(all_urls) < 1:
3646                 if not opts.update_self:
3647                         parser.error(u'you must provide at least one URL')
3648                 else:
3649                         sys.exit()
3650         retcode = fd.download(all_urls)
3651
3652         # Dump cookie jar if requested
3653         if opts.cookiefile is not None:
3654                 try:
3655                         jar.save()
3656                 except (IOError, OSError), err:
3657                         sys.exit(u'ERROR: unable to save cookie jar')
3658
3659         sys.exit(retcode)
3660
3661
3662 if __name__ == '__main__':
3663         try:
3664                 main()
3665         except DownloadError:
3666                 sys.exit(1)
3667         except SameFileError:
3668                 sys.exit(u'ERROR: fixed output name but more than one file to download')
3669         except KeyboardInterrupt:
3670                 sys.exit(u'\nERROR: Interrupted by user')
3671
3672 # vim: set ts=4 sw=4 sts=4 noet ai si filetype=python: