Switch Vimeo to scraping: fixes #285
[youtube-dl.git] / youtube-dl
index 733e231..6497490 100755 (executable)
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__author__  = (
+__authors__  = (
        'Ricardo Garcia Gonzalez',
        'Danny Colligan',
        'Benjamin Johnson',
        'Ricardo Garcia Gonzalez',
        'Danny Colligan',
        'Benjamin Johnson',
@@ -12,15 +12,20 @@ __author__  = (
        'Rogério Brito',
        'Philipp Hagemeister',
        'Sören Schulze',
        'Rogério Brito',
        'Philipp Hagemeister',
        'Sören Schulze',
+       'Kevin Ngo',
+       'Ori Avtalion',
+       'shizeeg',
        )
 
 __license__ = 'Public Domain'
        )
 
 __license__ = 'Public Domain'
-__version__ = '2011.09.15'
+__version__ = '2012.02.27'
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 
+
 import cookielib
 import datetime
 import cookielib
 import datetime
+import getpass
 import gzip
 import htmlentitydefs
 import HTMLParser
 import gzip
 import htmlentitydefs
 import HTMLParser
@@ -28,9 +33,11 @@ import httplib
 import locale
 import math
 import netrc
 import locale
 import math
 import netrc
+import optparse
 import os
 import os.path
 import re
 import os
 import os.path
 import re
+import shlex
 import socket
 import string
 import subprocess
 import socket
 import string
 import subprocess
@@ -66,8 +73,8 @@ except ImportError:
 
 try:
        import xml.etree.ElementTree
 
 try:
        import xml.etree.ElementTree
-except ImportError: # Python<2.5
-       pass # Not officially supported, but let it slip
+except ImportError: # Python<2.5: Not officially supported, but let it slip
+       warnings.warn('xml.etree.ElementTree support is missing. Consider upgrading to Python >= 2.5 if you get related errors.')
 
 std_headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
 
 std_headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
@@ -77,8 +84,6 @@ std_headers = {
        'Accept-Language': 'en-us,en;q=0.5',
 }
 
        'Accept-Language': 'en-us,en;q=0.5',
 }
 
-simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
-
 try:
        import json
 except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
 try:
        import json
 except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
@@ -258,14 +263,14 @@ def sanitize_open(filename, open_mode):
                                import msvcrt
                                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
                        return (sys.stdout, filename)
                                import msvcrt
                                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
                        return (sys.stdout, filename)
-               stream = open(filename, open_mode)
+               stream = open(_encodeFilename(filename), open_mode)
                return (stream, filename)
        except (IOError, OSError), err:
                # In case of error, try to remove win32 forbidden chars
                filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 
                # An exception here should be caught in the caller
                return (stream, filename)
        except (IOError, OSError), err:
                # In case of error, try to remove win32 forbidden chars
                filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 
                # An exception here should be caught in the caller
-               stream = open(filename, open_mode)
+               stream = open(_encodeFilename(filename), open_mode)
                return (stream, filename)
 
 
                return (stream, filename)
 
 
@@ -277,6 +282,41 @@ def timeconvert(timestr):
                timestamp = email.utils.mktime_tz(timetuple)
        return timestamp
 
                timestamp = email.utils.mktime_tz(timetuple)
        return timestamp
 
+def _simplify_title(title):
+       expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE)
+       return expr.sub(u'_', title).strip(u'_')
+
+def _orderedSet(iterable):
+       """ Remove all duplicates from the input iterable """
+       res = []
+       for el in iterable:
+               if el not in res:
+                       res.append(el)
+       return res
+
+def _unescapeHTML(s):
+       """
+       @param s a string (of type unicode)
+       """
+       assert type(s) == type(u'')
+
+       htmlParser = HTMLParser.HTMLParser()
+       return htmlParser.unescape(s)
+
+def _encodeFilename(s):
+       """
+       @param s The name of the file (of type unicode)
+       """
+
+       assert type(s) == type(u'')
+
+       if sys.platform == 'win32' and sys.getwindowsversion().major >= 5:
+               # Pass u'' directly to use Unicode APIs on Windows 2000 and up
+               # (Detecting Windows NT 4 is tricky because 'major >= 4' would
+               # match Windows 9x series as well. Besides, NT 4 is obsolete.)
+               return s
+       else:
+               return s.encode(sys.getfilesystemencoding(), 'ignore')
 
 class DownloadError(Exception):
        """Download Error exception.
 
 class DownloadError(Exception):
        """Download Error exception.
@@ -305,6 +345,10 @@ class PostProcessingError(Exception):
        """
        pass
 
        """
        pass
 
+class MaxDownloadsReached(Exception):
+       """ --max-downloads limit has been reached. """
+       pass
+
 
 class UnavailableVideoError(Exception):
        """Unavailable Format exception.
 
 class UnavailableVideoError(Exception):
        """Unavailable Format exception.
@@ -538,16 +582,17 @@ class FileDownloader(object):
                self._pps.append(pp)
                pp.set_downloader(self)
 
                self._pps.append(pp)
                pp.set_downloader(self)
 
-       def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
+       def to_screen(self, message, skip_eol=False):
                """Print message to stdout if not in quiet mode."""
                """Print message to stdout if not in quiet mode."""
-               try:
-                       if not self.params.get('quiet', False):
-                               terminator = [u'\n', u''][skip_eol]
-                               print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
+               assert type(message) == type(u'')
+               if not self.params.get('quiet', False):
+                       terminator = [u'\n', u''][skip_eol]
+                       output = message + terminator
+
+                       if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+                               output = output.encode(preferredencoding(), 'ignore')
+                       self._screen_file.write(output)
                        self._screen_file.flush()
                        self._screen_file.flush()
-               except (UnicodeEncodeError), err:
-                       if not ignore_encoding_errors:
-                               raise
 
        def to_stderr(self, message):
                """Print message to stderr."""
 
        def to_stderr(self, message):
                """Print message to stderr."""
@@ -597,7 +642,7 @@ class FileDownloader(object):
        def temp_name(self, filename):
                """Returns a temporary filename for the given filename."""
                if self.params.get('nopart', False) or filename == u'-' or \
        def temp_name(self, filename):
                """Returns a temporary filename for the given filename."""
                if self.params.get('nopart', False) or filename == u'-' or \
-                               (os.path.exists(filename) and not os.path.isfile(filename)):
+                               (os.path.exists(_encodeFilename(filename)) and not os.path.isfile(_encodeFilename(filename))):
                        return filename
                return filename + u'.part'
 
                        return filename
                return filename + u'.part'
 
@@ -610,7 +655,7 @@ class FileDownloader(object):
                try:
                        if old_filename == new_filename:
                                return
                try:
                        if old_filename == new_filename:
                                return
-                       os.rename(old_filename, new_filename)
+                       os.rename(_encodeFilename(old_filename), _encodeFilename(new_filename))
                except (IOError, OSError), err:
                        self.trouble(u'ERROR: unable to rename file')
 
                except (IOError, OSError), err:
                        self.trouble(u'ERROR: unable to rename file')
 
@@ -618,30 +663,31 @@ class FileDownloader(object):
                """Try to set the last-modified time of the given file."""
                if last_modified_hdr is None:
                        return
                """Try to set the last-modified time of the given file."""
                if last_modified_hdr is None:
                        return
-               if not os.path.isfile(filename):
+               if not os.path.isfile(_encodeFilename(filename)):
                        return
                timestr = last_modified_hdr
                if timestr is None:
                        return
                filetime = timeconvert(timestr)
                if filetime is None:
                        return
                timestr = last_modified_hdr
                if timestr is None:
                        return
                filetime = timeconvert(timestr)
                if filetime is None:
-                       return
+                       return filetime
                try:
                        os.utime(filename, (time.time(), filetime))
                except:
                        pass
                try:
                        os.utime(filename, (time.time(), filetime))
                except:
                        pass
+               return filetime
 
        def report_writedescription(self, descfn):
                """ Report that the description file is being written """
 
        def report_writedescription(self, descfn):
                """ Report that the description file is being written """
-               self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True)
+               self.to_screen(u'[info] Writing video description to: ' + descfn)
 
        def report_writeinfojson(self, infofn):
                """ Report that the metadata file has been written """
 
        def report_writeinfojson(self, infofn):
                """ Report that the metadata file has been written """
-               self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True)
+               self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 
        def report_destination(self, filename):
                """Report destination filename."""
 
        def report_destination(self, filename):
                """Report destination filename."""
-               self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
+               self.to_screen(u'[download] Destination: ' + filename)
 
        def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
                """Report download progress."""
 
        def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
                """Report download progress."""
@@ -694,45 +740,57 @@ class FileDownloader(object):
                        self.trouble(u'ERROR: invalid system charset or erroneous output template')
                        return None
 
                        self.trouble(u'ERROR: invalid system charset or erroneous output template')
                        return None
 
+       def _match_entry(self, info_dict):
+               """ Returns None iff the file should be downloaded """
+
+               title = info_dict['title']
+               matchtitle = self.params.get('matchtitle', False)
+               if matchtitle and not re.search(matchtitle, title, re.IGNORECASE):
+                       return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
+               rejecttitle = self.params.get('rejecttitle', False)
+               if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE):
+                       return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
+               return None
+
        def process_info(self, info_dict):
                """Process a single dictionary returned by an InfoExtractor."""
        def process_info(self, info_dict):
                """Process a single dictionary returned by an InfoExtractor."""
+
+               reason = self._match_entry(info_dict)
+               if reason is not None:
+                       self.to_screen(u'[download] ' + reason)
+                       return
+
+               max_downloads = self.params.get('max_downloads')
+               if max_downloads is not None:
+                       if self._num_downloads > int(max_downloads):
+                               raise MaxDownloadsReached()
+
                filename = self.prepare_filename(info_dict)
                filename = self.prepare_filename(info_dict)
+               
+               # Forced printings
+               if self.params.get('forcetitle', False):
+                       print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
+               if self.params.get('forceurl', False):
+                       print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
+               if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
+                       print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
+               if self.params.get('forcedescription', False) and 'description' in info_dict:
+                       print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
+               if self.params.get('forcefilename', False) and filename is not None:
+                       print filename.encode(preferredencoding(), 'xmlcharrefreplace')
+               if self.params.get('forceformat', False):
+                       print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace')
+
                # Do nothing else if in simulate mode
                if self.params.get('simulate', False):
                # Do nothing else if in simulate mode
                if self.params.get('simulate', False):
-                       # Forced printings
-                       if self.params.get('forcetitle', False):
-                               print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
-                       if self.params.get('forceurl', False):
-                               print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
-                       if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
-                               print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
-                       if self.params.get('forcedescription', False) and 'description' in info_dict:
-                               print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
-                       if self.params.get('forcefilename', False) and filename is not None:
-                               print filename.encode(preferredencoding(), 'xmlcharrefreplace')
-
                        return
 
                if filename is None:
                        return
 
                        return
 
                if filename is None:
                        return
 
-               matchtitle=self.params.get('matchtitle',False)
-               rejecttitle=self.params.get('rejecttitle',False)
-               title=info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
-               if matchtitle and not re.search(matchtitle, title, re.IGNORECASE):
-                       self.to_screen(u'[download] "%s" title did not match pattern "%s"' % (title, matchtitle))
-                       return
-               if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE):
-                       self.to_screen(u'[download] "%s" title matched reject pattern "%s"' % (title, rejecttitle))
-                       return
-                       
-               if self.params.get('nooverwrites', False) and os.path.exists(filename):
-                       self.to_stderr(u'WARNING: file exists and will be skipped')
-                       return
-
                try:
                try:
-                       dn = os.path.dirname(filename)
-                       if dn != '' and not os.path.exists(dn):
+                       dn = os.path.dirname(_encodeFilename(filename))
+                       if dn != '' and not os.path.exists(dn): # dn is already encoded
                                os.makedirs(dn)
                except (OSError, IOError), err:
                        self.trouble(u'ERROR: unable to create directory ' + unicode(err))
                                os.makedirs(dn)
                except (OSError, IOError), err:
                        self.trouble(u'ERROR: unable to create directory ' + unicode(err))
@@ -740,9 +798,9 @@ class FileDownloader(object):
 
                if self.params.get('writedescription', False):
                        try:
 
                if self.params.get('writedescription', False):
                        try:
-                               descfn = filename + '.description'
+                               descfn = filename + u'.description'
                                self.report_writedescription(descfn)
                                self.report_writedescription(descfn)
-                               descfile = open(descfn, 'wb')
+                               descfile = open(_encodeFilename(descfn), 'wb')
                                try:
                                        descfile.write(info_dict['description'].encode('utf-8'))
                                finally:
                                try:
                                        descfile.write(info_dict['description'].encode('utf-8'))
                                finally:
@@ -752,7 +810,7 @@ class FileDownloader(object):
                                return
 
                if self.params.get('writeinfojson', False):
                                return
 
                if self.params.get('writeinfojson', False):
-                       infofn = filename + '.info.json'
+                       infofn = filename + u'.info.json'
                        self.report_writeinfojson(infofn)
                        try:
                                json.dump
                        self.report_writeinfojson(infofn)
                        try:
                                json.dump
@@ -760,32 +818,37 @@ class FileDownloader(object):
                                self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
                                return
                        try:
                                self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
                                return
                        try:
-                               infof = open(infofn, 'wb')
+                               infof = open(_encodeFilename(infofn), 'wb')
                                try:
                                try:
-                                       json.dump(info_dict, infof)
+                                       json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
+                                       json.dump(json_info_dict, infof)
                                finally:
                                        infof.close()
                        except (OSError, IOError):
                                self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
                                return
 
                                finally:
                                        infof.close()
                        except (OSError, IOError):
                                self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
                                return
 
-               try:
-                       success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
-               except (OSError, IOError), err:
-                       raise UnavailableVideoError
-               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-                       self.trouble(u'ERROR: unable to download video data: %s' % str(err))
-                       return
-               except (ContentTooShortError, ), err:
-                       self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
-                       return
-
-               if success:
-                       try:
-                               self.post_process(filename, info_dict)
-                       except (PostProcessingError), err:
-                               self.trouble(u'ERROR: postprocessing: %s' % str(err))
-                               return
+               if not self.params.get('skip_download', False):
+                       if self.params.get('nooverwrites', False) and os.path.exists(_encodeFilename(filename)):
+                               success = True
+                       else:
+                               try:
+                                       success = self._do_download(filename, info_dict)
+                               except (OSError, IOError), err:
+                                       raise UnavailableVideoError
+                               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                                       self.trouble(u'ERROR: unable to download video data: %s' % str(err))
+                                       return
+                               except (ContentTooShortError, ), err:
+                                       self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                                       return
+       
+                       if success:
+                               try:
+                                       self.post_process(filename, info_dict)
+                               except (PostProcessingError), err:
+                                       self.trouble(u'ERROR: postprocessing: %s' % str(err))
+                                       return
 
        def download(self, url_list):
                """Download a given list of URLs."""
 
        def download(self, url_list):
                """Download a given list of URLs."""
@@ -837,13 +900,21 @@ class FileDownloader(object):
                # the connection was interrumpted and resuming appears to be
                # possible. This is part of rtmpdump's normal usage, AFAIK.
                basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
                # the connection was interrumpted and resuming appears to be
                # possible. This is part of rtmpdump's normal usage, AFAIK.
                basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
-               retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
+               args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
+               if self.params.get('verbose', False):
+                       try:
+                               import pipes
+                               shell_quote = lambda args: ' '.join(map(pipes.quote, args))
+                       except ImportError:
+                               shell_quote = repr
+                       self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
+               retval = subprocess.call(args)
                while retval == 2 or retval == 1:
                while retval == 2 or retval == 1:
-                       prevsize = os.path.getsize(tmpfilename)
+                       prevsize = os.path.getsize(_encodeFilename(tmpfilename))
                        self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
                        time.sleep(5.0) # This seems to be needed
                        retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
                        self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
                        time.sleep(5.0) # This seems to be needed
                        retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
-                       cursize = os.path.getsize(tmpfilename)
+                       cursize = os.path.getsize(_encodeFilename(tmpfilename))
                        if prevsize == cursize and retval == 1:
                                break
                         # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
                        if prevsize == cursize and retval == 1:
                                break
                         # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
@@ -852,16 +923,19 @@ class FileDownloader(object):
                                retval = 0
                                break
                if retval == 0:
                                retval = 0
                                break
                if retval == 0:
-                       self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
+                       self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(_encodeFilename(tmpfilename)))
                        self.try_rename(tmpfilename, filename)
                        return True
                else:
                        self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
                        return False
 
                        self.try_rename(tmpfilename, filename)
                        return True
                else:
                        self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
                        return False
 
-       def _do_download(self, filename, url, player_url):
+       def _do_download(self, filename, info_dict):
+               url = info_dict['url']
+               player_url = info_dict.get('player_url', None)
+
                # Check file already present
                # Check file already present
-               if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
+               if self.params.get('continuedl', False) and os.path.isfile(_encodeFilename(filename)) and not self.params.get('nopart', False):
                        self.report_file_already_downloaded(filename)
                        return True
 
                        self.report_file_already_downloaded(filename)
                        return True
 
@@ -871,7 +945,6 @@ class FileDownloader(object):
 
                tmpfilename = self.temp_name(filename)
                stream = None
 
                tmpfilename = self.temp_name(filename)
                stream = None
-               open_mode = 'wb'
 
                # Do not include the Accept-Encoding header
                headers = {'Youtubedl-no-compression': 'True'}
 
                # Do not include the Accept-Encoding header
                headers = {'Youtubedl-no-compression': 'True'}
@@ -879,22 +952,27 @@ class FileDownloader(object):
                request = urllib2.Request(url, None, headers)
 
                # Establish possible resume length
                request = urllib2.Request(url, None, headers)
 
                # Establish possible resume length
-               if os.path.isfile(tmpfilename):
-                       resume_len = os.path.getsize(tmpfilename)
+               if os.path.isfile(_encodeFilename(tmpfilename)):
+                       resume_len = os.path.getsize(_encodeFilename(tmpfilename))
                else:
                        resume_len = 0
 
                else:
                        resume_len = 0
 
-               # Request parameters in case of being able to resume
-               if self.params.get('continuedl', False) and resume_len != 0:
-                       self.report_resuming_byte(resume_len)
-                       request.add_header('Range', 'bytes=%d-' % resume_len)
-                       open_mode = 'ab'
+               open_mode = 'wb'
+               if resume_len != 0:
+                       if self.params.get('continuedl', False):
+                               self.report_resuming_byte(resume_len)
+                               request.add_header('Range','bytes=%d-' % resume_len)
+                               open_mode = 'ab'
+                       else:
+                               resume_len = 0
 
                count = 0
                retries = self.params.get('retries', 0)
                while count <= retries:
                        # Establish connection
                        try:
 
                count = 0
                retries = self.params.get('retries', 0)
                while count <= retries:
                        # Establish connection
                        try:
+                               if count == 0 and 'urlhandle' in info_dict:
+                                       data = info_dict['urlhandle']
                                data = urllib2.urlopen(request)
                                break
                        except (urllib2.HTTPError, ), err:
                                data = urllib2.urlopen(request)
                                break
                        except (urllib2.HTTPError, ), err:
@@ -972,10 +1050,13 @@ class FileDownloader(object):
                        block_size = self.best_block_size(after - before, len(data_block))
 
                        # Progress message
                        block_size = self.best_block_size(after - before, len(data_block))
 
                        # Progress message
-                       percent_str = self.calc_percent(byte_counter, data_len)
-                       eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
                        speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
                        speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
-                       self.report_progress(percent_str, data_len_str, speed_str, eta_str)
+                       if data_len is None:
+                               self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
+                       else:
+                               percent_str = self.calc_percent(byte_counter, data_len)
+                               eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
+                               self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 
                        # Apply rate limit
                        self.slow_down(start, byte_counter - resume_len)
 
                        # Apply rate limit
                        self.slow_down(start, byte_counter - resume_len)
@@ -991,7 +1072,7 @@ class FileDownloader(object):
 
                # Update file modification time
                if self.params.get('updatetime', True):
 
                # Update file modification time
                if self.params.get('updatetime', True):
-                       self.try_utime(filename, data.info().get('last-modified', None))
+                       info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 
                return True
 
 
                return True
 
@@ -1026,9 +1107,8 @@ class InfoExtractor(object):
        description:    One-line video description.
 
        Subclasses of this one should re-define the _real_initialize() and
        description:    One-line video description.
 
        Subclasses of this one should re-define the _real_initialize() and
-       _real_extract() methods, as well as the suitable() static method.
-       Probably, they should also be instantiated and added to the main
-       downloader.
+       _real_extract() methods and define a _VALID_URL regexp.
+       Probably, they should also be added to the list of extractors.
        """
 
        _ready = False
        """
 
        _ready = False
@@ -1039,10 +1119,9 @@ class InfoExtractor(object):
                self._ready = False
                self.set_downloader(downloader)
 
                self._ready = False
                self.set_downloader(downloader)
 
-       @staticmethod
-       def suitable(url):
+       def suitable(self, url):
                """Receives a URL and returns True if suitable for this IE."""
                """Receives a URL and returns True if suitable for this IE."""
-               return False
+               return re.match(self._VALID_URL, url) is not None
 
        def initialize(self):
                """Initializes an instance (authentication, etc)."""
 
        def initialize(self):
                """Initializes an instance (authentication, etc)."""
@@ -1071,13 +1150,14 @@ class InfoExtractor(object):
 class YoutubeIE(InfoExtractor):
        """Information extractor for youtube.com."""
 
 class YoutubeIE(InfoExtractor):
        """Information extractor for youtube.com."""
 
-       _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
+       _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
        _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
        _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
        _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
        _NETRC_MACHINE = 'youtube'
        # Listed in order of quality
        _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
        _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
        _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
        _NETRC_MACHINE = 'youtube'
        # Listed in order of quality
-       _available_formats = ['38', '37', '45', '22', '43', '35', '34', '18', '6', '5', '17', '13']
+       _available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
+       _available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
        _video_extensions = {
                '13': '3gp',
                '17': 'mp4',
        _video_extensions = {
                '13': '3gp',
                '17': 'mp4',
@@ -1086,12 +1166,25 @@ class YoutubeIE(InfoExtractor):
                '37': 'mp4',
                '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
                '43': 'webm',
                '37': 'mp4',
                '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
                '43': 'webm',
+               '44': 'webm',
                '45': 'webm',
        }
                '45': 'webm',
        }
-
-       @staticmethod
-       def suitable(url):
-               return (re.match(YoutubeIE._VALID_URL, url) is not None)
+       _video_dimensions = {
+               '5': '240x400',
+               '6': '???',
+               '13': '???',
+               '17': '144x176',
+               '18': '360x640',
+               '22': '720x1280',
+               '34': '360x640',
+               '35': '480x854',
+               '37': '1080x1920',
+               '38': '3072x4096',
+               '43': '360x640',
+               '44': '480x854',
+               '45': '720x1280',
+       }       
+       IE_NAME = u'youtube'
 
        def report_lang(self):
                """Report attempt to set language."""
 
        def report_lang(self):
                """Report attempt to set language."""
@@ -1125,6 +1218,11 @@ class YoutubeIE(InfoExtractor):
                """Indicate the download will use the RTMP protocol."""
                self._downloader.to_screen(u'[youtube] RTMP download detected')
 
                """Indicate the download will use the RTMP protocol."""
                self._downloader.to_screen(u'[youtube] RTMP download detected')
 
+       def _print_formats(self, formats):
+               print 'Available formats:'
+               for x in formats:
+                       print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))
+
        def _real_initialize(self):
                if self._downloader is None:
                        return
        def _real_initialize(self):
                if self._downloader is None:
                        return
@@ -1204,7 +1302,7 @@ class YoutubeIE(InfoExtractor):
 
                # Get video webpage
                self.report_video_webpage_download(video_id)
 
                # Get video webpage
                self.report_video_webpage_download(video_id)
-               request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&amp;has_verified=1' % video_id)
+               request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
                try:
                        video_webpage = urllib2.urlopen(request).read()
                except (urllib2.URLError, httplib.HTTPException, socket.error), err:
                try:
                        video_webpage = urllib2.urlopen(request).read()
                except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@@ -1257,8 +1355,7 @@ class YoutubeIE(InfoExtractor):
                video_title = sanitize_title(video_title)
 
                # simplified title
                video_title = sanitize_title(video_title)
 
                # simplified title
-               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
-               simple_title = simple_title.strip(ur'_')
+               simple_title = _simplify_title(video_title)
 
                # thumbnail image
                if 'thumbnail_url' not in video_info:
 
                # thumbnail image
                if 'thumbnail_url' not in video_info:
@@ -1284,10 +1381,9 @@ class YoutubeIE(InfoExtractor):
                        lxml.etree
                except NameError:
                        video_description = u'No description available.'
                        lxml.etree
                except NameError:
                        video_description = u'No description available.'
-                       if self._downloader.params.get('forcedescription', False) or self._downloader.params.get('writedescription', False):
-                               mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
-                               if mobj is not None:
-                                       video_description = mobj.group(1).decode('utf-8')
+                       mobj = re.search(r'<meta name="description" content="(.*?)">', video_webpage)
+                       if mobj is not None:
+                               video_description = mobj.group(1).decode('utf-8')
                else:
                        html_parser = lxml.etree.HTMLParser(encoding='utf-8')
                        vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
                else:
                        html_parser = lxml.etree.HTMLParser(encoding='utf-8')
                        vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
@@ -1310,24 +1406,36 @@ class YoutubeIE(InfoExtractor):
                        url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
 
                        format_limit = self._downloader.params.get('format_limit', None)
                        url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
 
                        format_limit = self._downloader.params.get('format_limit', None)
-                       if format_limit is not None and format_limit in self._available_formats:
-                               format_list = self._available_formats[self._available_formats.index(format_limit):]
+                       available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
+                       if format_limit is not None and format_limit in available_formats:
+                               format_list = available_formats[available_formats.index(format_limit):]
                        else:
                        else:
-                               format_list = self._available_formats
+                               format_list = available_formats
                        existing_formats = [x for x in format_list if x in url_map]
                        if len(existing_formats) == 0:
                                self._downloader.trouble(u'ERROR: no known formats available for video')
                                return
                        existing_formats = [x for x in format_list if x in url_map]
                        if len(existing_formats) == 0:
                                self._downloader.trouble(u'ERROR: no known formats available for video')
                                return
-                       if req_format is None:
+                       if self._downloader.params.get('listformats', None):
+                               self._print_formats(existing_formats)
+                               return
+                       if req_format is None or req_format == 'best':
                                video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
                                video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
-                       elif req_format == '-1':
+                       elif req_format == 'worst':
+                               video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
+                       elif req_format in ('-1', 'all'):
                                video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
                        else:
                                video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
                        else:
-                               # Specific format
-                               if req_format not in url_map:
+                               # Specific formats. We pick the first in a slash-delimeted sequence.
+                               # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
+                               req_formats = req_format.split('/')
+                               video_url_list = None
+                               for rf in req_formats:
+                                       if rf in url_map:
+                                               video_url_list = [(rf, url_map[rf])]
+                                               break
+                               if video_url_list is None:
                                        self._downloader.trouble(u'ERROR: requested format not available')
                                        return
                                        self._downloader.trouble(u'ERROR: requested format not available')
                                        return
-                               video_url_list = [(req_format, url_map[req_format])] # Specific format
                else:
                        self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
                        return
                else:
                        self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
                        return
@@ -1365,15 +1473,12 @@ class MetacafeIE(InfoExtractor):
        _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
        _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
        _youtube_ie = None
        _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
        _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
        _youtube_ie = None
+       IE_NAME = u'metacafe'
 
        def __init__(self, youtube_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._youtube_ie = youtube_ie
 
 
        def __init__(self, youtube_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._youtube_ie = youtube_ie
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(MetacafeIE._VALID_URL, url) is not None)
-
        def report_disclaimer(self):
                """Report disclaimer retrieval."""
                self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
        def report_disclaimer(self):
                """Report disclaimer retrieval."""
                self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
@@ -1507,14 +1612,11 @@ class DailymotionIE(InfoExtractor):
        """Information Extractor for Dailymotion"""
 
        _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
        """Information Extractor for Dailymotion"""
 
        _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
+       IE_NAME = u'dailymotion'
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(DailymotionIE._VALID_URL, url) is not None)
-
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
@@ -1523,9 +1625,6 @@ class DailymotionIE(InfoExtractor):
                """Report information extraction."""
                self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
 
                """Report information extraction."""
                self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
 
-       def _real_initialize(self):
-               return
-
        def _real_extract(self, url):
                # Extract id and simplified title from URL
                mobj = re.match(self._VALID_URL, url)
        def _real_extract(self, url):
                # Extract id and simplified title from URL
                mobj = re.match(self._VALID_URL, url)
@@ -1537,7 +1636,6 @@ class DailymotionIE(InfoExtractor):
                self._downloader.increment_downloads()
                video_id = mobj.group(1)
 
                self._downloader.increment_downloads()
                video_id = mobj.group(1)
 
-               simple_title = mobj.group(2).decode('utf-8')
                video_extension = 'flv'
 
                # Retrieve video webpage to extract further information
                video_extension = 'flv'
 
                # Retrieve video webpage to extract further information
@@ -1567,12 +1665,13 @@ class DailymotionIE(InfoExtractor):
 
                video_url = mediaURL
 
 
                video_url = mediaURL
 
-               mobj = re.search(r'(?im)<title>Dailymotion\s*-\s*(.+)\s*-\s*[^<]+?</title>', webpage)
+               mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
                if mobj is None:
                        self._downloader.trouble(u'ERROR: unable to extract title')
                        return
                if mobj is None:
                        self._downloader.trouble(u'ERROR: unable to extract title')
                        return
-               video_title = mobj.group(1).decode('utf-8')
+               video_title = _unescapeHTML(mobj.group('title').decode('utf-8'))
                video_title = sanitize_title(video_title)
                video_title = sanitize_title(video_title)
+               simple_title = _simplify_title(video_title)
 
                mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
                if mobj is None:
 
                mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
                if mobj is None:
@@ -1601,14 +1700,11 @@ class GoogleIE(InfoExtractor):
        """Information extractor for video.google.com."""
 
        _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
        """Information extractor for video.google.com."""
 
        _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
+       IE_NAME = u'video.google'
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(GoogleIE._VALID_URL, url) is not None)
-
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
@@ -1617,9 +1713,6 @@ class GoogleIE(InfoExtractor):
                """Report information extraction."""
                self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
 
                """Report information extraction."""
                self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
 
-       def _real_initialize(self):
-               return
-
        def _real_extract(self, url):
                # Extract id from URL
                mobj = re.match(self._VALID_URL, url)
        def _real_extract(self, url):
                # Extract id from URL
                mobj = re.match(self._VALID_URL, url)
@@ -1663,7 +1756,7 @@ class GoogleIE(InfoExtractor):
                        return
                video_title = mobj.group(1).decode('utf-8')
                video_title = sanitize_title(video_title)
                        return
                video_title = mobj.group(1).decode('utf-8')
                video_title = sanitize_title(video_title)
-               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
+               simple_title = _simplify_title(video_title)
 
                # Extract video description
                mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
 
                # Extract video description
                mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
@@ -1711,14 +1804,11 @@ class PhotobucketIE(InfoExtractor):
        """Information extractor for photobucket.com."""
 
        _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
        """Information extractor for photobucket.com."""
 
        _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
+       IE_NAME = u'photobucket'
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(PhotobucketIE._VALID_URL, url) is not None)
-
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
@@ -1727,9 +1817,6 @@ class PhotobucketIE(InfoExtractor):
                """Report information extraction."""
                self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
 
                """Report information extraction."""
                self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
 
-       def _real_initialize(self):
-               return
-
        def _real_extract(self, url):
                # Extract id from URL
                mobj = re.match(self._VALID_URL, url)
        def _real_extract(self, url):
                # Extract id from URL
                mobj = re.match(self._VALID_URL, url)
@@ -1768,7 +1855,7 @@ class PhotobucketIE(InfoExtractor):
                        return
                video_title = mobj.group(1).decode('utf-8')
                video_title = sanitize_title(video_title)
                        return
                video_title = mobj.group(1).decode('utf-8')
                video_title = sanitize_title(video_title)
-               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
+               simple_title = _simplify_title(vide_title)
 
                video_uploader = mobj.group(2).decode('utf-8')
 
 
                video_uploader = mobj.group(2).decode('utf-8')
 
@@ -1796,14 +1883,11 @@ class YahooIE(InfoExtractor):
        # _VPAGE_URL matches only the extractable '/watch/' URLs
        _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
        _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
        # _VPAGE_URL matches only the extractable '/watch/' URLs
        _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
        _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
+       IE_NAME = u'video.yahoo'
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(YahooIE._VALID_URL, url) is not None)
-
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
@@ -1812,9 +1896,6 @@ class YahooIE(InfoExtractor):
                """Report information extraction."""
                self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
 
                """Report information extraction."""
                self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
 
-       def _real_initialize(self):
-               return
-
        def _real_extract(self, url, new_video=True):
                # Extract ID from URL
                mobj = re.match(self._VALID_URL, url)
        def _real_extract(self, url, new_video=True):
                # Extract ID from URL
                mobj = re.match(self._VALID_URL, url)
@@ -1868,7 +1949,7 @@ class YahooIE(InfoExtractor):
                        self._downloader.trouble(u'ERROR: unable to extract video title')
                        return
                video_title = mobj.group(1).decode('utf-8')
                        self._downloader.trouble(u'ERROR: unable to extract video title')
                        return
                video_title = mobj.group(1).decode('utf-8')
-               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
+               simple_title = _simplify_title(video_title)
 
                mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
                if mobj is None:
 
                mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
                if mobj is None:
@@ -1952,14 +2033,11 @@ class VimeoIE(InfoExtractor):
 
        # _VALID_URL matches Vimeo URLs
        _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)'
 
        # _VALID_URL matches Vimeo URLs
        _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)'
+       IE_NAME = u'vimeo'
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(VimeoIE._VALID_URL, url) is not None)
-
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
@@ -1968,9 +2046,6 @@ class VimeoIE(InfoExtractor):
                """Report information extraction."""
                self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
 
                """Report information extraction."""
                self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
 
-       def _real_initialize(self):
-               return
-
        def _real_extract(self, url, new_video=True):
                # Extract ID from URL
                mobj = re.match(self._VALID_URL, url)
        def _real_extract(self, url, new_video=True):
                # Extract ID from URL
                mobj = re.match(self._VALID_URL, url)
@@ -1983,7 +2058,7 @@ class VimeoIE(InfoExtractor):
                video_id = mobj.group(1)
 
                # Retrieve video webpage to extract further information
                video_id = mobj.group(1)
 
                # Retrieve video webpage to extract further information
-               request = urllib2.Request("http://vimeo.com/moogaloop/load/clip:%s" % video_id, None, std_headers)
+               request = urllib2.Request(url, None, std_headers)
                try:
                        self.report_download_webpage(video_id)
                        webpage = urllib2.urlopen(request).read()
                try:
                        self.report_download_webpage(video_id)
                        webpage = urllib2.urlopen(request).read()
@@ -1996,65 +2071,66 @@ class VimeoIE(InfoExtractor):
                # and latter we extract those that are Vimeo specific.
                self.report_extraction(video_id)
 
                # and latter we extract those that are Vimeo specific.
                self.report_extraction(video_id)
 
-               # Extract title
-               mobj = re.search(r'<caption>(.*?)</caption>', webpage)
-               if mobj is None:
-                       self._downloader.trouble(u'ERROR: unable to extract video title')
+               # Extract the config JSON
+               config = webpage.split(' = {config:')[1].split(',assets:')[0]
+               try:
+                       config = json.loads(config)
+               except:
+                       self._downloader.trouble(u'ERROR: unable to extract info section')
                        return
                        return
-               video_title = mobj.group(1).decode('utf-8')
-               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
+               
+               # Extract title
+               video_title = config["video"]["title"]
+               simple_title = _simplify_title(video_title)
 
                # Extract uploader
 
                # Extract uploader
-               mobj = re.search(r'<uploader_url>http://vimeo.com/(.*?)</uploader_url>', webpage)
-               if mobj is None:
-                       self._downloader.trouble(u'ERROR: unable to extract video uploader')
-                       return
-               video_uploader = mobj.group(1).decode('utf-8')
+               video_uploader = config["video"]["owner"]["name"]
 
                # Extract video thumbnail
 
                # Extract video thumbnail
-               mobj = re.search(r'<thumbnail>(.*?)</thumbnail>', webpage)
-               if mobj is None:
-                       self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
-                       return
-               video_thumbnail = mobj.group(1).decode('utf-8')
+               video_thumbnail = config["video"]["thumbnail"]
 
 
-               # # Extract video description
-               # mobj = re.search(r'<meta property="og:description" content="(.*)" />', webpage)
-               # if mobj is None:
-               #       self._downloader.trouble(u'ERROR: unable to extract video description')
-               #       return
-               # video_description = mobj.group(1).decode('utf-8')
-               # if not video_description: video_description = 'No description available.'
-               video_description = 'Foo.'
-
-               # Vimeo specific: extract request signature
-               mobj = re.search(r'<request_signature>(.*?)</request_signature>', webpage)
-               if mobj is None:
-                       self._downloader.trouble(u'ERROR: unable to extract request signature')
-                       return
-               sig = mobj.group(1).decode('utf-8')
+               # Extract video description
+               try:
+                       lxml.etree
+               except NameError:
+                       video_description = u'No description available.'
+                       mobj = re.search(r'<meta name="description" content="(.*?)" />', webpage, re.MULTILINE)
+                       if mobj is not None:
+                               video_description = mobj.group(1)
+               else:
+                       html_parser = lxml.etree.HTMLParser()
+                       vwebpage_doc = lxml.etree.parse(StringIO.StringIO(webpage), html_parser)
+                       video_description = u''.join(vwebpage_doc.xpath('id("description")//text()')).strip()
+                       # TODO use another parser
 
 
-               # Vimeo specific: Extract request signature expiration
-               mobj = re.search(r'<request_signature_expires>(.*?)</request_signature_expires>', webpage)
-               if mobj is None:
-                       self._downloader.trouble(u'ERROR: unable to extract request signature expiration')
-                       return
-               sig_exp = mobj.group(1).decode('utf-8')
+               # Extract upload date
+               video_upload_date = u'NA'
+               mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
+               if mobj is not None:
+                       video_upload_date = mobj.group(1)
+
+               # Vimeo specific: extract request signature and timestamp
+               sig = config['request']['signature']
+               timestamp = config['request']['timestamp']
 
 
-               video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s" % (video_id, sig, sig_exp)
+               # Vimeo specific: extract video quality information
+               # TODO bind to format param
+               if 'hd' in config["video"]["files"]["h264"]: quality = 'hd'
+               else: quality = 'sd'
+
+               video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=H264&type=moogaloop_local&embed_location=" \
+                                       %(video_id, sig, timestamp, quality)
 
                try:
                        # Process video information
                        self._downloader.process_info({
 
                try:
                        # Process video information
                        self._downloader.process_info({
-                               'id':           video_id.decode('utf-8'),
+                               'id':           video_id,
                                'url':          video_url,
                                'uploader':     video_uploader,
                                'url':          video_url,
                                'uploader':     video_uploader,
-                               'upload_date':  u'NA',
+                               'upload_date':  video_upload_date,
                                'title':        video_title,
                                'stitle':       simple_title,
                                'ext':          u'mp4',
                                'title':        video_title,
                                'stitle':       simple_title,
                                'ext':          u'mp4',
-                               'thumbnail':    video_thumbnail.decode('utf-8'),
-                               'description':  video_description,
                                'thumbnail':    video_thumbnail,
                                'description':  video_description,
                                'player_url':   None,
                                'thumbnail':    video_thumbnail,
                                'description':  video_description,
                                'player_url':   None,
@@ -2066,13 +2142,12 @@ class VimeoIE(InfoExtractor):
 class GenericIE(InfoExtractor):
        """Generic last-resort information extractor."""
 
 class GenericIE(InfoExtractor):
        """Generic last-resort information extractor."""
 
+       _VALID_URL = r'.*'
+       IE_NAME = u'generic'
+
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
-       @staticmethod
-       def suitable(url):
-               return True
-
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
@@ -2082,9 +2157,6 @@ class GenericIE(InfoExtractor):
                """Report information extraction."""
                self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
 
                """Report information extraction."""
                self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
 
-       def _real_initialize(self):
-               return
-
        def _real_extract(self, url):
                # At this point we have a new video
                self._downloader.increment_downloads()
        def _real_extract(self, url):
                # At this point we have a new video
                self._downloader.increment_downloads()
@@ -2138,7 +2210,7 @@ class GenericIE(InfoExtractor):
                        return
                video_title = mobj.group(1).decode('utf-8')
                video_title = sanitize_title(video_title)
                        return
                video_title = mobj.group(1).decode('utf-8')
                video_title = sanitize_title(video_title)
-               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
+               simple_title = _simplify_title(video_title)
 
                # video uploader is domain name
                mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
 
                # video uploader is domain name
                mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
@@ -2166,21 +2238,18 @@ class GenericIE(InfoExtractor):
 
 class YoutubeSearchIE(InfoExtractor):
        """Information Extractor for YouTube search queries."""
 
 class YoutubeSearchIE(InfoExtractor):
        """Information Extractor for YouTube search queries."""
-       _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
+       _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
        _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
        _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
        _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
        _youtube_ie = None
        _max_youtube_results = 1000
        _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
        _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
        _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
        _youtube_ie = None
        _max_youtube_results = 1000
+       IE_NAME = u'youtube:search'
 
        def __init__(self, youtube_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._youtube_ie = youtube_ie
 
 
        def __init__(self, youtube_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._youtube_ie = youtube_ie
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
-
        def report_download_page(self, query, pagenum):
                """Report attempt to download playlist page with given number."""
                query = query.decode(preferredencoding())
        def report_download_page(self, query, pagenum):
                """Report attempt to download playlist page with given number."""
                query = query.decode(preferredencoding())
@@ -2190,7 +2259,7 @@ class YoutubeSearchIE(InfoExtractor):
                self._youtube_ie.initialize()
 
        def _real_extract(self, query):
                self._youtube_ie.initialize()
 
        def _real_extract(self, query):
-               mobj = re.match(self._VALID_QUERY, query)
+               mobj = re.match(self._VALID_URL, query)
                if mobj is None:
                        self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
                        return
                if mobj is None:
                        self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
                        return
@@ -2258,21 +2327,18 @@ class YoutubeSearchIE(InfoExtractor):
 
 class GoogleSearchIE(InfoExtractor):
        """Information Extractor for Google Video search queries."""
 
 class GoogleSearchIE(InfoExtractor):
        """Information Extractor for Google Video search queries."""
-       _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
+       _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
        _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
        _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
-       _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
-       _MORE_PAGES_INDICATOR = r'<span>Next</span>'
+       _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
+       _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
        _google_ie = None
        _max_google_results = 1000
        _google_ie = None
        _max_google_results = 1000
+       IE_NAME = u'video.google:search'
 
        def __init__(self, google_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._google_ie = google_ie
 
 
        def __init__(self, google_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._google_ie = google_ie
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
-
        def report_download_page(self, query, pagenum):
                """Report attempt to download playlist page with given number."""
                query = query.decode(preferredencoding())
        def report_download_page(self, query, pagenum):
                """Report attempt to download playlist page with given number."""
                query = query.decode(preferredencoding())
@@ -2282,7 +2348,7 @@ class GoogleSearchIE(InfoExtractor):
                self._google_ie.initialize()
 
        def _real_extract(self, query):
                self._google_ie.initialize()
 
        def _real_extract(self, query):
-               mobj = re.match(self._VALID_QUERY, query)
+               mobj = re.match(self._VALID_URL, query)
                if mobj is None:
                        self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
                        return
                if mobj is None:
                        self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
                        return
@@ -2315,12 +2381,11 @@ class GoogleSearchIE(InfoExtractor):
                """Downloads a specified number of results for a query"""
 
                video_ids = []
                """Downloads a specified number of results for a query"""
 
                video_ids = []
-               already_seen = set()
-               pagenum = 1
+               pagenum = 0
 
                while True:
                        self.report_download_page(query, pagenum)
 
                while True:
                        self.report_download_page(query, pagenum)
-                       result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
+                       result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum*10)
                        request = urllib2.Request(result_url)
                        try:
                                page = urllib2.urlopen(request).read()
                        request = urllib2.Request(result_url)
                        try:
                                page = urllib2.urlopen(request).read()
@@ -2331,9 +2396,8 @@ class GoogleSearchIE(InfoExtractor):
                        # Extract video identifiers
                        for mobj in re.finditer(self._VIDEO_INDICATOR, page):
                                video_id = mobj.group(1)
                        # Extract video identifiers
                        for mobj in re.finditer(self._VIDEO_INDICATOR, page):
                                video_id = mobj.group(1)
-                               if video_id not in already_seen:
+                               if video_id not in video_ids:
                                        video_ids.append(video_id)
                                        video_ids.append(video_id)
-                                       already_seen.add(video_id)
                                        if len(video_ids) == n:
                                                # Specified n videos reached
                                                for id in video_ids:
                                        if len(video_ids) == n:
                                                # Specified n videos reached
                                                for id in video_ids:
@@ -2350,21 +2414,18 @@ class GoogleSearchIE(InfoExtractor):
 
 class YahooSearchIE(InfoExtractor):
        """Information Extractor for Yahoo! Video search queries."""
 
 class YahooSearchIE(InfoExtractor):
        """Information Extractor for Yahoo! Video search queries."""
-       _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
+       _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
        _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
        _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
        _MORE_PAGES_INDICATOR = r'\s*Next'
        _yahoo_ie = None
        _max_yahoo_results = 1000
        _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
        _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
        _MORE_PAGES_INDICATOR = r'\s*Next'
        _yahoo_ie = None
        _max_yahoo_results = 1000
+       IE_NAME = u'video.yahoo:search'
 
        def __init__(self, yahoo_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._yahoo_ie = yahoo_ie
 
 
        def __init__(self, yahoo_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._yahoo_ie = yahoo_ie
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
-
        def report_download_page(self, query, pagenum):
                """Report attempt to download playlist page with given number."""
                query = query.decode(preferredencoding())
        def report_download_page(self, query, pagenum):
                """Report attempt to download playlist page with given number."""
                query = query.decode(preferredencoding())
@@ -2374,7 +2435,7 @@ class YahooSearchIE(InfoExtractor):
                self._yahoo_ie.initialize()
 
        def _real_extract(self, query):
                self._yahoo_ie.initialize()
 
        def _real_extract(self, query):
-               mobj = re.match(self._VALID_QUERY, query)
+               mobj = re.match(self._VALID_URL, query)
                if mobj is None:
                        self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
                        return
                if mobj is None:
                        self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
                        return
@@ -2443,20 +2504,17 @@ class YahooSearchIE(InfoExtractor):
 class YoutubePlaylistIE(InfoExtractor):
        """Information Extractor for YouTube playlists."""
 
 class YoutubePlaylistIE(InfoExtractor):
        """Information Extractor for YouTube playlists."""
 
-       _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+       _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
        _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
        _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
        _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
        _youtube_ie = None
        _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
        _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
        _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
        _youtube_ie = None
+       IE_NAME = u'youtube:playlist'
 
        def __init__(self, youtube_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._youtube_ie = youtube_ie
 
 
        def __init__(self, youtube_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._youtube_ie = youtube_ie
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
-
        def report_download_page(self, playlist_id, pagenum):
                """Report attempt to download playlist page with given number."""
                self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
        def report_download_page(self, playlist_id, pagenum):
                """Report attempt to download playlist page with given number."""
                self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
@@ -2490,7 +2548,8 @@ class YoutubePlaylistIE(InfoExtractor):
 
                while True:
                        self.report_download_page(playlist_id, pagenum)
 
                while True:
                        self.report_download_page(playlist_id, pagenum)
-                       request = urllib2.Request(self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum))
+                       url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
+                       request = urllib2.Request(url)
                        try:
                                page = urllib2.urlopen(request).read()
                        except (urllib2.URLError, httplib.HTTPException, socket.error), err:
                        try:
                                page = urllib2.urlopen(request).read()
                        except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@@ -2520,21 +2579,18 @@ class YoutubePlaylistIE(InfoExtractor):
 class YoutubeUserIE(InfoExtractor):
        """Information Extractor for YouTube users."""
 
 class YoutubeUserIE(InfoExtractor):
        """Information Extractor for YouTube users."""
 
-       _VALID_URL = r'(?:(?:(?:http://)?(?:\w+\.)?youtube.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
+       _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
        _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
        _GDATA_PAGE_SIZE = 50
        _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
        _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
        _GDATA_PAGE_SIZE = 50
        _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
-       _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
+       _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
        _youtube_ie = None
        _youtube_ie = None
+       IE_NAME = u'youtube:user'
 
        def __init__(self, youtube_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._youtube_ie = youtube_ie
 
 
        def __init__(self, youtube_ie, downloader=None):
                InfoExtractor.__init__(self, downloader)
                self._youtube_ie = youtube_ie
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
-
        def report_download_page(self, username, start_index):
                """Report attempt to download user page."""
                self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
        def report_download_page(self, username, start_index):
                """Report attempt to download user page."""
                self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
@@ -2601,7 +2657,7 @@ class YoutubeUserIE(InfoExtractor):
                else:
                        video_ids = video_ids[playliststart:playlistend]
 
                else:
                        video_ids = video_ids[playliststart:playlistend]
 
-               self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" %
+               self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
                                (username, all_ids_count, len(video_ids)))
 
                for video_id in video_ids:
                                (username, all_ids_count, len(video_ids)))
 
                for video_id in video_ids:
@@ -2611,15 +2667,12 @@ class YoutubeUserIE(InfoExtractor):
 class DepositFilesIE(InfoExtractor):
        """Information extractor for depositfiles.com"""
 
 class DepositFilesIE(InfoExtractor):
        """Information extractor for depositfiles.com"""
 
-       _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)'
+       _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
+       IE_NAME = u'DepositFiles'
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(DepositFilesIE._VALID_URL, url) is not None)
-
        def report_download_webpage(self, file_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
        def report_download_webpage(self, file_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
@@ -2628,9 +2681,6 @@ class DepositFilesIE(InfoExtractor):
                """Report information extraction."""
                self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
 
                """Report information extraction."""
                self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
 
-       def _real_initialize(self):
-               return
-
        def _real_extract(self, url):
                # At this point we have a new file
                self._downloader.increment_downloads()
        def _real_extract(self, url):
                # At this point we have a new file
                self._downloader.increment_downloads()
@@ -2691,22 +2741,20 @@ class DepositFilesIE(InfoExtractor):
 class FacebookIE(InfoExtractor):
        """Information Extractor for Facebook"""
 
 class FacebookIE(InfoExtractor):
        """Information Extractor for Facebook"""
 
-       _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook.com/video/video.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
+       _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
        _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
        _NETRC_MACHINE = 'facebook'
        _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
        _NETRC_MACHINE = 'facebook'
-       _available_formats = ['highqual', 'lowqual']
+       _available_formats = ['video', 'highqual', 'lowqual']
        _video_extensions = {
        _video_extensions = {
+               'video': 'mp4',
                'highqual': 'mp4',
                'lowqual': 'mp4',
        }
                'highqual': 'mp4',
                'lowqual': 'mp4',
        }
+       IE_NAME = u'facebook'
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
 
-       @staticmethod
-       def suitable(url):
-               return (re.match(FacebookIE._VALID_URL, url) is not None)
-
        def _reporter(self, message):
                """Add header and report message."""
                self._downloader.to_screen(u'[facebook] %s' % message)
        def _reporter(self, message):
                """Add header and report message."""
                self._downloader.to_screen(u'[facebook] %s' % message)
@@ -2726,10 +2774,9 @@ class FacebookIE(InfoExtractor):
        def _parse_page(self, video_webpage):
                """Extract video information from page"""
                # General data
        def _parse_page(self, video_webpage):
                """Extract video information from page"""
                # General data
-               data = {'title': r'class="video_title datawrap">(.*?)</',
+               data = {'title': r'\("video_title", "(.*?)"\)',
                        'description': r'<div class="datawrap">(.*?)</div>',
                        'owner': r'\("video_owner_name", "(.*?)"\)',
                        'description': r'<div class="datawrap">(.*?)</div>',
                        'owner': r'\("video_owner_name", "(.*?)"\)',
-                       'upload_date': r'data-date="(.*?)"',
                        'thumbnail':  r'\("thumb_url", "(?P<THUMB>.*?)"\)',
                        }
                video_info = {}
                        'thumbnail':  r'\("thumb_url", "(?P<THUMB>.*?)"\)',
                        }
                video_info = {}
@@ -2831,9 +2878,7 @@ class FacebookIE(InfoExtractor):
                video_title = video_title.decode('utf-8')
                video_title = sanitize_title(video_title)
 
                video_title = video_title.decode('utf-8')
                video_title = sanitize_title(video_title)
 
-               # simplified title
-               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
-               simple_title = simple_title.strip(ur'_')
+               simple_title = _simplify_title(video_title)
 
                # thumbnail image
                if 'thumbnail' not in video_info:
 
                # thumbnail image
                if 'thumbnail' not in video_info:
@@ -2872,6 +2917,8 @@ class FacebookIE(InfoExtractor):
                                return
                        if req_format is None:
                                video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
                                return
                        if req_format is None:
                                video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
+                       elif req_format == 'worst':
+                               video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
                        elif req_format == '-1':
                                video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
                        else:
                        elif req_format == '-1':
                                video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
                        else:
@@ -2912,19 +2959,15 @@ class BlipTVIE(InfoExtractor):
 
        _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
        _URL_EXT = r'^.*\.([a-z0-9]+)$'
 
        _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
        _URL_EXT = r'^.*\.([a-z0-9]+)$'
-
-       @staticmethod
-       def suitable(url):
-               return (re.match(BlipTVIE._VALID_URL, url) is not None)
+       IE_NAME = u'blip.tv'
 
        def report_extraction(self, file_id):
                """Report information extraction."""
 
        def report_extraction(self, file_id):
                """Report information extraction."""
-               self._downloader.to_screen(u'[blip.tv] %s: Extracting information' % file_id)
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
 
 
-       def _simplify_title(self, title):
-               res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
-               res = res.strip(ur'_')
-               return res
+       def report_direct_download(self, title):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
 
        def _real_extract(self, url):
                mobj = re.match(self._VALID_URL, url)
 
        def _real_extract(self, url):
                mobj = re.match(self._VALID_URL, url)
@@ -2939,43 +2982,65 @@ class BlipTVIE(InfoExtractor):
                json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
                request = urllib2.Request(json_url)
                self.report_extraction(mobj.group(1))
                json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
                request = urllib2.Request(json_url)
                self.report_extraction(mobj.group(1))
+               info = None
                try:
                try:
-                       json_code = urllib2.urlopen(request).read()
+                       urlh = urllib2.urlopen(request)
+                       if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
+                               basename = url.split('/')[-1]
+                               title,ext = os.path.splitext(basename)
+                               title = title.decode('UTF-8')
+                               ext = ext.replace('.', '')
+                               self.report_direct_download(title)
+                               info = {
+                                       'id': title,
+                                       'url': url,
+                                       'title': title,
+                                       'stitle': _simplify_title(title),
+                                       'ext': ext,
+                                       'urlhandle': urlh
+                               }
                except (urllib2.URLError, httplib.HTTPException, socket.error), err:
                        self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
                        return
                except (urllib2.URLError, httplib.HTTPException, socket.error), err:
                        self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
                        return
-               try:
-                       json_data = json.loads(json_code)
-                       if 'Post' in json_data:
-                               data = json_data['Post']
-                       else:
-                               data = json_data
-
-                       upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
-                       video_url = data['media']['url']
-                       umobj = re.match(self._URL_EXT, video_url)
-                       if umobj is None:
-                               raise ValueError('Can not determine filename extension')
-                       ext = umobj.group(1)
+               if info is None: # Regular URL
+                       try:
+                               json_code = urlh.read()
+                       except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                               self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % str(err))
+                               return
 
 
-                       self._downloader.increment_downloads()
+                       try:
+                               json_data = json.loads(json_code)
+                               if 'Post' in json_data:
+                                       data = json_data['Post']
+                               else:
+                                       data = json_data
+       
+                               upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
+                               video_url = data['media']['url']
+                               umobj = re.match(self._URL_EXT, video_url)
+                               if umobj is None:
+                                       raise ValueError('Can not determine filename extension')
+                               ext = umobj.group(1)
+       
+                               info = {
+                                       'id': data['item_id'],
+                                       'url': video_url,
+                                       'uploader': data['display_name'],
+                                       'upload_date': upload_date,
+                                       'title': data['title'],
+                                       'stitle': _simplify_title(data['title']),
+                                       'ext': ext,
+                                       'format': data['media']['mimeType'],
+                                       'thumbnail': data['thumbnailUrl'],
+                                       'description': data['description'],
+                                       'player_url': data['embedUrl']
+                               }
+                       except (ValueError,KeyError), err:
+                               self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
+                               return
 
 
-                       info = {
-                               'id': data['item_id'],
-                               'url': video_url,
-                               'uploader': data['display_name'],
-                               'upload_date': upload_date,
-                               'title': data['title'],
-                               'stitle': self._simplify_title(data['title']),
-                               'ext': ext,
-                               'format': data['media']['mimeType'],
-                               'thumbnail': data['thumbnailUrl'],
-                               'description': data['description'],
-                               'player_url': data['embedUrl']
-                       }
-               except (ValueError,KeyError), err:
-                       self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
-                       return
+               self._downloader.increment_downloads()
 
                try:
                        self._downloader.process_info(info)
 
                try:
                        self._downloader.process_info(info)
@@ -2987,14 +3052,11 @@ class MyVideoIE(InfoExtractor):
        """Information Extractor for myvideo.de."""
 
        _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
        """Information Extractor for myvideo.de."""
 
        _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
+       IE_NAME = u'myvideo'
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
        
 
        def __init__(self, downloader=None):
                InfoExtractor.__init__(self, downloader)
        
-       @staticmethod
-       def suitable(url):
-               return (re.match(MyVideoIE._VALID_URL, url) is not None)
-
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
        def report_download_webpage(self, video_id):
                """Report webpage download."""
                self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
@@ -3003,9 +3065,6 @@ class MyVideoIE(InfoExtractor):
                """Report information extraction."""
                self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
 
                """Report information extraction."""
                self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
 
-       def _real_initialize(self):
-               return
-
        def _real_extract(self,url):
                mobj = re.match(self._VALID_URL, url)
                if mobj is None:
        def _real_extract(self,url):
                mobj = re.match(self._VALID_URL, url)
                if mobj is None:
@@ -3013,10 +3072,6 @@ class MyVideoIE(InfoExtractor):
                        return
 
                video_id = mobj.group(1)
                        return
 
                video_id = mobj.group(1)
-               simple_title = mobj.group(2).decode('utf-8')
-               # should actually not be necessary
-               simple_title = sanitize_title(simple_title)
-               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', simple_title)
 
                # Get video webpage
                request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
 
                # Get video webpage
                request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
@@ -3043,8 +3098,9 @@ class MyVideoIE(InfoExtractor):
                video_title = mobj.group(1)
                video_title = sanitize_title(video_title)
 
                video_title = mobj.group(1)
                video_title = sanitize_title(video_title)
 
+               simple_title = _simplify_title(video_title)
+
                try:
                try:
-                       print(video_url)
                        self._downloader.process_info({
                                'id':           video_id,
                                'url':          video_url,
                        self._downloader.process_info({
                                'id':           video_id,
                                'url':          video_url,
@@ -3062,11 +3118,8 @@ class MyVideoIE(InfoExtractor):
 class ComedyCentralIE(InfoExtractor):
        """Information extractor for The Daily Show and Colbert Report """
 
 class ComedyCentralIE(InfoExtractor):
        """Information extractor for The Daily Show and Colbert Report """
 
-       _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
-
-       @staticmethod
-       def suitable(url):
-               return (re.match(ComedyCentralIE._VALID_URL, url) is not None)
+       _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
+       IE_NAME = u'comedycentral'
 
        def report_extraction(self, episode_id):
                self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
 
        def report_extraction(self, episode_id):
                self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
@@ -3080,11 +3133,6 @@ class ComedyCentralIE(InfoExtractor):
        def report_player_url(self, episode_id):
                self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
 
        def report_player_url(self, episode_id):
                self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
 
-       def _simplify_title(self, title):
-               res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
-               res = res.strip(ur'_')
-               return res
-
        def _real_extract(self, url):
                mobj = re.match(self._VALID_URL, url)
                if mobj is None:
        def _real_extract(self, url):
                mobj = re.match(self._VALID_URL, url)
                if mobj is None:
@@ -3093,9 +3141,9 @@ class ComedyCentralIE(InfoExtractor):
 
                if mobj.group('shortname'):
                        if mobj.group('shortname') in ('tds', 'thedailyshow'):
 
                if mobj.group('shortname'):
                        if mobj.group('shortname') in ('tds', 'thedailyshow'):
-                               url = 'http://www.thedailyshow.com/full-episodes/'
+                               url = u'http://www.thedailyshow.com/full-episodes/'
                        else:
                        else:
-                               url = 'http://www.colbertnation.com/full-episodes/'
+                               url = u'http://www.colbertnation.com/full-episodes/'
                        mobj = re.match(self._VALID_URL, url)
                        assert mobj is not None
 
                        mobj = re.match(self._VALID_URL, url)
                        assert mobj is not None
 
@@ -3124,7 +3172,7 @@ class ComedyCentralIE(InfoExtractor):
                                return
                        epTitle = mobj.group('episode')
 
                                return
                        epTitle = mobj.group('episode')
 
-               mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"/>', html)
+               mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
                if len(mMovieParams) == 0:
                        self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
                        return
                if len(mMovieParams) == 0:
                        self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
                        return
@@ -3181,14 +3229,14 @@ class ComedyCentralIE(InfoExtractor):
 
                        self._downloader.increment_downloads()
 
 
                        self._downloader.increment_downloads()
 
-                       effTitle = showId + '-' + epTitle
+                       effTitle = showId + u'-' + epTitle
                        info = {
                                'id': shortMediaId,
                                'url': video_url,
                                'uploader': showId,
                                'upload_date': officialDate,
                                'title': effTitle,
                        info = {
                                'id': shortMediaId,
                                'url': video_url,
                                'uploader': showId,
                                'upload_date': officialDate,
                                'title': effTitle,
-                               'stitle': self._simplify_title(effTitle),
+                               'stitle': _simplify_title(effTitle),
                                'ext': 'mp4',
                                'format': format,
                                'thumbnail': None,
                                'ext': 'mp4',
                                'format': format,
                                'thumbnail': None,
@@ -3206,11 +3254,8 @@ class ComedyCentralIE(InfoExtractor):
 class EscapistIE(InfoExtractor):
        """Information extractor for The Escapist """
 
 class EscapistIE(InfoExtractor):
        """Information extractor for The Escapist """
 
-       _VALID_URL = r'^(https?://)?(www\.)escapistmagazine.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?].*$'
-
-       @staticmethod
-       def suitable(url):
-               return (re.match(EscapistIE._VALID_URL, url) is not None)
+       _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
+       IE_NAME = u'escapist'
 
        def report_extraction(self, showName):
                self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
 
        def report_extraction(self, showName):
                self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
@@ -3218,11 +3263,6 @@ class EscapistIE(InfoExtractor):
        def report_config_download(self, showName):
                self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
 
        def report_config_download(self, showName):
                self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
 
-       def _simplify_title(self, title):
-               res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
-               res = res.strip(ur'_')
-               return res
-
        def _real_extract(self, url):
                htmlParser = HTMLParser.HTMLParser()
 
        def _real_extract(self, url):
                htmlParser = HTMLParser.HTMLParser()
 
@@ -3275,7 +3315,7 @@ class EscapistIE(InfoExtractor):
                        'uploader': showName,
                        'upload_date': None,
                        'title': showName,
                        'uploader': showName,
                        'upload_date': None,
                        'title': showName,
-                       'stitle': self._simplify_title(showName),
+                       'stitle': _simplify_title(showName),
                        'ext': 'flv',
                        'format': 'flv',
                        'thumbnail': imgUrl,
                        'ext': 'flv',
                        'format': 'flv',
                        'thumbnail': imgUrl,
@@ -3289,43 +3329,706 @@ class EscapistIE(InfoExtractor):
                        self._downloader.trouble(u'\nERROR: unable to download ' + videoId)
 
 
                        self._downloader.trouble(u'\nERROR: unable to download ' + videoId)
 
 
+class CollegeHumorIE(InfoExtractor):
+       """Information extractor for collegehumor.com"""
 
 
-class PostProcessor(object):
-       """Post Processor class.
+       _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
+       IE_NAME = u'collegehumor'
 
 
-       PostProcessor objects can be added to downloaders with their
-       add_post_processor() method. When the downloader has finished a
-       successful download, it will take its internal chain of PostProcessors
-       and start calling the run() method on each one of them, first with
-       an initial argument and then with the returned value of the previous
-       PostProcessor.
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
 
 
-       The chain will be stopped if one of them ever returns None or the end
-       of the chain is reached.
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
 
 
-       PostProcessor objects follow a "mutual registration" process similar
-       to InfoExtractor objects.
-       """
+       def _real_extract(self, url):
+               htmlParser = HTMLParser.HTMLParser()
 
 
-       _downloader = None
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+               video_id = mobj.group('videoid')
 
 
-       def __init__(self, downloader=None):
-               self._downloader = downloader
+               self.report_webpage(video_id)
+               request = urllib2.Request(url)
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
 
 
-       def set_downloader(self, downloader):
-               """Sets the downloader for this PP."""
-               self._downloader = downloader
+               m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
+               if m is None:
+                       self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
+                       return
+               internal_video_id = m.group('internalvideoid')
 
 
-       def run(self, information):
-               """Run the PostProcessor.
+               info = {
+                       'id': video_id,
+                       'internal_id': internal_video_id,
+               }
 
 
-               The "information" argument is a dictionary like the ones
-               composed by InfoExtractors. The only difference is that this
-               one has an extra field called "filepath" that points to the
-               downloaded file.
+               self.report_extraction(video_id)
+               xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
+               try:
+                       metaXml = urllib2.urlopen(xmlUrl).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
+                       return
 
 
-               When this method returns None, the postprocessing chain is
-               stopped. However, this method may return an information
+               mdoc = xml.etree.ElementTree.fromstring(metaXml)
+               try:
+                       videoNode = mdoc.findall('./video')[0]
+                       info['description'] = videoNode.findall('./description')[0].text
+                       info['title'] = videoNode.findall('./caption')[0].text
+                       info['stitle'] = _simplify_title(info['title'])
+                       info['url'] = videoNode.findall('./file')[0].text
+                       info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
+                       info['ext'] = info['url'].rpartition('.')[2]
+                       info['format'] = info['ext']
+               except IndexError:
+                       self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
+                       return
+
+               self._downloader.increment_downloads()
+
+               try:
+                       self._downloader.process_info(info)
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'\nERROR: unable to download video')
+
+
+class XVideosIE(InfoExtractor):
+       """Information extractor for xvideos.com"""
+
+       _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
+       IE_NAME = u'xvideos'
+
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+       def _real_extract(self, url):
+               htmlParser = HTMLParser.HTMLParser()
+
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+               video_id = mobj.group(1).decode('utf-8')
+
+               self.report_webpage(video_id)
+
+               request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
+
+               self.report_extraction(video_id)
+
+
+               # Extract video URL
+               mobj = re.search(r'flv_url=(.+?)&', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video url')
+                       return
+               video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
+
+
+               # Extract title
+               mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video title')
+                       return
+               video_title = mobj.group(1).decode('utf-8')
+
+
+               # Extract video thumbnail
+               mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
+                       return
+               video_thumbnail = mobj.group(1).decode('utf-8')
+
+
+
+               self._downloader.increment_downloads()
+               info = {
+                       'id': video_id,
+                       'url': video_url,
+                       'uploader': None,
+                       'upload_date': None,
+                       'title': video_title,
+                       'stitle': _simplify_title(video_title),
+                       'ext': 'flv',
+                       'format': 'flv',
+                       'thumbnail': video_thumbnail,
+                       'description': None,
+                       'player_url': None,
+               }
+
+               try:
+                       self._downloader.process_info(info)
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
+
+
+class SoundcloudIE(InfoExtractor):
+       """Information extractor for soundcloud.com
+          To access the media, the uid of the song and a stream token
+          must be extracted from the page source and the script must make
+          a request to media.soundcloud.com/crossdomain.xml. Then
+          the media can be grabbed by requesting from an url composed
+          of the stream token and uid
+        """
+
+       _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
+       IE_NAME = u'soundcloud'
+
+       def __init__(self, downloader=None):
+               InfoExtractor.__init__(self, downloader)
+
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+       def _real_extract(self, url):
+               htmlParser = HTMLParser.HTMLParser()
+
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+
+               # extract uploader (which is in the url)
+               uploader = mobj.group(1).decode('utf-8')
+               # extract simple title (uploader + slug of song title)
+               slug_title =  mobj.group(2).decode('utf-8')
+               simple_title = uploader + '-' + slug_title
+
+               self.report_webpage('%s/%s' % (uploader, slug_title))
+
+               request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
+
+               self.report_extraction('%s/%s' % (uploader, slug_title))
+
+               # extract uid and stream token that soundcloud hands out for access
+               mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
+               if mobj:
+                       video_id = mobj.group(1)
+                       stream_token = mobj.group(2)
+
+               # extract unsimplified title
+               mobj = re.search('"title":"(.*?)",', webpage)
+               if mobj:
+                       title = mobj.group(1)
+
+               # construct media url (with uid/token)
+               mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
+               mediaURL = mediaURL % (video_id, stream_token)
+
+               # description
+               description = u'No description available'
+               mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
+               if mobj:
+                       description = mobj.group(1)
+               
+               # upload date
+               upload_date = None
+               mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
+               if mobj:
+                       try:
+                               upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
+                       except Exception, e:
+                               print str(e)
+
+               # for soundcloud, a request to a cross domain is required for cookies
+               request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
+
+               try:
+                       self._downloader.process_info({
+                               'id':           video_id.decode('utf-8'),
+                               'url':          mediaURL,
+                               'uploader':     uploader.decode('utf-8'),
+                               'upload_date':  upload_date,
+                               'title':        simple_title.decode('utf-8'),
+                               'stitle':       simple_title.decode('utf-8'),
+                               'ext':          u'mp3',
+                               'format':       u'NA',
+                               'player_url':   None,
+                               'description': description.decode('utf-8')
+                       })
+               except UnavailableVideoError:
+                       self._downloader.trouble(u'\nERROR: unable to download video')
+
+
+class InfoQIE(InfoExtractor):
+       """Information extractor for infoq.com"""
+
+       _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
+       IE_NAME = u'infoq'
+
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+       def _real_extract(self, url):
+               htmlParser = HTMLParser.HTMLParser()
+
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+
+               self.report_webpage(url)
+
+               request = urllib2.Request(url)
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
+
+               self.report_extraction(url)
+
+
+               # Extract video URL
+               mobj = re.search(r"jsclassref='([^']*)'", webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video url')
+                       return
+               video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
+
+
+               # Extract title
+               mobj = re.search(r'contentTitle = "(.*?)";', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video title')
+                       return
+               video_title = mobj.group(1).decode('utf-8')
+
+               # Extract description
+               video_description = u'No description available.'
+               mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
+               if mobj is not None:
+                       video_description = mobj.group(1).decode('utf-8')
+
+               video_filename = video_url.split('/')[-1]
+               video_id, extension = video_filename.split('.')
+
+               self._downloader.increment_downloads()
+               info = {
+                       'id': video_id,
+                       'url': video_url,
+                       'uploader': None,
+                       'upload_date': None,
+                       'title': video_title,
+                       'stitle': _simplify_title(video_title),
+                       'ext': extension,
+                       'format': extension, # Extension is always(?) mp4, but seems to be flv
+                       'thumbnail': None,
+                       'description': video_description,
+                       'player_url': None,
+               }
+
+               try:
+                       self._downloader.process_info(info)
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'\nERROR: unable to download ' + video_url)
+
+class MixcloudIE(InfoExtractor):
+       """Information extractor for www.mixcloud.com"""
+       _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
+       IE_NAME = u'mixcloud'
+
+       def __init__(self, downloader=None):
+               InfoExtractor.__init__(self, downloader)
+
+       def report_download_json(self, file_id):
+               """Report JSON download."""
+               self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
+
+       def report_extraction(self, file_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
+
+       def get_urls(self, jsonData, fmt, bitrate='best'):
+               """Get urls from 'audio_formats' section in json"""
+               file_url = None
+               try:
+                       bitrate_list = jsonData[fmt]
+                       if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
+                               bitrate = max(bitrate_list) # select highest
+
+                       url_list = jsonData[fmt][bitrate]
+               except TypeError: # we have no bitrate info.
+                       url_list = jsonData[fmt]
+                               
+               return url_list
+
+       def check_urls(self, url_list):
+               """Returns 1st active url from list"""
+               for url in url_list:
+                       try:
+                               urllib2.urlopen(url)
+                               return url
+                       except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                               url = None
+
+               return None
+
+       def _print_formats(self, formats):
+               print 'Available formats:'
+               for fmt in formats.keys():
+                       for b in formats[fmt]:
+                               try:
+                                       ext = formats[fmt][b][0]
+                                       print '%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1])
+                               except TypeError: # we have no bitrate info
+                                       ext = formats[fmt][0]
+                                       print '%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1])
+                                       break
+
+       def _real_extract(self, url):
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+               # extract uploader & filename from url
+               uploader = mobj.group(1).decode('utf-8')
+               file_id = uploader + "-" + mobj.group(2).decode('utf-8')
+
+               # construct API request
+               file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
+               # retrieve .json file with links to files
+               request = urllib2.Request(file_url)
+               try:
+                       self.report_download_json(file_url)
+                       jsonData = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % str(err))
+                       return
+
+               # parse JSON
+               json_data = json.loads(jsonData)
+               player_url = json_data['player_swf_url']
+               formats = dict(json_data['audio_formats'])
+
+               req_format = self._downloader.params.get('format', None)
+               bitrate = None
+
+               if self._downloader.params.get('listformats', None):
+                       self._print_formats(formats)
+                       return
+
+               if req_format is None or req_format == 'best':
+                       for format_param in formats.keys():
+                               url_list = self.get_urls(formats, format_param)
+                               # check urls
+                               file_url = self.check_urls(url_list)
+                               if file_url is not None:
+                                       break # got it!
+               else:
+                       if req_format not in formats.keys():
+                               self._downloader.trouble(u'ERROR: format is not available')
+                               return
+
+                       url_list = self.get_urls(formats, req_format)
+                       file_url = self.check_urls(url_list)
+                       format_param = req_format
+
+               # We have audio
+               self._downloader.increment_downloads()
+               try:
+                       # Process file information
+                       self._downloader.process_info({
+                               'id': file_id.decode('utf-8'),
+                               'url': file_url.decode('utf-8'),
+                               'uploader':     uploader.decode('utf-8'),
+                               'upload_date': u'NA',
+                               'title': json_data['name'],
+                               'stitle': _simplify_title(json_data['name']),
+                               'ext': file_url.split('.')[-1].decode('utf-8'),
+                               'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
+                               'thumbnail': json_data['thumbnail_url'],
+                               'description': json_data['description'],
+                               'player_url': player_url.decode('utf-8'),
+                       })
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'ERROR: unable to download file')
+
+class StanfordOpenClassroomIE(InfoExtractor):
+       """Information extractor for Stanford's Open ClassRoom"""
+
+       _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
+       IE_NAME = u'stanfordoc'
+
+       def report_download_webpage(self, objid):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+       def _real_extract(self, url):
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+
+               if mobj.group('course') and mobj.group('video'): # A specific video
+                       course = mobj.group('course')
+                       video = mobj.group('video')
+                       info = {
+                               'id': _simplify_title(course + '_' + video),
+                       }
+       
+                       self.report_extraction(info['id'])
+                       baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
+                       xmlUrl = baseUrl + video + '.xml'
+                       try:
+                               metaXml = urllib2.urlopen(xmlUrl).read()
+                       except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                               self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err))
+                               return
+                       mdoc = xml.etree.ElementTree.fromstring(metaXml)
+                       try:
+                               info['title'] = mdoc.findall('./title')[0].text
+                               info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
+                       except IndexError:
+                               self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
+                               return
+                       info['stitle'] = _simplify_title(info['title'])
+                       info['ext'] = info['url'].rpartition('.')[2]
+                       info['format'] = info['ext']
+                       self._downloader.increment_downloads()
+                       try:
+                               self._downloader.process_info(info)
+                       except UnavailableVideoError, err:
+                               self._downloader.trouble(u'\nERROR: unable to download video')
+               elif mobj.group('course'): # A course page
+                       unescapeHTML = HTMLParser.HTMLParser().unescape
+
+                       course = mobj.group('course')
+                       info = {
+                               'id': _simplify_title(course),
+                               'type': 'playlist',
+                       }
+
+                       self.report_download_webpage(info['id'])
+                       try:
+                               coursepage = urllib2.urlopen(url).read()
+                       except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                               self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+                               return
+
+                       m = re.search('<h1>([^<]+)</h1>', coursepage)
+                       if m:
+                               info['title'] = unescapeHTML(m.group(1))
+                       else:
+                               info['title'] = info['id']
+                       info['stitle'] = _simplify_title(info['title'])
+
+                       m = re.search('<description>([^<]+)</description>', coursepage)
+                       if m:
+                               info['description'] = unescapeHTML(m.group(1))
+
+                       links = _orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
+                       info['list'] = [
+                               {
+                                       'type': 'reference',
+                                       'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
+                               }
+                                       for vpage in links]
+
+                       for entry in info['list']:
+                               assert entry['type'] == 'reference'
+                               self.extract(entry['url'])
+               else: # Root page
+                       unescapeHTML = HTMLParser.HTMLParser().unescape
+
+                       info = {
+                               'id': 'Stanford OpenClassroom',
+                               'type': 'playlist',
+                       }
+
+                       self.report_download_webpage(info['id'])
+                       rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
+                       try:
+                               rootpage = urllib2.urlopen(rootURL).read()
+                       except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                               self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+                               return
+
+                       info['title'] = info['id']
+                       info['stitle'] = _simplify_title(info['title'])
+
+                       links = _orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
+                       info['list'] = [
+                               {
+                                       'type': 'reference',
+                                       'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
+                               }
+                                       for cpage in links]
+
+                       for entry in info['list']:
+                               assert entry['type'] == 'reference'
+                               self.extract(entry['url'])
+
+class MTVIE(InfoExtractor):
+       """Information extractor for MTV.com"""
+
+       _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
+       IE_NAME = u'mtv'
+
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+       def _real_extract(self, url):
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+               if not mobj.group('proto'):
+                       url = 'http://' + url
+               video_id = mobj.group('videoid')
+               self.report_webpage(video_id)
+
+               request = urllib2.Request(url)
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
+
+               mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract song name')
+                       return
+               song_name = _unescapeHTML(mobj.group(1).decode('iso-8859-1'))
+               mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract performer')
+                       return
+               performer = _unescapeHTML(mobj.group(1).decode('iso-8859-1'))
+               video_title = performer + ' - ' + song_name 
+
+               mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to mtvn_uri')
+                       return
+               mtvn_uri = mobj.group(1)
+
+               mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract content id')
+                       return
+               content_id = mobj.group(1)
+
+               videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
+               self.report_extraction(video_id)
+               request = urllib2.Request(videogen_url)
+               try:
+                       metadataXml = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err))
+                       return
+
+               mdoc = xml.etree.ElementTree.fromstring(metadataXml)
+               renditions = mdoc.findall('.//rendition')
+
+               # For now, always pick the highest quality.
+               rendition = renditions[-1]
+
+               try:
+                       _,_,ext = rendition.attrib['type'].partition('/')
+                       format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
+                       video_url = rendition.find('./src').text
+               except KeyError:
+                       self._downloader.trouble('Invalid rendition field.')
+                       return
+
+               self._downloader.increment_downloads()
+               info = {
+                       'id': video_id,
+                       'url': video_url,
+                       'uploader': performer,
+                       'title': video_title,
+                       'stitle': _simplify_title(video_title),
+                       'ext': ext,
+                       'format': format,
+               }
+
+               try:
+                       self._downloader.process_info(info)
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
+
+
+class PostProcessor(object):
+       """Post Processor class.
+
+       PostProcessor objects can be added to downloaders with their
+       add_post_processor() method. When the downloader has finished a
+       successful download, it will take its internal chain of PostProcessors
+       and start calling the run() method on each one of them, first with
+       an initial argument and then with the returned value of the previous
+       PostProcessor.
+
+       The chain will be stopped if one of them ever returns None or the end
+       of the chain is reached.
+
+       PostProcessor objects follow a "mutual registration" process similar
+       to InfoExtractor objects.
+       """
+
+       _downloader = None
+
+       def __init__(self, downloader=None):
+               self._downloader = downloader
+
+       def set_downloader(self, downloader):
+               """Sets the downloader for this PP."""
+               self._downloader = downloader
+
+       def run(self, information):
+               """Run the PostProcessor.
+
+               The "information" argument is a dictionary like the ones
+               composed by InfoExtractors. The only difference is that this
+               one has an extra field called "filepath" that points to the
+               downloaded file.
+
+               When this method returns None, the postprocessing chain is
+               stopped. However, this method may return an information
                dictionary that will be passed to the next postprocessing
                object in the chain. It can be the one it received after
                changing some fields.
                dictionary that will be passed to the next postprocessing
                object in the chain. It can be the one it received after
                changing some fields.
@@ -3336,19 +4039,24 @@ class PostProcessor(object):
                """
                return information # by default, do nothing
 
                """
                return information # by default, do nothing
 
+class AudioConversionError(BaseException):
+       def __init__(self, message):
+               self.message = message
 
 class FFmpegExtractAudioPP(PostProcessor):
 
 
 class FFmpegExtractAudioPP(PostProcessor):
 
-       def __init__(self, downloader=None, preferredcodec=None):
+       def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
                PostProcessor.__init__(self, downloader)
                if preferredcodec is None:
                        preferredcodec = 'best'
                self._preferredcodec = preferredcodec
                PostProcessor.__init__(self, downloader)
                if preferredcodec is None:
                        preferredcodec = 'best'
                self._preferredcodec = preferredcodec
+               self._preferredquality = preferredquality
+               self._keepvideo = keepvideo
 
        @staticmethod
        def get_audio_codec(path):
                try:
 
        @staticmethod
        def get_audio_codec(path):
                try:
-                       cmd = ['ffprobe', '-show_streams', '--', path]
+                       cmd = ['ffprobe', '-show_streams', '--', _encodeFilename(path)]
                        handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
                        output = handle.communicate()[0]
                        if handle.wait() != 0:
                        handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
                        output = handle.communicate()[0]
                        if handle.wait() != 0:
@@ -3365,12 +4073,23 @@ class FFmpegExtractAudioPP(PostProcessor):
 
        @staticmethod
        def run_ffmpeg(path, out_path, codec, more_opts):
 
        @staticmethod
        def run_ffmpeg(path, out_path, codec, more_opts):
+               if codec is None:
+                       acodec_opts = []
+               else:
+                       acodec_opts = ['-acodec', codec]
+               cmd = ['ffmpeg', '-y', '-i', _encodeFilename(path), '-vn'] + acodec_opts + more_opts + ['--', _encodeFilename(out_path)]
                try:
                try:
-                       cmd = ['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + ['--', out_path]
-                       ret = subprocess.call(cmd, stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
-                       return (ret == 0)
+                       p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                       stdout,stderr = p.communicate()
                except (IOError, OSError):
                except (IOError, OSError):
-                       return False
+                       e = sys.exc_info()[1]
+                       if isinstance(e, OSError) and e.errno == 2:
+                               raise AudioConversionError('ffmpeg not found. Please install ffmpeg.')
+                       else:
+                               raise e
+               if p.returncode != 0:
+                       msg = stderr.strip().split('\n')[-1]
+                       raise AudioConversionError(msg)
 
        def run(self, information):
                path = information['filepath']
 
        def run(self, information):
                path = information['filepath']
@@ -3381,41 +4100,71 @@ class FFmpegExtractAudioPP(PostProcessor):
                        return None
 
                more_opts = []
                        return None
 
                more_opts = []
-               if self._preferredcodec == 'best' or self._preferredcodec == filecodec:
-                       if filecodec == 'aac' or filecodec == 'mp3':
+               if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
+                       if self._preferredcodec == 'm4a' and filecodec == 'aac':
+                               # Lossless, but in another container
+                               acodec = 'copy'
+                               extension = self._preferredcodec
+                               more_opts = ['-absf', 'aac_adtstoasc']
+                       elif filecodec in ['aac', 'mp3', 'vorbis']:
                                # Lossless if possible
                                acodec = 'copy'
                                extension = filecodec
                                if filecodec == 'aac':
                                        more_opts = ['-f', 'adts']
                                # Lossless if possible
                                acodec = 'copy'
                                extension = filecodec
                                if filecodec == 'aac':
                                        more_opts = ['-f', 'adts']
+                               if filecodec == 'vorbis':
+                                       extension = 'ogg'
                        else:
                                # MP3 otherwise.
                                acodec = 'libmp3lame'
                                extension = 'mp3'
                        else:
                                # MP3 otherwise.
                                acodec = 'libmp3lame'
                                extension = 'mp3'
-                               more_opts = ['-ab', '128k']
+                               more_opts = []
+                               if self._preferredquality is not None:
+                                       more_opts += ['-ab', self._preferredquality]
                else:
                        # We convert the audio (lossy)
                else:
                        # We convert the audio (lossy)
-                       acodec = {'mp3': 'libmp3lame', 'aac': 'aac'}[self._preferredcodec]
+                       acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
                        extension = self._preferredcodec
                        extension = self._preferredcodec
-                       more_opts = ['-ab', '128k']
+                       more_opts = []
+                       if self._preferredquality is not None:
+                               more_opts += ['-ab', self._preferredquality]
                        if self._preferredcodec == 'aac':
                                more_opts += ['-f', 'adts']
                        if self._preferredcodec == 'aac':
                                more_opts += ['-f', 'adts']
-
-               (prefix, ext) = os.path.splitext(path)
-               new_path = prefix + '.' + extension
-               self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path)
-               status = self.run_ffmpeg(path, new_path, acodec, more_opts)
-
-               if not status:
-                       self._downloader.to_stderr(u'WARNING: error running ffmpeg')
-                       return None
-
+                       if self._preferredcodec == 'm4a':
+                               more_opts += ['-absf', 'aac_adtstoasc']
+                       if self._preferredcodec == 'vorbis':
+                               extension = 'ogg'
+                       if self._preferredcodec == 'wav':
+                               extension = 'wav'
+                               more_opts += ['-f', 'wav']
+
+               prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+               new_path = prefix + sep + extension
+               self._downloader.to_screen(u'[ffmpeg] Destination: ' + new_path)
                try:
                try:
-                       os.remove(path)
-               except (IOError, OSError):
-                       self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
+                       self.run_ffmpeg(path, new_path, acodec, more_opts)
+               except:
+                       etype,e,tb = sys.exc_info()
+                       if isinstance(e, AudioConversionError):
+                               self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
+                       else:
+                               self._downloader.to_stderr(u'ERROR: error running ffmpeg')
                        return None
 
                        return None
 
+               # Try to update the date time for extracted audio file.
+               if information.get('filetime') is not None:
+                       try:
+                               os.utime(_encodeFilename(new_path), (time.time(), information['filetime']))
+                       except:
+                               self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
+
+               if not self._keepvideo:
+                       try:
+                               os.remove(_encodeFilename(path))
+                       except (IOError, OSError):
+                               self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
+                               return None
+
                information['filepath'] = new_path
                return information
 
                information['filepath'] = new_path
                return information
 
@@ -3426,12 +4175,17 @@ def updateSelf(downloader, filename):
        if not os.access(filename, os.W_OK):
                sys.exit('ERROR: no write permissions on %s' % filename)
 
        if not os.access(filename, os.W_OK):
                sys.exit('ERROR: no write permissions on %s' % filename)
 
-       downloader.to_screen('Updating to latest version...')
+       downloader.to_screen(u'Updating to latest version...')
 
        try:
                try:
                        urlh = urllib.urlopen(UPDATE_URL)
                        newcontent = urlh.read()
 
        try:
                try:
                        urlh = urllib.urlopen(UPDATE_URL)
                        newcontent = urlh.read()
+                       
+                       vmatch = re.search("__version__ = '([^']+)'", newcontent)
+                       if vmatch is not None and vmatch.group(1) == __version__:
+                               downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+                               return
                finally:
                        urlh.close()
        except (IOError, OSError), err:
                finally:
                        urlh.close()
        except (IOError, OSError), err:
@@ -3446,12 +4200,21 @@ def updateSelf(downloader, filename):
        except (IOError, OSError), err:
                sys.exit('ERROR: unable to overwrite current version')
 
        except (IOError, OSError), err:
                sys.exit('ERROR: unable to overwrite current version')
 
-       downloader.to_screen('Updated youtube-dl. Restart to use the new version.')
+       downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
 
 def parseOpts():
 
 def parseOpts():
-       # Deferred imports
-       import getpass
-       import optparse
+       def _readOptions(filename_bytes):
+               try:
+                       optionf = open(filename_bytes)
+               except IOError:
+                       return [] # silently skip if file is not present
+               try:
+                       res = []
+                       for l in optionf:
+                               res += shlex.split(l, comments=True)
+               finally:
+                       optionf.close()
+               return res
 
        def _format_option_string(option):
                ''' ('-o', '--option') -> -o, --format METAVAR'''
 
        def _format_option_string(option):
                ''' ('-o', '--option') -> -o, --format METAVAR'''
@@ -3522,6 +4285,9 @@ def parseOpts():
        general.add_option('--dump-user-agent',
                        action='store_true', dest='dump_user_agent',
                        help='display the current browser identification', default=False)
        general.add_option('--dump-user-agent',
                        action='store_true', dest='dump_user_agent',
                        help='display the current browser identification', default=False)
+       general.add_option('--list-extractors',
+                       action='store_true', dest='list_extractors',
+                       help='List all supported extractors and the URLs they would handle', default=False)
 
        selection.add_option('--playlist-start',
                        dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
 
        selection.add_option('--playlist-start',
                        dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
@@ -3529,6 +4295,7 @@ def parseOpts():
                        dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
        selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
        selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
                        dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
        selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
        selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
+       selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
 
        authentication.add_option('-u', '--username',
                        dest='username', metavar='USERNAME', help='account username')
 
        authentication.add_option('-u', '--username',
                        dest='username', metavar='USERNAME', help='account username')
@@ -3541,15 +4308,21 @@ def parseOpts():
        video_format.add_option('-f', '--format',
                        action='store', dest='format', metavar='FORMAT', help='video format code')
        video_format.add_option('--all-formats',
        video_format.add_option('-f', '--format',
                        action='store', dest='format', metavar='FORMAT', help='video format code')
        video_format.add_option('--all-formats',
-                       action='store_const', dest='format', help='download all available video formats', const='-1')
+                       action='store_const', dest='format', help='download all available video formats', const='all')
+       video_format.add_option('--prefer-free-formats',
+                       action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
        video_format.add_option('--max-quality',
                        action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
        video_format.add_option('--max-quality',
                        action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
+       video_format.add_option('-F', '--list-formats',
+                       action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 
 
        verbosity.add_option('-q', '--quiet',
                        action='store_true', dest='quiet', help='activates quiet mode', default=False)
        verbosity.add_option('-s', '--simulate',
 
 
        verbosity.add_option('-q', '--quiet',
                        action='store_true', dest='quiet', help='activates quiet mode', default=False)
        verbosity.add_option('-s', '--simulate',
-                       action='store_true', dest='simulate', help='do not download video', default=False)
+                       action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
+       verbosity.add_option('--skip-download',
+                       action='store_true', dest='skip_download', help='do not download the video', default=False)
        verbosity.add_option('-g', '--get-url',
                        action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
        verbosity.add_option('-e', '--get-title',
        verbosity.add_option('-g', '--get-url',
                        action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
        verbosity.add_option('-e', '--get-title',
@@ -3563,11 +4336,16 @@ def parseOpts():
        verbosity.add_option('--get-filename',
                        action='store_true', dest='getfilename',
                        help='simulate, quiet but print output filename', default=False)
        verbosity.add_option('--get-filename',
                        action='store_true', dest='getfilename',
                        help='simulate, quiet but print output filename', default=False)
+       verbosity.add_option('--get-format',
+                       action='store_true', dest='getformat',
+                       help='simulate, quiet but print output format', default=False)
        verbosity.add_option('--no-progress',
                        action='store_true', dest='noprogress', help='do not print progress bar', default=False)
        verbosity.add_option('--console-title',
                        action='store_true', dest='consoletitle',
                        help='display progress in console titlebar', default=False)
        verbosity.add_option('--no-progress',
                        action='store_true', dest='noprogress', help='do not print progress bar', default=False)
        verbosity.add_option('--console-title',
                        action='store_true', dest='consoletitle',
                        help='display progress in console titlebar', default=False)
+       verbosity.add_option('-v', '--verbose',
+                       action='store_true', dest='verbose', help='print various debugging information', default=False)
 
 
        filesystem.add_option('-t', '--title',
 
 
        filesystem.add_option('-t', '--title',
@@ -3578,15 +4356,18 @@ def parseOpts():
                        action='store_true', dest='autonumber',
                        help='number downloaded files starting from 00000', default=False)
        filesystem.add_option('-o', '--output',
                        action='store_true', dest='autonumber',
                        help='number downloaded files starting from 00000', default=False)
        filesystem.add_option('-o', '--output',
-                       dest='outtmpl', metavar='TEMPLATE', help='output filename template')
+                       dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), and %% for a literal percent. Use - to output to stdout.')
        filesystem.add_option('-a', '--batch-file',
                        dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
        filesystem.add_option('-w', '--no-overwrites',
                        action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
        filesystem.add_option('-c', '--continue',
        filesystem.add_option('-a', '--batch-file',
                        dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
        filesystem.add_option('-w', '--no-overwrites',
                        action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
        filesystem.add_option('-c', '--continue',
-                       action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
+                       action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
+       filesystem.add_option('--no-continue',
+                       action='store_false', dest='continue_dl',
+                       help='do not resume partially downloaded files (restart from beginning)')
        filesystem.add_option('--cookies',
        filesystem.add_option('--cookies',
-                       dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
+                       dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
        filesystem.add_option('--no-part',
                        action='store_true', dest='nopart', help='do not use .part files', default=False)
        filesystem.add_option('--no-mtime',
        filesystem.add_option('--no-part',
                        action='store_true', dest='nopart', help='do not use .part files', default=False)
        filesystem.add_option('--no-mtime',
@@ -3603,7 +4384,11 @@ def parseOpts():
        postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
                        help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
        postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
        postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
                        help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
        postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-                       help='"best", "aac" or "mp3"; best by default')
+                       help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default')
+       postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K',
+                       help='ffmpeg audio bitrate specification, 128k by default')
+       postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
+                       help='keeps the video file on disk after the post-processing; the video is erased by default')
 
 
        parser.add_option_group(general)
 
 
        parser.add_option_group(general)
@@ -3614,11 +4399,54 @@ def parseOpts():
        parser.add_option_group(authentication)
        parser.add_option_group(postproc)
 
        parser.add_option_group(authentication)
        parser.add_option_group(postproc)
 
-       opts, args = parser.parse_args()
+       xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
+       if xdg_config_home:
+               userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
+       else:
+               userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+       argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
+       opts, args = parser.parse_args(argv)
 
        return parser, opts, args
 
 
        return parser, opts, args
 
-def main():
+def gen_extractors():
+       """ Return a list of an instance of every supported extractor.
+       The order does matter; the first extractor matched is the one handling the URL.
+       """
+       youtube_ie = YoutubeIE()
+       google_ie = GoogleIE()
+       yahoo_ie = YahooIE()
+       return [
+               YoutubePlaylistIE(youtube_ie),
+               YoutubeUserIE(youtube_ie),
+               YoutubeSearchIE(youtube_ie),
+               youtube_ie,
+               MetacafeIE(youtube_ie),
+               DailymotionIE(),
+               google_ie,
+               GoogleSearchIE(google_ie),
+               PhotobucketIE(),
+               yahoo_ie,
+               YahooSearchIE(yahoo_ie),
+               DepositFilesIE(),
+               FacebookIE(),
+               BlipTVIE(),
+               VimeoIE(),
+               MyVideoIE(),
+               ComedyCentralIE(),
+               EscapistIE(),
+               CollegeHumorIE(),
+               XVideosIE(),
+               SoundcloudIE(),
+               InfoQIE(),
+               MixcloudIE(),
+               StanfordOpenClassroomIE(),
+               MTVIE(),
+
+               GenericIE()
+       ]
+
+def _real_main():
        parser, opts, args = parseOpts()
 
        # Open appropriate CookieJar
        parser, opts, args = parseOpts()
 
        # Open appropriate CookieJar
@@ -3637,12 +4465,6 @@ def main():
                print std_headers['User-Agent']
                sys.exit(0)
 
                print std_headers['User-Agent']
                sys.exit(0)
 
-       # General configuration
-       cookie_processor = urllib2.HTTPCookieProcessor(jar)
-       opener = urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())
-       urllib2.install_opener(opener)
-       socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
-
        # Batch file verification
        batchurls = []
        if opts.batchfile is not None:
        # Batch file verification
        batchurls = []
        if opts.batchfile is not None:
@@ -3658,6 +4480,27 @@ def main():
                        sys.exit(u'ERROR: batch file could not be read')
        all_urls = batchurls + args
 
                        sys.exit(u'ERROR: batch file could not be read')
        all_urls = batchurls + args
 
+       # General configuration
+       cookie_processor = urllib2.HTTPCookieProcessor(jar)
+       proxy_handler = urllib2.ProxyHandler()
+       opener = urllib2.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+       urllib2.install_opener(opener)
+       socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
+
+       if opts.verbose:
+               print(u'[debug] Proxy map: ' + str(proxy_handler.proxies))
+
+       extractors = gen_extractors()
+
+       if opts.list_extractors:
+               for ie in extractors:
+                       print(ie.IE_NAME)
+                       matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
+                       all_urls = filter(lambda url: url not in matchedUrls, all_urls)
+                       for mu in matchedUrls:
+                               print(u'  ' + mu)
+               sys.exit(0)
+
        # Conflicting, missing and erroneous options
        if opts.usenetrc and (opts.username is not None or opts.password is not None):
                parser.error(u'using .netrc conflicts with giving username/password')
        # Conflicting, missing and erroneous options
        if opts.usenetrc and (opts.username is not None or opts.password is not None):
                parser.error(u'using .netrc conflicts with giving username/password')
@@ -3692,50 +4535,26 @@ def main():
        except (TypeError, ValueError), err:
                parser.error(u'invalid playlist end number specified')
        if opts.extractaudio:
        except (TypeError, ValueError), err:
                parser.error(u'invalid playlist end number specified')
        if opts.extractaudio:
-               if opts.audioformat not in ['best', 'aac', 'mp3']:
+               if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
                        parser.error(u'invalid audio format specified')
 
                        parser.error(u'invalid audio format specified')
 
-       # Information extractors
-       youtube_ie = YoutubeIE()
-       google_ie = GoogleIE()
-       yahoo_ie = YahooIE()
-       extractors = [ # Order does matter
-               youtube_ie,
-               MetacafeIE(youtube_ie),
-               DailymotionIE(),
-               YoutubePlaylistIE(youtube_ie),
-               YoutubeUserIE(youtube_ie),
-               YoutubeSearchIE(youtube_ie),
-               google_ie,
-               GoogleSearchIE(google_ie),
-               PhotobucketIE(),
-               yahoo_ie,
-               YahooSearchIE(yahoo_ie),
-               DepositFilesIE(),
-               FacebookIE(),
-               BlipTVIE(),
-               VimeoIE(),
-               MyVideoIE(),
-               ComedyCentralIE(),
-               EscapistIE(),
-
-               GenericIE()
-       ]
-
        # File downloader
        fd = FileDownloader({
                'usenetrc': opts.usenetrc,
                'username': opts.username,
                'password': opts.password,
        # File downloader
        fd = FileDownloader({
                'usenetrc': opts.usenetrc,
                'username': opts.username,
                'password': opts.password,
-               'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
+               'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
                'forceurl': opts.geturl,
                'forcetitle': opts.gettitle,
                'forcethumbnail': opts.getthumbnail,
                'forcedescription': opts.getdescription,
                'forcefilename': opts.getfilename,
                'forceurl': opts.geturl,
                'forcetitle': opts.gettitle,
                'forcethumbnail': opts.getthumbnail,
                'forcedescription': opts.getdescription,
                'forcefilename': opts.getfilename,
-               'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
+               'forceformat': opts.getformat,
+               'simulate': opts.simulate,
+               'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
                'format': opts.format,
                'format_limit': opts.format_limit,
                'format': opts.format,
                'format_limit': opts.format_limit,
+               'listformats': opts.listformats,
                'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
                        or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
                        or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
                'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
                        or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
                        or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
@@ -3762,13 +4581,16 @@ def main():
                'writeinfojson': opts.writeinfojson,
                'matchtitle': opts.matchtitle,
                'rejecttitle': opts.rejecttitle,
                'writeinfojson': opts.writeinfojson,
                'matchtitle': opts.matchtitle,
                'rejecttitle': opts.rejecttitle,
+               'max_downloads': opts.max_downloads,
+               'prefer_free_formats': opts.prefer_free_formats,
+               'verbose': opts.verbose,
                })
        for extractor in extractors:
                fd.add_info_extractor(extractor)
 
        # PostProcessors
        if opts.extractaudio:
                })
        for extractor in extractors:
                fd.add_info_extractor(extractor)
 
        # PostProcessors
        if opts.extractaudio:
-               fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat))
+               fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
 
        # Update version
        if opts.update_self:
 
        # Update version
        if opts.update_self:
@@ -3780,7 +4602,12 @@ def main():
                        parser.error(u'you must provide at least one URL')
                else:
                        sys.exit()
                        parser.error(u'you must provide at least one URL')
                else:
                        sys.exit()
-       retcode = fd.download(all_urls)
+       
+       try:
+               retcode = fd.download(all_urls)
+       except MaxDownloadsReached:
+               fd.to_screen(u'--max-download limit reached, aborting.')
+               retcode = 101
 
        # Dump cookie jar if requested
        if opts.cookiefile is not None:
 
        # Dump cookie jar if requested
        if opts.cookiefile is not None:
@@ -3791,10 +4618,9 @@ def main():
 
        sys.exit(retcode)
 
 
        sys.exit(retcode)
 
-
-if __name__ == '__main__':
+def main():
        try:
        try:
-               main()
+               _real_main()
        except DownloadError:
                sys.exit(1)
        except SameFileError:
        except DownloadError:
                sys.exit(1)
        except SameFileError:
@@ -3802,4 +4628,7 @@ if __name__ == '__main__':
        except KeyboardInterrupt:
                sys.exit(u'\nERROR: Interrupted by user')
 
        except KeyboardInterrupt:
                sys.exit(u'\nERROR: Interrupted by user')
 
+if __name__ == '__main__':
+       main()
+
 # vim: set ts=4 sw=4 sts=4 noet ai si filetype=python:
 # vim: set ts=4 sw=4 sts=4 noet ai si filetype=python: