s#phihag#rg3
[youtube-dl.git] / youtube-dl
index 3f45e92..0e8347a 100755 (executable)
@@ -11,12 +11,13 @@ __author__  = (
        'Gergely Imreh',
        'Rogério Brito',
        'Philipp Hagemeister',
+       'Sören Schulze',
        )
 
 __license__ = 'Public Domain'
-__version__ = '2011.08.28-phihag'
+__version__ = '2011.09.13'
 
-UPDATE_URL = 'https://raw.github.com/phihag/youtube-dl/master/youtube-dl'
+UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 
 import cookielib
 import datetime
@@ -62,6 +63,11 @@ try:
 except ImportError:
        pass # Handled below
 
+try:
+       import xml.etree.ElementTree
+except ImportError: # Python<2.5
+       pass # Not officially supported, but let it slip
+
 std_headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
        'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
@@ -201,6 +207,7 @@ def preferredencoding():
                        yield pref
        return yield_preferredencoding().next()
 
+
 def htmlentity_transform(matchobj):
        """Transforms an HTML entity to a Unicode character.
 
@@ -227,11 +234,13 @@ def htmlentity_transform(matchobj):
        # Unknown entity in name, return its literal representation
        return (u'&%s;' % entity)
 
+
 def sanitize_title(utitle):
        """Sanitizes a video title so it could be used as part of a filename."""
        utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
        return utitle.replace(unicode(os.sep), u'%')
 
+
 def sanitize_open(filename, open_mode):
        """Try to open the given filename, and slightly tweak it if this fails.
 
@@ -258,13 +267,15 @@ def sanitize_open(filename, open_mode):
                stream = open(filename, open_mode)
                return (stream, filename)
 
+
 def timeconvert(timestr):
-    """Convert RFC 2822 defined time string into system timestamp"""
-    timestamp = None
-    timetuple = email.utils.parsedate_tz(timestr)
-    if timetuple is not None:
-        timestamp = email.utils.mktime_tz(timetuple)
-    return timestamp
+       """Convert RFC 2822 defined time string into system timestamp"""
+       timestamp = None
+       timetuple = email.utils.parsedate_tz(timestr)
+       if timetuple is not None:
+               timestamp = email.utils.mktime_tz(timetuple)
+       return timestamp
+
 
 class DownloadError(Exception):
        """Download Error exception.
@@ -275,6 +286,7 @@ class DownloadError(Exception):
        """
        pass
 
+
 class SameFileError(Exception):
        """Same File exception.
 
@@ -283,6 +295,7 @@ class SameFileError(Exception):
        """
        pass
 
+
 class PostProcessingError(Exception):
        """Post Processing exception.
 
@@ -291,6 +304,7 @@ class PostProcessingError(Exception):
        """
        pass
 
+
 class UnavailableVideoError(Exception):
        """Unavailable Format exception.
 
@@ -299,6 +313,7 @@ class UnavailableVideoError(Exception):
        """
        pass
 
+
 class ContentTooShortError(Exception):
        """Content Too Short exception.
 
@@ -314,6 +329,7 @@ class ContentTooShortError(Exception):
                self.downloaded = downloaded
                self.expected = expected
 
+
 class YoutubeDLHandler(urllib2.HTTPHandler):
        """Handler for HTTP requests and responses.
 
@@ -323,11 +339,11 @@ class YoutubeDLHandler(urllib2.HTTPHandler):
        a particular request, the original request in the program code only has
        to include the HTTP header "Youtubedl-No-Compression", which will be
        removed before making the real request.
-       
+
        Part of this code was copied from:
 
-         http://techknack.net/python-urllib2-handlers/
-         
+       http://techknack.net/python-urllib2-handlers/
+
        Andrew Rowls, the author of that code, agreed to release it to the
        public domain.
        """
@@ -338,7 +354,7 @@ class YoutubeDLHandler(urllib2.HTTPHandler):
                        return zlib.decompress(data, -zlib.MAX_WBITS)
                except zlib.error:
                        return zlib.decompress(data)
-       
+
        @staticmethod
        def addinfourl_wrapper(stream, headers, url, code):
                if hasattr(urllib2.addinfourl, 'getcode'):
@@ -346,7 +362,7 @@ class YoutubeDLHandler(urllib2.HTTPHandler):
                ret = urllib2.addinfourl(stream, headers, url)
                ret.code = code
                return ret
-       
+
        def http_request(self, req):
                for h in std_headers:
                        if h in req.headers:
@@ -372,6 +388,7 @@ class YoutubeDLHandler(urllib2.HTTPHandler):
                        resp.msg = old_resp.msg
                return resp
 
+
 class FileDownloader(object):
        """File Downloader class.
 
@@ -445,16 +462,6 @@ class FileDownloader(object):
                self.params = params
 
        @staticmethod
-       def pmkdir(filename):
-               """Create directory components in filename. Similar to Unix "mkdir -p"."""
-               components = filename.split(os.sep)
-               aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
-               aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
-               for dir in aggregate:
-                       if not os.path.exists(dir):
-                               os.mkdir(dir)
-
-       @staticmethod
        def format_bytes(bytes):
                if bytes is None:
                        return 'N/A'
@@ -465,7 +472,7 @@ class FileDownloader(object):
                else:
                        exponent = long(math.log(bytes, 1024.0))
                suffix = 'bkMGTPEZY'[exponent]
-               converted = float(bytes) / float(1024**exponent)
+               converted = float(bytes) / float(1024 ** exponent)
                return '%.2f%s' % (converted, suffix)
 
        @staticmethod
@@ -603,7 +610,7 @@ class FileDownloader(object):
                        os.rename(old_filename, new_filename)
                except (IOError, OSError), err:
                        self.trouble(u'ERROR: unable to rename file')
-       
+
        def try_utime(self, filename, last_modified_hdr):
                """Try to set the last-modified time of the given file."""
                if last_modified_hdr is None:
@@ -617,7 +624,7 @@ class FileDownloader(object):
                if filetime is None:
                        return
                try:
-                       os.utime(filename,(time.time(), filetime))
+                       os.utime(filename, (time.time(), filetime))
                except:
                        pass
 
@@ -710,9 +717,11 @@ class FileDownloader(object):
                        return
 
                try:
-                       self.pmkdir(filename)
+                       dn = os.path.dirname(filename)
+                       if dn != '' and not os.path.exists(dn):
+                               os.makedirs(dn)
                except (OSError, IOError), err:
-                       self.trouble(u'ERROR: unable to create directories: %s' % str(err))
+                       self.trouble(u'ERROR: unable to create directory ' + unicode(err))
                        return
 
                if self.params.get('writedescription', False):
@@ -725,7 +734,7 @@ class FileDownloader(object):
                                finally:
                                        descfile.close()
                        except (OSError, IOError):
-                               self.trouble(u'ERROR: Cannot write description file: %s' % str(descfn))
+                               self.trouble(u'ERROR: Cannot write description file ' + descfn)
                                return
 
                if self.params.get('writeinfojson', False):
@@ -743,7 +752,7 @@ class FileDownloader(object):
                                finally:
                                        infof.close()
                        except (OSError, IOError):
-                               self.trouble(u'ERROR: Cannot write metadata to JSON file: %s' % str(infofn))
+                               self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
                                return
 
                try:
@@ -813,7 +822,7 @@ class FileDownloader(object):
                # Download using rtmpdump. rtmpdump returns exit code 2 when
                # the connection was interrumpted and resuming appears to be
                # possible. This is part of rtmpdump's normal usage, AFAIK.
-               basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
+               basic_args = ['rtmpdump'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
                retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
                while retval == 2 or retval == 1:
                        prevsize = os.path.getsize(tmpfilename)
@@ -859,7 +868,7 @@ class FileDownloader(object):
                # Request parameters in case of being able to resume
                if self.params.get('continuedl', False) and resume_len != 0:
                        self.report_resuming_byte(resume_len)
-                       request.add_header('Range','bytes=%d-' % resume_len)
+                       request.add_header('Range', 'bytes=%d-' % resume_len)
                        open_mode = 'ab'
 
                count = 0
@@ -885,7 +894,7 @@ class FileDownloader(object):
                                        else:
                                                # Examine the reported length
                                                if (content_length is not None and
-                                                       (resume_len - 100 < long(content_length) < resume_len + 100)):
+                                                               (resume_len - 100 < long(content_length) < resume_len + 100)):
                                                        # The file had already been fully downloaded.
                                                        # Explanation to the above condition: in issue #175 it was revealed that
                                                        # YouTube sometimes adds or removes a few bytes from the end of the file,
@@ -967,6 +976,7 @@ class FileDownloader(object):
 
                return True
 
+
 class InfoExtractor(object):
        """Information Extractor class.
 
@@ -1038,6 +1048,7 @@ class InfoExtractor(object):
                """Real extraction process. Redefine in subclasses."""
                pass
 
+
 class YoutubeIE(InfoExtractor):
        """Information extractor for youtube.com."""
 
@@ -1192,7 +1203,7 @@ class YoutubeIE(InfoExtractor):
                self.report_video_info_webpage_download(video_id)
                for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
                        video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                                          % (video_id, el_type))
+                                       % (video_id, el_type))
                        request = urllib2.Request(video_info_url)
                        try:
                                video_info_webpage = urllib2.urlopen(request).read()
@@ -1512,6 +1523,7 @@ class DailymotionIE(InfoExtractor):
 
                # Retrieve video webpage to extract further information
                request = urllib2.Request(url)
+               request.add_header('Cookie', 'family_filter=off')
                try:
                        self.report_download_webpage(video_id)
                        webpage = urllib2.urlopen(request).read()
@@ -1521,25 +1533,29 @@ class DailymotionIE(InfoExtractor):
 
                # Extract URL, uploader and title from webpage
                self.report_extraction(video_id)
-               mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
+               mobj = re.search(r'(?i)addVariable\(\"sequence\"\s*,\s*\"([^\"]+?)\"\)', webpage)
                if mobj is None:
                        self._downloader.trouble(u'ERROR: unable to extract media URL')
                        return
-               mediaURL = urllib.unquote(mobj.group(1))
+               sequence = urllib.unquote(mobj.group(1))
+               mobj = re.search(r',\"sdURL\"\:\"([^\"]+?)\",', sequence)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract media URL')
+                       return
+               mediaURL = urllib.unquote(mobj.group(1)).replace('\\', '')
 
                # if needed add http://www.dailymotion.com/ if relative URL
 
                video_url = mediaURL
 
-               # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
-               mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
+               mobj = re.search(r'(?im)<title>Dailymotion\s*-\s*(.+)\s*-\s*[^<]+?</title>', webpage)
                if mobj is None:
                        self._downloader.trouble(u'ERROR: unable to extract title')
                        return
                video_title = mobj.group(1).decode('utf-8')
                video_title = sanitize_title(video_title)
 
-               mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
+               mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
                if mobj is None:
                        self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
                        return
@@ -1561,6 +1577,7 @@ class DailymotionIE(InfoExtractor):
                except UnavailableVideoError:
                        self._downloader.trouble(u'\nERROR: unable to download video')
 
+
 class GoogleIE(InfoExtractor):
        """Information extractor for video.google.com."""
 
@@ -1654,7 +1671,6 @@ class GoogleIE(InfoExtractor):
                else:   # we need something to pass to process_info
                        video_thumbnail = ''
 
-
                try:
                        # Process video information
                        self._downloader.process_info({
@@ -1854,7 +1870,8 @@ class YahooIE(InfoExtractor):
                        self._downloader.trouble(u'ERROR: unable to extract video description')
                        return
                video_description = mobj.group(1).decode('utf-8')
-               if not video_description: video_description = 'No description available.'
+               if not video_description:
+                       video_description = 'No description available.'
 
                # Extract video height and width
                mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
@@ -1875,8 +1892,8 @@ class YahooIE(InfoExtractor):
                yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
                yv_bitrate = '700'  # according to Wikipedia this is hard-coded
                request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
-                                                                 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
-                                                                 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
+                               '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
+                               '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
                try:
                        self.report_download_webpage(video_id)
                        webpage = urllib2.urlopen(request).read()
@@ -1905,7 +1922,6 @@ class YahooIE(InfoExtractor):
                                'thumbnail':    video_thumbnail.decode('utf-8'),
                                'description':  video_description,
                                'thumbnail':    video_thumbnail,
-                               'description':  video_description,
                                'player_url':   None,
                        })
                except UnavailableVideoError:
@@ -2085,11 +2101,11 @@ class GenericIE(InfoExtractor):
                        return
 
                video_url = urllib.unquote(mobj.group(1))
-               video_id  = os.path.basename(video_url)
+               video_id = os.path.basename(video_url)
 
                # here's a fun little line of code for you:
                video_extension = os.path.splitext(video_id)[1][1:]
-               video_id        = os.path.splitext(video_id)[0]
+               video_id = os.path.splitext(video_id)[0]
 
                # it's tempting to parse this further, but you would
                # have to take into account all the variations like
@@ -2162,7 +2178,7 @@ class YoutubeSearchIE(InfoExtractor):
 
                prefix, query = query.split(':')
                prefix = prefix[8:]
-               query  = query.encode('utf-8')
+               query = query.encode('utf-8')
                if prefix == '':
                        self._download_n_results(query, 1)
                        return
@@ -2176,7 +2192,7 @@ class YoutubeSearchIE(InfoExtractor):
                                        self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
                                        return
                                elif n > self._max_youtube_results:
-                                       self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
+                                       self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
                                        n = self._max_youtube_results
                                self._download_n_results(query, n)
                                return
@@ -2220,6 +2236,7 @@ class YoutubeSearchIE(InfoExtractor):
 
                        pagenum = pagenum + 1
 
+
 class GoogleSearchIE(InfoExtractor):
        """Information Extractor for Google Video search queries."""
        _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
@@ -2253,7 +2270,7 @@ class GoogleSearchIE(InfoExtractor):
 
                prefix, query = query.split(':')
                prefix = prefix[8:]
-               query  = query.encode('utf-8')
+               query = query.encode('utf-8')
                if prefix == '':
                        self._download_n_results(query, 1)
                        return
@@ -2267,7 +2284,7 @@ class GoogleSearchIE(InfoExtractor):
                                        self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
                                        return
                                elif n > self._max_google_results:
-                                       self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)'  % (self._max_google_results, n))
+                                       self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
                                        n = self._max_google_results
                                self._download_n_results(query, n)
                                return
@@ -2311,6 +2328,7 @@ class GoogleSearchIE(InfoExtractor):
 
                        pagenum = pagenum + 1
 
+
 class YahooSearchIE(InfoExtractor):
        """Information Extractor for Yahoo! Video search queries."""
        _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
@@ -2344,7 +2362,7 @@ class YahooSearchIE(InfoExtractor):
 
                prefix, query = query.split(':')
                prefix = prefix[8:]
-               query  = query.encode('utf-8')
+               query = query.encode('utf-8')
                if prefix == '':
                        self._download_n_results(query, 1)
                        return
@@ -2358,7 +2376,7 @@ class YahooSearchIE(InfoExtractor):
                                        self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
                                        return
                                elif n > self._max_yahoo_results:
-                                       self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)'  % (self._max_yahoo_results, n))
+                                       self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
                                        n = self._max_yahoo_results
                                self._download_n_results(query, n)
                                return
@@ -2402,6 +2420,7 @@ class YahooSearchIE(InfoExtractor):
 
                        pagenum = pagenum + 1
 
+
 class YoutubePlaylistIE(InfoExtractor):
        """Information Extractor for YouTube playlists."""
 
@@ -2478,6 +2497,7 @@ class YoutubePlaylistIE(InfoExtractor):
                        self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
                return
 
+
 class YoutubeUserIE(InfoExtractor):
        """Information Extractor for YouTube users."""
 
@@ -2499,7 +2519,7 @@ class YoutubeUserIE(InfoExtractor):
        def report_download_page(self, username, start_index):
                """Report attempt to download user page."""
                self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
-                                          (username, start_index, start_index + self._GDATA_PAGE_SIZE))
+                               (username, start_index, start_index + self._GDATA_PAGE_SIZE))
 
        def _real_initialize(self):
                self._youtube_ie.initialize()
@@ -2563,7 +2583,7 @@ class YoutubeUserIE(InfoExtractor):
                        video_ids = video_ids[playliststart:playlistend]
 
                self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" %
-                                                                 (username, all_ids_count, len(video_ids)))
+                               (username, all_ids_count, len(video_ids)))
 
                for video_id in video_ids:
                        self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id)
@@ -2648,6 +2668,7 @@ class DepositFilesIE(InfoExtractor):
                except UnavailableVideoError, err:
                        self._downloader.trouble(u'ERROR: unable to download file')
 
+
 class FacebookIE(InfoExtractor):
        """Information Extractor for Facebook"""
 
@@ -2943,6 +2964,221 @@ class BlipTVIE(InfoExtractor):
                        self._downloader.trouble(u'\nERROR: unable to download video')
 
 
+class MyVideoIE(InfoExtractor):
+       """Information Extractor for myvideo.de."""
+
+       _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
+
+       def __init__(self, downloader=None):
+               InfoExtractor.__init__(self, downloader)
+       
+       @staticmethod
+       def suitable(url):
+               return (re.match(MyVideoIE._VALID_URL, url) is not None)
+
+       def report_download_webpage(self, video_id):
+               """Report webpage download."""
+               self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
+
+       def _real_initialize(self):
+               return
+
+       def _real_extract(self,url):
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._download.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+
+               video_id = mobj.group(1)
+               simple_title = mobj.group(2).decode('utf-8')
+               # should actually not be necessary
+               simple_title = sanitize_title(simple_title)
+               simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', simple_title)
+
+               # Get video webpage
+               request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
+               try:
+                       self.report_download_webpage(video_id)
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+                       return
+
+               self.report_extraction(video_id)
+               mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
+                                webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract media URL')
+                       return
+               video_url = mobj.group(1) + ('/%s.flv' % video_id)
+
+               mobj = re.search('<title>([^<]+)</title>', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract title')
+                       return
+
+               video_title = mobj.group(1)
+               video_title = sanitize_title(video_title)
+
+               try:
+                       print(video_url)
+                       self._downloader.process_info({
+                               'id':           video_id,
+                               'url':          video_url,
+                               'uploader':     u'NA',
+                               'upload_date':  u'NA',
+                               'title':        video_title,
+                               'stitle':       simple_title,
+                               'ext':          u'flv',
+                               'format':       u'NA',
+                               'player_url':   None,
+                       })
+               except UnavailableVideoError:
+                       self._downloader.trouble(u'\nERROR: Unable to download video')
+
+class ComedyCentralIE(InfoExtractor):
+       """Information extractor for The Daily Show and Colbert Report """
+
+       _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
+
+       @staticmethod
+       def suitable(url):
+               return (re.match(ComedyCentralIE._VALID_URL, url) is not None)
+
+       def report_extraction(self, episode_id):
+               self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
+       
+       def report_config_download(self, episode_id):
+               self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
+
+       def report_player_url(self, episode_id):
+               self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
+
+       def _simplify_title(self, title):
+               res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+               res = res.strip(ur'_')
+               return res
+
+       def _real_extract(self, url):
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+
+               if mobj.group('shortname'):
+                       if mobj.group('shortname') in ('tds', 'thedailyshow'):
+                               url = 'http://www.thedailyshow.com/full-episodes/'
+                       else:
+                               url = 'http://www.colbertnation.com/full-episodes/'
+                       mobj = re.match(self._VALID_URL, url)
+                       assert mobj is not None
+
+               dlNewest = not mobj.group('episode')
+               if dlNewest:
+                       epTitle = mobj.group('showname')
+               else:
+                       epTitle = mobj.group('episode')
+
+               req = urllib2.Request(url)
+               self.report_extraction(epTitle)
+               try:
+                       htmlHandle = urllib2.urlopen(req)
+                       html = htmlHandle.read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+                       return
+               if dlNewest:
+                       url = htmlHandle.geturl()
+                       mobj = re.match(self._VALID_URL, url)
+                       if mobj is None:
+                               self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
+                               return
+                       if mobj.group('episode') == '':
+                               self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
+                               return
+                       epTitle = mobj.group('episode')
+
+               mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/(.*?:episode:([^:]*):)(.*?))"/>', html)
+               if len(mMovieParams) == 0:
+                       self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
+                       return
+               show_id = mMovieParams[0][2]
+               ACT_COUNT = { # TODO: Detect this dynamically
+                       'thedailyshow.com': 4,
+                       'colbertnation.com': 3,
+               }.get(show_id, 4)
+               OFFSET = {
+                       'thedailyshow.com': 1,
+                       'colbertnation.com': 1,
+               }.get(show_id, 1)
+
+               first_player_url = mMovieParams[0][0]
+               startMediaNum = int(mMovieParams[0][3]) + OFFSET
+               movieId = mMovieParams[0][1]
+
+               playerReq = urllib2.Request(first_player_url)
+               self.report_player_url(epTitle)
+               try:
+                       playerResponse = urllib2.urlopen(playerReq)
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download player: %s' % unicode(err))
+                       return
+               player_url = playerResponse.geturl()
+
+               for actNum in range(ACT_COUNT):
+                       mediaNum = startMediaNum + actNum
+                       mediaId = movieId + str(mediaNum)
+                       configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
+                                               urllib.urlencode({'uri': mediaId}))
+                       configReq = urllib2.Request(configUrl)
+                       self.report_config_download(epTitle)
+                       try:
+                               configXml = urllib2.urlopen(configReq).read()
+                       except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                               self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+                               return
+
+                       cdoc = xml.etree.ElementTree.fromstring(configXml)
+                       turls = []
+                       for rendition in cdoc.findall('.//rendition'):
+                               finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
+                               turls.append(finfo)
+
+                       if len(turls) == 0:
+                               self._downloader.trouble(u'\nERROR: unable to download ' + str(mediaNum) + ': No videos found')
+                               continue
+
+                       # For now, just pick the highest bitrate
+                       format,video_url = turls[-1]
+
+                       self._downloader.increment_downloads()
+
+                       effTitle = show_id.replace('.com', '') + '-' + epTitle
+                       info = {
+                               'id': str(mediaNum),
+                               'url': video_url,
+                               'uploader': show_id,
+                               'upload_date': 'NA',
+                               'title': effTitle,
+                               'stitle': self._simplify_title(effTitle),
+                               'ext': 'mp4',
+                               'format': format,
+                               'thumbnail': None,
+                               'description': 'TODO: Not yet supported',
+                               'player_url': player_url
+                       }
+
+                       try:
+                               self._downloader.process_info(info)
+                       except UnavailableVideoError, err:
+                               self._downloader.trouble(u'\nERROR: unable to download ' + str(mediaNum))
+                               continue
+
+
 class PostProcessor(object):
        """Post Processor class.
 
@@ -2989,6 +3225,7 @@ class PostProcessor(object):
                """
                return information # by default, do nothing
 
+
 class FFmpegExtractAudioPP(PostProcessor):
 
        def __init__(self, downloader=None, preferredcodec=None):
@@ -3286,7 +3523,8 @@ def main():
 
        # General configuration
        cookie_processor = urllib2.HTTPCookieProcessor(jar)
-       urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler()))
+       opener = urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())
+       urllib2.install_opener(opener)
        socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
 
        # Batch file verification
@@ -3357,6 +3595,9 @@ def main():
        facebook_ie = FacebookIE()
        bliptv_ie = BlipTVIE()
        vimeo_ie = VimeoIE()
+       myvideo_ie = MyVideoIE()
+       comedycentral_ie = ComedyCentralIE()
+
        generic_ie = GenericIE()
 
        # File downloader
@@ -3413,6 +3654,8 @@ def main():
        fd.add_info_extractor(facebook_ie)
        fd.add_info_extractor(bliptv_ie)
        fd.add_info_extractor(vimeo_ie)
+       fd.add_info_extractor(myvideo_ie)
+       fd.add_info_extractor(comedycentral_ie)
 
        # This must come last since it's the
        # fallback if none of the others work