)
__license__ = 'Public Domain'
-__version__ = '2011.09.13'
+__version__ = '2011.09.14'
-UPDATE_URL = 'https://raw.github.com/phihag/youtube-dl/master/youtube-dl'
+UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
import cookielib
import datetime
# Download using rtmpdump. rtmpdump returns exit code 2 when
# the connection was interrumpted and resuming appears to be
# possible. This is part of rtmpdump's normal usage, AFAIK.
- basic_args = ['rtmpdump'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
+ basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
while retval == 2 or retval == 1:
prevsize = os.path.getsize(tmpfilename)
cursize = os.path.getsize(tmpfilename)
if prevsize == cursize and retval == 1:
break
+ # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
+ if prevsize == cursize and retval == 2 and cursize > 1024:
+ self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
+ retval = 0
+ break
if retval == 0:
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
self.try_rename(tmpfilename, filename)
# Retrieve video webpage to extract further information
request = urllib2.Request(url)
+ request.add_header('Cookie', 'family_filter=off')
try:
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
# Extract URL, uploader and title from webpage
self.report_extraction(video_id)
- mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
+ mobj = re.search(r'(?i)addVariable\(\"sequence\"\s*,\s*\"([^\"]+?)\"\)', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract media URL')
return
- mediaURL = urllib.unquote(mobj.group(1))
+ sequence = urllib.unquote(mobj.group(1))
+ mobj = re.search(r',\"sdURL\"\:\"([^\"]+?)\",', sequence)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract media URL')
+ return
+ mediaURL = urllib.unquote(mobj.group(1)).replace('\\', '')
# if needed add http://www.dailymotion.com/ if relative URL
video_url = mediaURL
- # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
- mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
+ mobj = re.search(r'(?im)<title>Dailymotion\s*-\s*(.+)\s*-\s*[^<]+?</title>', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
video_title = sanitize_title(video_title)
- mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
+ mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
return
def report_config_download(self, episode_id):
self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
+ def report_index_download(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
+
def report_player_url(self, episode_id):
self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
return
epTitle = mobj.group('episode')
- mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/(.*?:episode:([^:]*):)(.*?))"/>', html)
+ mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"/>', html)
if len(mMovieParams) == 0:
self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
return
- show_id = mMovieParams[0][2]
- ACT_COUNT = { # TODO: Detect this dynamically
- 'thedailyshow.com': 4,
- 'colbertnation.com': 3,
- }.get(show_id, 4)
- OFFSET = {
- 'thedailyshow.com': 1,
- 'colbertnation.com': 1,
- }.get(show_id, 1)
-
- first_player_url = mMovieParams[0][0]
- startMediaNum = int(mMovieParams[0][3]) + OFFSET
- movieId = mMovieParams[0][1]
-
- playerReq = urllib2.Request(first_player_url)
+
+ playerUrl_raw = mMovieParams[0][0]
self.report_player_url(epTitle)
try:
- playerResponse = urllib2.urlopen(playerReq)
+ urlHandle = urllib2.urlopen(playerUrl_raw)
+ playerUrl = urlHandle.geturl()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download player: %s' % unicode(err))
+ self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
return
- player_url = playerResponse.geturl()
- for actNum in range(ACT_COUNT):
- mediaNum = startMediaNum + actNum
- mediaId = movieId + str(mediaNum)
+ uri = mMovieParams[0][1]
+ indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri})
+ self.report_index_download(epTitle)
+ try:
+ indexXml = urllib2.urlopen(indexUrl).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
+ return
+
+ idoc = xml.etree.ElementTree.fromstring(indexXml)
+ itemEls = idoc.findall('.//item')
+ for itemEl in itemEls:
+ mediaId = itemEl.findall('./guid')[0].text
+ shortMediaId = mediaId.split(':')[-1]
+ showId = mediaId.split(':')[-2].replace('.com', '')
+ officialTitle = itemEl.findall('./title')[0].text
+ officialDate = itemEl.findall('./pubDate')[0].text
+
configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
urllib.urlencode({'uri': mediaId}))
configReq = urllib2.Request(configUrl)
turls.append(finfo)
if len(turls) == 0:
- self._downloader.trouble(u'\nERROR: unable to download ' + str(mediaNum) + ': No videos found')
+ self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
continue
# For now, just pick the highest bitrate
self._downloader.increment_downloads()
- effTitle = show_id.replace('.com', '') + '-' + epTitle
+ effTitle = showId + '-' + epTitle
info = {
- 'id': str(mediaNum),
+ 'id': shortMediaId,
'url': video_url,
- 'uploader': show_id,
- 'upload_date': 'NA',
+ 'uploader': showId,
+ 'upload_date': officialDate,
'title': effTitle,
'stitle': self._simplify_title(effTitle),
'ext': 'mp4',
'format': format,
'thumbnail': None,
- 'description': 'TODO: Not yet supported',
- 'player_url': player_url
+ 'description': officialTitle,
+ 'player_url': playerUrl
}
try:
self._downloader.process_info(info)
except UnavailableVideoError, err:
- self._downloader.trouble(u'\nERROR: unable to download ' + str(mediaNum))
+ self._downloader.trouble(u'\nERROR: unable to download ' + mediaId)
continue
kw = {
'version' : __version__,
'formatter' : fmt,
- 'usage' : '%prog [options] url...',
+ 'usage' : '%prog [options] url [url...]',
'conflict_handler' : 'resolve',
}
# Information extractors
youtube_ie = YoutubeIE()
- metacafe_ie = MetacafeIE(youtube_ie)
- dailymotion_ie = DailymotionIE()
- youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
- youtube_user_ie = YoutubeUserIE(youtube_ie)
- youtube_search_ie = YoutubeSearchIE(youtube_ie)
google_ie = GoogleIE()
- google_search_ie = GoogleSearchIE(google_ie)
- photobucket_ie = PhotobucketIE()
yahoo_ie = YahooIE()
- yahoo_search_ie = YahooSearchIE(yahoo_ie)
- deposit_files_ie = DepositFilesIE()
- facebook_ie = FacebookIE()
- bliptv_ie = BlipTVIE()
- vimeo_ie = VimeoIE()
- myvideo_ie = MyVideoIE()
- comedycentral_ie = ComedyCentralIE()
-
- generic_ie = GenericIE()
+ extractors = [ # Order does matter
+ youtube_ie,
+ MetacafeIE(youtube_ie),
+ DailymotionIE(),
+ YoutubePlaylistIE(youtube_ie),
+ YoutubeUserIE(youtube_ie),
+ YoutubeSearchIE(youtube_ie),
+ google_ie,
+ GoogleSearchIE(google_ie),
+ PhotobucketIE(),
+ yahoo_ie,
+ YahooSearchIE(yahoo_ie),
+ DepositFilesIE(),
+ FacebookIE(),
+ BlipTVIE(),
+ VimeoIE(),
+ MyVideoIE(),
+ ComedyCentralIE(),
+
+ GenericIE()
+ ]
# File downloader
fd = FileDownloader({
'writedescription': opts.writedescription,
'writeinfojson': opts.writeinfojson,
})
- fd.add_info_extractor(youtube_search_ie)
- fd.add_info_extractor(youtube_pl_ie)
- fd.add_info_extractor(youtube_user_ie)
- fd.add_info_extractor(metacafe_ie)
- fd.add_info_extractor(dailymotion_ie)
- fd.add_info_extractor(youtube_ie)
- fd.add_info_extractor(google_ie)
- fd.add_info_extractor(google_search_ie)
- fd.add_info_extractor(photobucket_ie)
- fd.add_info_extractor(yahoo_ie)
- fd.add_info_extractor(yahoo_search_ie)
- fd.add_info_extractor(deposit_files_ie)
- fd.add_info_extractor(facebook_ie)
- fd.add_info_extractor(bliptv_ie)
- fd.add_info_extractor(vimeo_ie)
- fd.add_info_extractor(myvideo_ie)
- fd.add_info_extractor(comedycentral_ie)
-
- # This must come last since it's the
- # fallback if none of the others work
- fd.add_info_extractor(generic_ie)
+ for extractor in extractors:
+ fd.add_info_extractor(extractor)
# PostProcessors
if opts.extractaudio:
git://git.jankratochvil.net / youtube-dl.git / blobdiff