'Gergely Imreh',
'Rogério Brito',
'Philipp Hagemeister',
+ 'Sören Schulze',
)
__license__ = 'Public Domain'
-__version__ = '2011.08.28-phihag'
+__version__ = '2011.09.14'
-UPDATE_URL = 'https://raw.github.com/phihag/youtube-dl/master/youtube-dl'
+UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
import cookielib
import datetime
except ImportError:
pass # Handled below
+try:
+ import xml.etree.ElementTree
+except ImportError: # Python<2.5
+ pass # Not officially supported, but let it slip
+
std_headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
self.params = params
@staticmethod
- def pmkdir(filename):
- """Create directory components in filename. Similar to Unix "mkdir -p"."""
- components = filename.split(os.sep)
- aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
- aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
- for dir in aggregate:
- if not os.path.exists(dir):
- os.mkdir(dir)
-
- @staticmethod
def format_bytes(bytes):
if bytes is None:
return 'N/A'
return
try:
- self.pmkdir(filename)
+ dn = os.path.dirname(filename)
+ if dn != '' and not os.path.exists(dn):
+ os.makedirs(dn)
except (OSError, IOError), err:
- self.trouble(u'ERROR: unable to create directories: %s' % str(err))
+ self.trouble(u'ERROR: unable to create directory ' + unicode(err))
return
if self.params.get('writedescription', False):
finally:
descfile.close()
except (OSError, IOError):
- self.trouble(u'ERROR: Cannot write description file: %s' % str(descfn))
+ self.trouble(u'ERROR: Cannot write description file ' + descfn)
return
if self.params.get('writeinfojson', False):
finally:
infof.close()
except (OSError, IOError):
- self.trouble(u'ERROR: Cannot write metadata to JSON file: %s' % str(infofn))
+ self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
return
try:
cursize = os.path.getsize(tmpfilename)
if prevsize == cursize and retval == 1:
break
+ # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
+ if prevsize == cursize and retval == 2 and cursize > 1024:
+ self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
+ retval = 0
+ break
if retval == 0:
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
self.try_rename(tmpfilename, filename)
# Retrieve video webpage to extract further information
request = urllib2.Request(url)
+ request.add_header('Cookie', 'family_filter=off')
try:
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
# Extract URL, uploader and title from webpage
self.report_extraction(video_id)
- mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
+ mobj = re.search(r'(?i)addVariable\(\"sequence\"\s*,\s*\"([^\"]+?)\"\)', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract media URL')
return
- mediaURL = urllib.unquote(mobj.group(1))
+ sequence = urllib.unquote(mobj.group(1))
+ mobj = re.search(r',\"sdURL\"\:\"([^\"]+?)\",', sequence)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract media URL')
+ return
+ mediaURL = urllib.unquote(mobj.group(1)).replace('\\', '')
# if needed add http://www.dailymotion.com/ if relative URL
video_url = mediaURL
- # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
- mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
+ mobj = re.search(r'(?im)<title>Dailymotion\s*-\s*(.+)\s*-\s*[^<]+?</title>', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
video_title = sanitize_title(video_title)
- mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
+ mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
return
'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description,
'thumbnail': video_thumbnail,
- 'description': video_description,
'player_url': None,
})
except UnavailableVideoError:
self._downloader.trouble(u'\nERROR: unable to download video')
+class MyVideoIE(InfoExtractor):
+ """Information Extractor for myvideo.de."""
+
+ _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
+
+ def __init__(self, downloader=None):
+ InfoExtractor.__init__(self, downloader)
+
+ @staticmethod
+ def suitable(url):
+ return (re.match(MyVideoIE._VALID_URL, url) is not None)
+
+ def report_download_webpage(self, video_id):
+ """Report webpage download."""
+ self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
+
+ def _real_initialize(self):
+ return
+
+ def _real_extract(self,url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._download.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ video_id = mobj.group(1)
+ simple_title = mobj.group(2).decode('utf-8')
+ # should actually not be necessary
+ simple_title = sanitize_title(simple_title)
+ simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', simple_title)
+
+ # Get video webpage
+ request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
+ try:
+ self.report_download_webpage(video_id)
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+ return
+
+ self.report_extraction(video_id)
+ mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
+ webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract media URL')
+ return
+ video_url = mobj.group(1) + ('/%s.flv' % video_id)
+
+ mobj = re.search('<title>([^<]+)</title>', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract title')
+ return
+
+ video_title = mobj.group(1)
+ video_title = sanitize_title(video_title)
+
+ try:
+ print(video_url)
+ self._downloader.process_info({
+ 'id': video_id,
+ 'url': video_url,
+ 'uploader': u'NA',
+ 'upload_date': u'NA',
+ 'title': video_title,
+ 'stitle': simple_title,
+ 'ext': u'flv',
+ 'format': u'NA',
+ 'player_url': None,
+ })
+ except UnavailableVideoError:
+ self._downloader.trouble(u'\nERROR: Unable to download video')
+
+class ComedyCentralIE(InfoExtractor):
+ """Information extractor for The Daily Show and Colbert Report """
+
+ _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
+
+ @staticmethod
+ def suitable(url):
+ return (re.match(ComedyCentralIE._VALID_URL, url) is not None)
+
+ def report_extraction(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
+
+ def report_config_download(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
+
+ def report_index_download(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
+
+ def report_player_url(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
+
+ def _simplify_title(self, title):
+ res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+ res = res.strip(ur'_')
+ return res
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ if mobj.group('shortname'):
+ if mobj.group('shortname') in ('tds', 'thedailyshow'):
+ url = 'http://www.thedailyshow.com/full-episodes/'
+ else:
+ url = 'http://www.colbertnation.com/full-episodes/'
+ mobj = re.match(self._VALID_URL, url)
+ assert mobj is not None
+
+ dlNewest = not mobj.group('episode')
+ if dlNewest:
+ epTitle = mobj.group('showname')
+ else:
+ epTitle = mobj.group('episode')
+
+ req = urllib2.Request(url)
+ self.report_extraction(epTitle)
+ try:
+ htmlHandle = urllib2.urlopen(req)
+ html = htmlHandle.read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+ return
+ if dlNewest:
+ url = htmlHandle.geturl()
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
+ return
+ if mobj.group('episode') == '':
+ self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
+ return
+ epTitle = mobj.group('episode')
+
+ mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"/>', html)
+ if len(mMovieParams) == 0:
+ self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
+ return
+
+ playerUrl_raw = mMovieParams[0][0]
+ self.report_player_url(epTitle)
+ try:
+ urlHandle = urllib2.urlopen(playerUrl_raw)
+ playerUrl = urlHandle.geturl()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
+ return
+
+ uri = mMovieParams[0][1]
+ indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri})
+ self.report_index_download(epTitle)
+ try:
+ indexXml = urllib2.urlopen(indexUrl).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
+ return
+
+ idoc = xml.etree.ElementTree.fromstring(indexXml)
+ itemEls = idoc.findall('.//item')
+ for itemEl in itemEls:
+ mediaId = itemEl.findall('./guid')[0].text
+ shortMediaId = mediaId.split(':')[-1]
+ showId = mediaId.split(':')[-2].replace('.com', '')
+ officialTitle = itemEl.findall('./title')[0].text
+ officialDate = itemEl.findall('./pubDate')[0].text
+
+ configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
+ urllib.urlencode({'uri': mediaId}))
+ configReq = urllib2.Request(configUrl)
+ self.report_config_download(epTitle)
+ try:
+ configXml = urllib2.urlopen(configReq).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+ return
+
+ cdoc = xml.etree.ElementTree.fromstring(configXml)
+ turls = []
+ for rendition in cdoc.findall('.//rendition'):
+ finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
+ turls.append(finfo)
+
+ if len(turls) == 0:
+ self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
+ continue
+
+ # For now, just pick the highest bitrate
+ format,video_url = turls[-1]
+
+ self._downloader.increment_downloads()
+
+ effTitle = showId + '-' + epTitle
+ info = {
+ 'id': shortMediaId,
+ 'url': video_url,
+ 'uploader': showId,
+ 'upload_date': officialDate,
+ 'title': effTitle,
+ 'stitle': self._simplify_title(effTitle),
+ 'ext': 'mp4',
+ 'format': format,
+ 'thumbnail': None,
+ 'description': officialTitle,
+ 'player_url': playerUrl
+ }
+
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download ' + mediaId)
+ continue
+
+
class PostProcessor(object):
"""Post Processor class.
kw = {
'version' : __version__,
'formatter' : fmt,
- 'usage' : '%prog [options] url...',
+ 'usage' : '%prog [options] url [url...]',
'conflict_handler' : 'resolve',
}
# General configuration
cookie_processor = urllib2.HTTPCookieProcessor(jar)
- urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler()))
+ opener = urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())
+ urllib2.install_opener(opener)
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
# Batch file verification
facebook_ie = FacebookIE()
bliptv_ie = BlipTVIE()
vimeo_ie = VimeoIE()
+ myvideo_ie = MyVideoIE()
+ comedycentral_ie = ComedyCentralIE()
+
generic_ie = GenericIE()
# File downloader
fd.add_info_extractor(facebook_ie)
fd.add_info_extractor(bliptv_ie)
fd.add_info_extractor(vimeo_ie)
+ fd.add_info_extractor(myvideo_ie)
+ fd.add_info_extractor(comedycentral_ie)
# This must come last since it's the
# fallback if none of the others work
git://git.jankratochvil.net / youtube-dl.git / blobdiff