X-Git-Url: https://git.jankratochvil.net/?a=blobdiff_plain;f=youtube-dl;h=a5af555ff1471630f72853bb3c97cd6cfe7c6990;hb=cb6568bf21e891143dab378d4a40988fc52b9f1d;hp=6770678d2d4c4a655ea46e547bfd0e3169f00969;hpb=0f9b77223e92a25eb1f4a626efcd7ff76beb9841;p=youtube-dl.git
diff --git a/youtube-dl b/youtube-dl
index 6770678..a5af555 100755
--- a/youtube-dl
+++ b/youtube-dl
@@ -12,10 +12,12 @@ __author__ = (
'Rogério Brito',
'Philipp Hagemeister',
'Sören Schulze',
+ 'Kevin Ngo',
+ 'Ori Avtalion',
)
__license__ = 'Public Domain'
-__version__ = '2011.09.30'
+__version__ = '2011.10.19'
UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
@@ -1560,9 +1562,6 @@ class DailymotionIE(InfoExtractor):
"""Report information extraction."""
self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
- def _real_initialize(self):
- return
-
def _real_extract(self, url):
# Extract id and simplified title from URL
mobj = re.match(self._VALID_URL, url)
@@ -1651,9 +1650,6 @@ class GoogleIE(InfoExtractor):
"""Report information extraction."""
self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
- def _real_initialize(self):
- return
-
def _real_extract(self, url):
# Extract id from URL
mobj = re.match(self._VALID_URL, url)
@@ -1758,9 +1754,6 @@ class PhotobucketIE(InfoExtractor):
"""Report information extraction."""
self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
- def _real_initialize(self):
- return
-
def _real_extract(self, url):
# Extract id from URL
mobj = re.match(self._VALID_URL, url)
@@ -1840,9 +1833,6 @@ class YahooIE(InfoExtractor):
"""Report information extraction."""
self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
- def _real_initialize(self):
- return
-
def _real_extract(self, url, new_video=True):
# Extract ID from URL
mobj = re.match(self._VALID_URL, url)
@@ -1993,9 +1983,6 @@ class VimeoIE(InfoExtractor):
"""Report information extraction."""
self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
- def _real_initialize(self):
- return
-
def _real_extract(self, url, new_video=True):
# Extract ID from URL
mobj = re.match(self._VALID_URL, url)
@@ -2059,6 +2046,18 @@ class VimeoIE(InfoExtractor):
return
sig = mobj.group(1).decode('utf-8')
+ # Vimeo specific: extract video quality information
+ mobj = re.search(r'(\d+)', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video quality information')
+ return
+ quality = mobj.group(1).decode('utf-8')
+
+ if int(quality) == 1:
+ quality = 'hd'
+ else:
+ quality = 'sd'
+
# Vimeo specific: Extract request signature expiration
mobj = re.search(r'(.*?)', webpage)
if mobj is None:
@@ -2066,7 +2065,7 @@ class VimeoIE(InfoExtractor):
return
sig_exp = mobj.group(1).decode('utf-8')
- video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s" % (video_id, sig, sig_exp)
+ video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s/?q=%s" % (video_id, sig, sig_exp, quality)
try:
# Process video information
@@ -2106,9 +2105,6 @@ class GenericIE(InfoExtractor):
"""Report information extraction."""
self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
- def _real_initialize(self):
- return
-
def _real_extract(self, url):
# At this point we have a new video
self._downloader.increment_downloads()
@@ -2458,7 +2454,7 @@ class YahooSearchIE(InfoExtractor):
class YoutubePlaylistIE(InfoExtractor):
"""Information Extractor for YouTube playlists."""
- _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+ _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
_MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*'
@@ -2502,7 +2498,8 @@ class YoutubePlaylistIE(InfoExtractor):
while True:
self.report_download_page(playlist_id, pagenum)
- request = urllib2.Request(self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum))
+ url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
+ request = urllib2.Request(url)
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@@ -2536,7 +2533,7 @@ class YoutubeUserIE(InfoExtractor):
_TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
_GDATA_PAGE_SIZE = 50
_GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
- _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
+ _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
_youtube_ie = None
IE_NAME = u'youtube:user'
@@ -2634,9 +2631,6 @@ class DepositFilesIE(InfoExtractor):
"""Report information extraction."""
self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
- def _real_initialize(self):
- return
-
def _real_extract(self, url):
# At this point we have a new file
self._downloader.increment_downloads()
@@ -2697,11 +2691,12 @@ class DepositFilesIE(InfoExtractor):
class FacebookIE(InfoExtractor):
"""Information Extractor for Facebook"""
- _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/video/video\.php\?(?:.*?)v=(?P\d+)(?:.*)'
+ _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P\d+)(?:.*)'
_LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
_NETRC_MACHINE = 'facebook'
- _available_formats = ['highqual', 'lowqual']
+ _available_formats = ['video', 'highqual', 'lowqual']
_video_extensions = {
+ 'video': 'mp4',
'highqual': 'mp4',
'lowqual': 'mp4',
}
@@ -2729,10 +2724,9 @@ class FacebookIE(InfoExtractor):
def _parse_page(self, video_webpage):
"""Extract video information from page"""
# General data
- data = {'title': r'class="video_title datawrap">(.*?)',
+ data = {'title': r'\("video_title", "(.*?)"\)',
'description': r'(.*?)
',
'owner': r'\("video_owner_name", "(.*?)"\)',
- 'upload_date': r'data-date="(.*?)"',
'thumbnail': r'\("thumb_url", "(?P.*?)"\)',
}
video_info = {}
@@ -3027,9 +3021,6 @@ class MyVideoIE(InfoExtractor):
"""Report information extraction."""
self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
- def _real_initialize(self):
- return
-
def _real_extract(self,url):
mobj = re.match(self._VALID_URL, url)
if mobj is None:
@@ -3469,6 +3460,189 @@ class XVideosIE(InfoExtractor):
self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
+class SoundcloudIE(InfoExtractor):
+ """Information extractor for soundcloud.com
+ To access the media, the uid of the song and a stream token
+ must be extracted from the page source and the script must make
+ a request to media.soundcloud.com/crossdomain.xml. Then
+ the media can be grabbed by requesting from an url composed
+ of the stream token and uid
+ """
+
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
+ IE_NAME = u'soundcloud'
+
+ def __init__(self, downloader=None):
+ InfoExtractor.__init__(self, downloader)
+
+ def report_webpage(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ htmlParser = HTMLParser.HTMLParser()
+
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ # extract uploader (which is in the url)
+ uploader = mobj.group(1).decode('utf-8')
+ # extract simple title (uploader + slug of song title)
+ slug_title = mobj.group(2).decode('utf-8')
+ simple_title = uploader + '-' + slug_title
+
+ self.report_webpage('%s/%s' % (uploader, slug_title))
+
+ request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
+ try:
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+ return
+
+ self.report_extraction('%s/%s' % (uploader, slug_title))
+
+ # extract uid and stream token that soundcloud hands out for access
+ mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
+ if mobj:
+ video_id = mobj.group(1)
+ stream_token = mobj.group(2)
+
+ # extract unsimplified title
+ mobj = re.search('"title":"(.*?)",', webpage)
+ if mobj:
+ title = mobj.group(1)
+
+ # construct media url (with uid/token)
+ mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
+ mediaURL = mediaURL % (video_id, stream_token)
+
+ # description
+ description = u'No description available'
+ mobj = re.search('track-description-value">(.*?)
', webpage)
+ if mobj:
+ description = mobj.group(1)
+
+ # upload date
+ upload_date = None
+ mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)", webpage)
+ if mobj:
+ try:
+ upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
+ except Exception as e:
+ print str(e)
+
+ # for soundcloud, a request to a cross domain is required for cookies
+ request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
+
+ try:
+ self._downloader.process_info({
+ 'id': video_id.decode('utf-8'),
+ 'url': mediaURL,
+ 'uploader': uploader.decode('utf-8'),
+ 'upload_date': upload_date,
+ 'title': simple_title.decode('utf-8'),
+ 'stitle': simple_title.decode('utf-8'),
+ 'ext': u'mp3',
+ 'format': u'NA',
+ 'player_url': None,
+ 'description': description.decode('utf-8')
+ })
+ except UnavailableVideoError:
+ self._downloader.trouble(u'\nERROR: unable to download video')
+
+
+class InfoQIE(InfoExtractor):
+ """Information extractor for infoq.com"""
+
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
+ IE_NAME = u'infoq'
+
+ def report_webpage(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _simplify_title(self, title):
+ res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+ res = res.strip(ur'_')
+ return res
+
+ def _real_extract(self, url):
+ htmlParser = HTMLParser.HTMLParser()
+
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ self.report_webpage(url)
+
+ request = urllib2.Request(url)
+ try:
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+ return
+
+ self.report_extraction(url)
+
+
+ # Extract video URL
+ mobj = re.search(r"jsclassref='([^']*)'", webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video url')
+ return
+ video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
+
+
+ # Extract title
+ mobj = re.search(r'contentTitle = "(.*?)";', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video title')
+ return
+ video_title = mobj.group(1).decode('utf-8')
+
+ # Extract description
+ video_description = u'No description available.'
+ mobj = re.search(r'', webpage)
+ if mobj is not None:
+ video_description = mobj.group(1).decode('utf-8')
+
+ video_filename = video_url.split('/')[-1]
+ video_id, extension = video_filename.split('.')
+
+ self._downloader.increment_downloads()
+ info = {
+ 'id': video_id,
+ 'url': video_url,
+ 'uploader': None,
+ 'upload_date': None,
+ 'title': video_title,
+ 'stitle': self._simplify_title(video_title),
+ 'ext': extension,
+ 'format': extension, # Extension is always(?) mp4, but seems to be flv
+ 'thumbnail': None,
+ 'description': video_description,
+ 'player_url': None,
+ }
+
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download ' + video_url)
+
+
+
class PostProcessor(object):
"""Post Processor class.
@@ -3865,11 +4039,13 @@ def gen_extractors():
EscapistIE(),
CollegeHumorIE(),
XVideosIE(),
+ SoundcloudIE(),
+ InfoQIE(),
GenericIE()
]
-def main():
+def _real_main():
parser, opts, args = parseOpts()
# Open appropriate CookieJar
@@ -4029,10 +4205,9 @@ def main():
sys.exit(retcode)
-
-if __name__ == '__main__':
+def main():
try:
- main()
+ _real_main()
except DownloadError:
sys.exit(1)
except SameFileError:
@@ -4040,4 +4215,7 @@ if __name__ == '__main__':
except KeyboardInterrupt:
sys.exit(u'\nERROR: Interrupted by user')
+if __name__ == '__main__':
+ main()
+
# vim: set ts=4 sw=4 sts=4 noet ai si filetype=python: