X-Git-Url: http://git.jankratochvil.net/?p=youtube-dl.git;a=blobdiff_plain;f=youtube-dl;h=949d2e532d38eb154d9063aedfe6660ad12bed81;hp=4f6706e2aff69f838d6b8bbb78e6378f0aec516c;hb=871be928a89be62f89ec21a1d9cb700dbddcaeca;hpb=9b4556c469d63ec49374f98237e623cfe88cda93

diff --git a/youtube-dl b/youtube-dl
index 4f6706e..949d2e5 100755
--- a/youtube-dl
+++ b/youtube-dl
@@ -15,7 +15,7 @@ __author__  = (
 	)
 
 __license__ = 'Public Domain'
-__version__ = '2011.09.15'
+__version__ = '2011.10.19'
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 
@@ -625,11 +625,12 @@ class FileDownloader(object):
 			return
 		filetime = timeconvert(timestr)
 		if filetime is None:
-			return
+			return filetime
 		try:
 			os.utime(filename, (time.time(), filetime))
 		except:
 			pass
+		return filetime
 
 	def report_writedescription(self, descfn):
 		""" Report that the description file is being written """
@@ -709,6 +710,8 @@ class FileDownloader(object):
 			print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
 		if self.params.get('forcefilename', False) and filename is not None:
 			print filename.encode(preferredencoding(), 'xmlcharrefreplace')
+		if self.params.get('forceformat', False):
+			print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace')
 
 		# Do nothing else if in simulate mode
 		if self.params.get('simulate', False):
@@ -763,7 +766,8 @@ class FileDownloader(object):
 			try:
 				infof = open(infofn, 'wb')
 				try:
-					json.dump(info_dict, infof)
+					json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
+					json.dump(json_info_dict, infof)
 				finally:
 					infof.close()
 			except (OSError, IOError):
@@ -772,7 +776,7 @@ class FileDownloader(object):
 
 		if not self.params.get('skip_download', False):
 			try:
-				success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
+				success = self._do_download(filename, info_dict)
 			except (OSError, IOError), err:
 				raise UnavailableVideoError
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@@ -861,7 +865,10 @@ class FileDownloader(object):
 			self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 			return False
 
-	def _do_download(self, filename, url, player_url):
+	def _do_download(self, filename, info_dict):
+		url = info_dict['url']
+		player_url = info_dict.get('player_url', None)
+
 		# Check file already present
 		if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
 			self.report_file_already_downloaded(filename)
@@ -873,7 +880,6 @@ class FileDownloader(object):
 
 		tmpfilename = self.temp_name(filename)
 		stream = None
-		open_mode = 'wb'
 
 		# Do not include the Accept-Encoding header
 		headers = {'Youtubedl-no-compression': 'True'}
@@ -886,17 +892,22 @@ class FileDownloader(object):
 		else:
 			resume_len = 0
 
-		# Request parameters in case of being able to resume
-		if self.params.get('continuedl', False) and resume_len != 0:
-			self.report_resuming_byte(resume_len)
-			request.add_header('Range', 'bytes=%d-' % resume_len)
-			open_mode = 'ab'
+		open_mode = 'wb'
+		if resume_len != 0:
+			if self.params.get('continuedl', False):
+				self.report_resuming_byte(resume_len)
+				request.add_header('Range','bytes=%d-' % resume_len)
+				open_mode = 'ab'
+			else:
+				resume_len = 0
 
 		count = 0
 		retries = self.params.get('retries', 0)
 		while count <= retries:
 			# Establish connection
 			try:
+				if count == 0 and 'urlhandle' in info_dict:
+					data = info_dict['urlhandle']
 				data = urllib2.urlopen(request)
 				break
 			except (urllib2.HTTPError, ), err:
@@ -974,10 +985,13 @@ class FileDownloader(object):
 			block_size = self.best_block_size(after - before, len(data_block))
 
 			# Progress message
-			percent_str = self.calc_percent(byte_counter, data_len)
-			eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 			speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
-			self.report_progress(percent_str, data_len_str, speed_str, eta_str)
+			if data_len is None:
+				self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
+			else:
+				percent_str = self.calc_percent(byte_counter, data_len)
+				eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
+				self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 
 			# Apply rate limit
 			self.slow_down(start, byte_counter - resume_len)
@@ -993,7 +1007,7 @@ class FileDownloader(object):
 
 		# Update file modification time
 		if self.params.get('updatetime', True):
-			self.try_utime(filename, data.info().get('last-modified', None))
+			info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 
 		return True
 
@@ -1071,13 +1085,13 @@ class InfoExtractor(object):
 class YoutubeIE(InfoExtractor):
 	"""Information extractor for youtube.com."""
 
-	_VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
+	_VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
 	_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 	_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
 	_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 	_NETRC_MACHINE = 'youtube'
 	# Listed in order of quality
-	_available_formats = ['38', '37', '45', '22', '43', '35', '34', '18', '6', '5', '17', '13']
+	_available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
 	_video_extensions = {
 		'13': '3gp',
 		'17': 'mp4',
@@ -1086,8 +1100,24 @@ class YoutubeIE(InfoExtractor):
 		'37': 'mp4',
 		'38': 'video', # You actually don't know if this will be MOV, AVI or whatever
 		'43': 'webm',
+		'44': 'webm',
 		'45': 'webm',
 	}
+	_video_dimensions = {
+		'5': '240x400',
+		'6': '???',
+		'13': '???',
+		'17': '144x176',
+		'18': '360x640',
+		'22': '720x1280',
+		'34': '360x640',
+		'35': '480x854',
+		'37': '1080x1920',
+		'38': '3072x4096',
+		'43': '360x640',
+		'44': '480x854',
+		'45': '720x1280',
+	}	
 	IE_NAME = u'youtube'
 
 	def report_lang(self):
@@ -1122,6 +1152,11 @@ class YoutubeIE(InfoExtractor):
 		"""Indicate the download will use the RTMP protocol."""
 		self._downloader.to_screen(u'[youtube] RTMP download detected')
 
+	def _print_formats(self, formats):
+		print 'Available formats:'
+		for x in formats:
+			print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))
+
 	def _real_initialize(self):
 		if self._downloader is None:
 			return
@@ -1201,7 +1236,7 @@ class YoutubeIE(InfoExtractor):
 
 		# Get video webpage
 		self.report_video_webpage_download(video_id)
-		request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&amp;has_verified=1' % video_id)
+		request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
 		try:
 			video_webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@@ -1315,16 +1350,27 @@ class YoutubeIE(InfoExtractor):
 			if len(existing_formats) == 0:
 				self._downloader.trouble(u'ERROR: no known formats available for video')
 				return
-			if req_format is None:
+			if self._downloader.params.get('listformats', None):
+				self._print_formats(existing_formats)
+				return
+			if req_format is None or req_format == 'best':
 				video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
-			elif req_format == '-1':
+			elif req_format == 'worst':
+				video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
+			elif req_format in ('-1', 'all'):
 				video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
 			else:
-				# Specific format
-				if req_format not in url_map:
+				# Specific formats. We pick the first in a slash-delimeted sequence.
+				# For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
+				req_formats = req_format.split('/')
+				video_url_list = None
+				for rf in req_formats:
+					if rf in url_map:
+						video_url_list = [(rf, url_map[rf])]
+						break
+				if video_url_list is None:
 					self._downloader.trouble(u'ERROR: requested format not available')
 					return
-				video_url_list = [(req_format, url_map[req_format])] # Specific format
 		else:
 			self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
 			return
@@ -2013,6 +2059,18 @@ class VimeoIE(InfoExtractor):
 			return
 		sig = mobj.group(1).decode('utf-8')
 
+		# Vimeo specific: extract video quality information
+		mobj = re.search(r'<isHD>(\d+)</isHD>', webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video quality information')
+			return
+		quality = mobj.group(1).decode('utf-8')
+
+		if int(quality) == 1:
+			quality = 'hd'
+		else:
+			quality = 'sd'
+
 		# Vimeo specific: Extract request signature expiration
 		mobj = re.search(r'<request_signature_expires>(.*?)</request_signature_expires>', webpage)
 		if mobj is None:
@@ -2020,7 +2078,7 @@ class VimeoIE(InfoExtractor):
 			return
 		sig_exp = mobj.group(1).decode('utf-8')
 
-		video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s" % (video_id, sig, sig_exp)
+		video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s/?q=%s" % (video_id, sig, sig_exp, quality)
 
 		try:
 			# Process video information
@@ -2412,7 +2470,7 @@ class YahooSearchIE(InfoExtractor):
 class YoutubePlaylistIE(InfoExtractor):
 	"""Information Extractor for YouTube playlists."""
 
-	_VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+	_VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
 	_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
 	_VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
 	_MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
@@ -2486,7 +2544,7 @@ class YoutubePlaylistIE(InfoExtractor):
 class YoutubeUserIE(InfoExtractor):
 	"""Information Extractor for YouTube users."""
 
-	_VALID_URL = r'(?:(?:(?:http://)?(?:\w+\.)?youtube.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
+	_VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
 	_TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
 	_GDATA_PAGE_SIZE = 50
 	_GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
@@ -2574,7 +2632,7 @@ class YoutubeUserIE(InfoExtractor):
 class DepositFilesIE(InfoExtractor):
 	"""Information extractor for depositfiles.com"""
 
-	_VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)'
+	_VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
 	IE_NAME = u'DepositFiles'
 
 	def __init__(self, downloader=None):
@@ -2651,11 +2709,12 @@ class DepositFilesIE(InfoExtractor):
 class FacebookIE(InfoExtractor):
 	"""Information Extractor for Facebook"""
 
-	_VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook.com/video/video.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
+	_VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
 	_LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
 	_NETRC_MACHINE = 'facebook'
-	_available_formats = ['highqual', 'lowqual']
+	_available_formats = ['video', 'highqual', 'lowqual']
 	_video_extensions = {
+		'video': 'mp4',
 		'highqual': 'mp4',
 		'lowqual': 'mp4',
 	}
@@ -2683,10 +2742,9 @@ class FacebookIE(InfoExtractor):
 	def _parse_page(self, video_webpage):
 		"""Extract video information from page"""
 		# General data
-		data = {'title': r'class="video_title datawrap">(.*?)</',
+		data = {'title': r'\("video_title", "(.*?)"\)',
 			'description': r'<div class="datawrap">(.*?)</div>',
 			'owner': r'\("video_owner_name", "(.*?)"\)',
-			'upload_date': r'data-date="(.*?)"',
 			'thumbnail':  r'\("thumb_url", "(?P<THUMB>.*?)"\)',
 			}
 		video_info = {}
@@ -2829,6 +2887,8 @@ class FacebookIE(InfoExtractor):
 				return
 			if req_format is None:
 				video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
+			elif req_format == 'worst':
+				video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
 			elif req_format == '-1':
 				video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
 			else:
@@ -2873,7 +2933,11 @@ class BlipTVIE(InfoExtractor):
 
 	def report_extraction(self, file_id):
 		"""Report information extraction."""
-		self._downloader.to_screen(u'[blip.tv] %s: Extracting information' % file_id)
+		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
+
+	def report_direct_download(self, title):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
 
 	def _simplify_title(self, title):
 		res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
@@ -2893,43 +2957,64 @@ class BlipTVIE(InfoExtractor):
 		json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
 		request = urllib2.Request(json_url)
 		self.report_extraction(mobj.group(1))
+		info = None
 		try:
-			json_code = urllib2.urlopen(request).read()
+			urlh = urllib2.urlopen(request)
+			if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
+				basename = url.split('/')[-1]
+				title,ext = os.path.splitext(basename)
+				ext = ext.replace('.', '')
+				self.report_direct_download(title)
+				info = {
+					'id': title,
+					'url': url,
+					'title': title,
+					'stitle': self._simplify_title(title),
+					'ext': ext,
+					'urlhandle': urlh
+				}
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 			self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 			return
-		try:
-			json_data = json.loads(json_code)
-			if 'Post' in json_data:
-				data = json_data['Post']
-			else:
-				data = json_data
-
-			upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
-			video_url = data['media']['url']
-			umobj = re.match(self._URL_EXT, video_url)
-			if umobj is None:
-				raise ValueError('Can not determine filename extension')
-			ext = umobj.group(1)
+		if info is None: # Regular URL
+			try:
+				json_code = urlh.read()
+			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+				self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % str(err))
+				return
 
-			self._downloader.increment_downloads()
+			try:
+				json_data = json.loads(json_code)
+				if 'Post' in json_data:
+					data = json_data['Post']
+				else:
+					data = json_data
+	
+				upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
+				video_url = data['media']['url']
+				umobj = re.match(self._URL_EXT, video_url)
+				if umobj is None:
+					raise ValueError('Can not determine filename extension')
+				ext = umobj.group(1)
+	
+				info = {
+					'id': data['item_id'],
+					'url': video_url,
+					'uploader': data['display_name'],
+					'upload_date': upload_date,
+					'title': data['title'],
+					'stitle': self._simplify_title(data['title']),
+					'ext': ext,
+					'format': data['media']['mimeType'],
+					'thumbnail': data['thumbnailUrl'],
+					'description': data['description'],
+					'player_url': data['embedUrl']
+				}
+			except (ValueError,KeyError), err:
+				self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
+				return
 
-			info = {
-				'id': data['item_id'],
-				'url': video_url,
-				'uploader': data['display_name'],
-				'upload_date': upload_date,
-				'title': data['title'],
-				'stitle': self._simplify_title(data['title']),
-				'ext': ext,
-				'format': data['media']['mimeType'],
-				'thumbnail': data['thumbnailUrl'],
-				'description': data['description'],
-				'player_url': data['embedUrl']
-			}
-		except (ValueError,KeyError), err:
-			self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
-			return
+		self._downloader.increment_downloads()
 
 		try:
 			self._downloader.process_info(info)
@@ -2995,7 +3080,6 @@ class MyVideoIE(InfoExtractor):
 		video_title = sanitize_title(video_title)
 
 		try:
-			print(video_url)
 			self._downloader.process_info({
 				'id':		video_id,
 				'url':		video_url,
@@ -3154,7 +3238,7 @@ class ComedyCentralIE(InfoExtractor):
 class EscapistIE(InfoExtractor):
 	"""Information extractor for The Escapist """
 
-	_VALID_URL = r'^(https?://)?(www\.)escapistmagazine.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?].*$'
+	_VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
 	IE_NAME = u'escapist'
 
 	def report_extraction(self, showName):
@@ -3234,6 +3318,264 @@ class EscapistIE(InfoExtractor):
 			self._downloader.trouble(u'\nERROR: unable to download ' + videoId)
 
 
+class CollegeHumorIE(InfoExtractor):
+	"""Information extractor for collegehumor.com"""
+
+	_VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
+	IE_NAME = u'collegehumor'
+
+	def report_webpage(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+	def report_extraction(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+	def _simplify_title(self, title):
+		res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+		res = res.strip(ur'_')
+		return res
+
+	def _real_extract(self, url):
+		htmlParser = HTMLParser.HTMLParser()
+
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+			return
+		video_id = mobj.group('videoid')
+
+		self.report_webpage(video_id)
+		request = urllib2.Request(url)
+		try:
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			return
+
+		m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
+		if m is None:
+			self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
+			return
+		internal_video_id = m.group('internalvideoid')
+
+		info = {
+			'id': video_id,
+			'internal_id': internal_video_id,
+		}
+
+		self.report_extraction(video_id)
+		xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
+		try:
+			metaXml = urllib2.urlopen(xmlUrl).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
+			return
+
+		mdoc = xml.etree.ElementTree.fromstring(metaXml)
+		try:
+			videoNode = mdoc.findall('./video')[0]
+			info['description'] = videoNode.findall('./description')[0].text
+			info['title'] = videoNode.findall('./caption')[0].text
+			info['stitle'] = self._simplify_title(info['title'])
+			info['url'] = videoNode.findall('./file')[0].text
+			info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
+			info['ext'] = info['url'].rpartition('.')[2]
+			info['format'] = info['ext']
+		except IndexError:
+			self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
+			return
+
+		self._downloader.increment_downloads()
+
+		try:
+			self._downloader.process_info(info)
+		except UnavailableVideoError, err:
+			self._downloader.trouble(u'\nERROR: unable to download video')
+
+
+class XVideosIE(InfoExtractor):
+	"""Information extractor for xvideos.com"""
+
+	_VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
+	IE_NAME = u'xvideos'
+
+	def report_webpage(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+	def report_extraction(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+	def _simplify_title(self, title):
+		res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+		res = res.strip(ur'_')
+		return res
+
+	def _real_extract(self, url):
+		htmlParser = HTMLParser.HTMLParser()
+
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+			return
+		video_id = mobj.group(1).decode('utf-8')
+
+		self.report_webpage(video_id)
+
+		request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
+		try:
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			return
+
+		self.report_extraction(video_id)
+
+
+		# Extract video URL
+		mobj = re.search(r'flv_url=(.+?)&', webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video url')
+			return
+		video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
+
+
+		# Extract title
+		mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video title')
+			return
+		video_title = mobj.group(1).decode('utf-8')
+
+
+		# Extract video thumbnail
+		mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
+			return
+		video_thumbnail = mobj.group(1).decode('utf-8')
+
+
+
+		self._downloader.increment_downloads()
+		info = {
+			'id': video_id,
+			'url': video_url,
+			'uploader': None,
+			'upload_date': None,
+			'title': video_title,
+			'stitle': self._simplify_title(video_title),
+			'ext': 'flv',
+			'format': 'flv',
+			'thumbnail': video_thumbnail,
+			'description': None,
+			'player_url': None,
+		}
+
+		try:
+			self._downloader.process_info(info)
+		except UnavailableVideoError, err:
+			self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
+
+
+class SoundcloudIE(InfoExtractor):
+	"""Information extractor for soundcloud.com
+	   To access the media, the uid of the song and a stream token
+	   must be extracted from the page source and the script must make
+	   a request to media.soundcloud.com/crossdomain.xml. Then
+	   the media can be grabbed by requesting from an url composed
+	   of the stream token and uid
+	 """
+
+	_VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
+	IE_NAME = u'soundcloud'
+
+	def __init__(self, downloader=None):
+		InfoExtractor.__init__(self, downloader)
+
+	def report_webpage(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+	def report_extraction(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+	def _real_initialize(self):
+		return
+
+	def _real_extract(self, url):
+		htmlParser = HTMLParser.HTMLParser()
+
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+			return
+
+		# extract uploader (which is in the url)
+		uploader = mobj.group(1).decode('utf-8')
+		# extract simple title (uploader + slug of song title)
+		slug_title =  mobj.group(2).decode('utf-8')
+		simple_title = uploader + '-' + slug_title
+
+		self.report_webpage('%s/%s' % (uploader, slug_title))
+
+		request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
+		try:
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			return
+
+		self.report_extraction('%s/%s' % (uploader, slug_title))
+
+		# extract uid and access token
+		mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)   
+		if mobj:
+			video_id = mobj.group(1)
+			stream_token = mobj.group(2)
+
+		# construct media url (with uid/token) to request song
+		mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
+		mediaURL = mediaURL % (video_id, stream_token)
+
+		# description
+		description = u'No description available'
+		mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
+		if mobj:
+			description = mobj.group(1)
+		
+		# upload date
+		upload_date = None
+		mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
+		if mobj:
+			try:
+				upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
+			except Exception as e:
+				print str(e)
+
+		# for soundcloud, a request must be made to a cross domain to establish
+		# needed cookies
+		request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
+
+		try:
+			self._downloader.process_info({
+				'id':		video_id,
+				'url':		mediaURL,
+				'uploader':	uploader,
+				'upload_date':  upload_date,
+				'title':	simple_title,
+				'stitle':	simple_title,
+				'ext':		u'mp3',
+				'format':	u'NA',
+				'player_url':	None,
+				'description': description
+			})
+		except UnavailableVideoError:
+			self._downloader.trouble(u'\nERROR: unable to download video')
 
 class PostProcessor(object):
 	"""Post Processor class.
@@ -3284,11 +3626,13 @@ class PostProcessor(object):
 
 class FFmpegExtractAudioPP(PostProcessor):
 
-	def __init__(self, downloader=None, preferredcodec=None):
+	def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
 		PostProcessor.__init__(self, downloader)
 		if preferredcodec is None:
 			preferredcodec = 'best'
 		self._preferredcodec = preferredcodec
+		self._preferredquality = preferredquality
+		self._keepvideo = keepvideo
 
 	@staticmethod
 	def get_audio_codec(path):
@@ -3327,24 +3671,32 @@ class FFmpegExtractAudioPP(PostProcessor):
 
 		more_opts = []
 		if self._preferredcodec == 'best' or self._preferredcodec == filecodec:
-			if filecodec == 'aac' or filecodec == 'mp3':
+			if filecodec in ['aac', 'mp3', 'vorbis']:
 				# Lossless if possible
 				acodec = 'copy'
 				extension = filecodec
 				if filecodec == 'aac':
 					more_opts = ['-f', 'adts']
+				if filecodec == 'vorbis':
+					extension = 'ogg'
 			else:
 				# MP3 otherwise.
 				acodec = 'libmp3lame'
 				extension = 'mp3'
-				more_opts = ['-ab', '128k']
+				more_opts = []
+				if self._preferredquality is not None:
+					more_opts += ['-ab', self._preferredquality]
 		else:
 			# We convert the audio (lossy)
-			acodec = {'mp3': 'libmp3lame', 'aac': 'aac'}[self._preferredcodec]
+			acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'vorbis': 'libvorbis'}[self._preferredcodec]
 			extension = self._preferredcodec
-			more_opts = ['-ab', '128k']
+			more_opts = []
+			if self._preferredquality is not None:
+				more_opts += ['-ab', self._preferredquality]
 			if self._preferredcodec == 'aac':
 				more_opts += ['-f', 'adts']
+			if self._preferredcodec == 'vorbis':
+				extension = 'ogg'
 
 		(prefix, ext) = os.path.splitext(path)
 		new_path = prefix + '.' + extension
@@ -3355,11 +3707,19 @@ class FFmpegExtractAudioPP(PostProcessor):
 			self._downloader.to_stderr(u'WARNING: error running ffmpeg')
 			return None
 
-		try:
-			os.remove(path)
-		except (IOError, OSError):
-			self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
-			return None
+ 		# Try to update the date time for extracted audio file.
+		if information.get('filetime') is not None:
+			try:
+				os.utime(new_path, (time.time(), information['filetime']))
+			except:
+				self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
+
+		if not self._keepvideo:
+			try:
+				os.remove(path)
+			except (IOError, OSError):
+				self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
+				return None
 
 		information['filepath'] = new_path
 		return information
@@ -3377,6 +3737,11 @@ def updateSelf(downloader, filename):
 		try:
 			urlh = urllib.urlopen(UPDATE_URL)
 			newcontent = urlh.read()
+			
+			vmatch = re.search("__version__ = '([^']+)'", newcontent)
+			if vmatch is not None and vmatch.group(1) == __version__:
+				downloader.to_screen('youtube-dl is up-to-date (' + __version__ + ')')
+				return
 		finally:
 			urlh.close()
 	except (IOError, OSError), err:
@@ -3391,7 +3756,7 @@ def updateSelf(downloader, filename):
 	except (IOError, OSError), err:
 		sys.exit('ERROR: unable to overwrite current version')
 
-	downloader.to_screen('Updated youtube-dl. Restart to use the new version.')
+	downloader.to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.')
 
 def parseOpts():
 	# Deferred imports
@@ -3489,9 +3854,11 @@ def parseOpts():
 	video_format.add_option('-f', '--format',
 			action='store', dest='format', metavar='FORMAT', help='video format code')
 	video_format.add_option('--all-formats',
-			action='store_const', dest='format', help='download all available video formats', const='-1')
+			action='store_const', dest='format', help='download all available video formats', const='all')
 	video_format.add_option('--max-quality',
 			action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
+	video_format.add_option('-F', '--list-formats',
+			action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 
 
 	verbosity.add_option('-q', '--quiet',
@@ -3513,6 +3880,9 @@ def parseOpts():
 	verbosity.add_option('--get-filename',
 			action='store_true', dest='getfilename',
 			help='simulate, quiet but print output filename', default=False)
+	verbosity.add_option('--get-format',
+			action='store_true', dest='getformat',
+			help='simulate, quiet but print output format', default=False)
 	verbosity.add_option('--no-progress',
 			action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 	verbosity.add_option('--console-title',
@@ -3528,15 +3898,18 @@ def parseOpts():
 			action='store_true', dest='autonumber',
 			help='number downloaded files starting from 00000', default=False)
 	filesystem.add_option('-o', '--output',
-			dest='outtmpl', metavar='TEMPLATE', help='output filename template')
+			dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, and %% for a literal percent')
 	filesystem.add_option('-a', '--batch-file',
 			dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 	filesystem.add_option('-w', '--no-overwrites',
 			action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 	filesystem.add_option('-c', '--continue',
 			action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
+	filesystem.add_option('--no-continue',
+			action='store_false', dest='continue_dl',
+			help='do not resume partially downloaded files (restart from beginning)')
 	filesystem.add_option('--cookies',
-			dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
+			dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 	filesystem.add_option('--no-part',
 			action='store_true', dest='nopart', help='do not use .part files', default=False)
 	filesystem.add_option('--no-mtime',
@@ -3553,7 +3926,11 @@ def parseOpts():
 	postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
 			help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
 	postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-			help='"best", "aac" or "mp3"; best by default')
+			help='"best", "aac", "vorbis" or "mp3"; best by default')
+	postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K',
+			help='ffmpeg audio bitrate specification, 128k by default')
+	postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
+			help='keeps the video file on disk after the post-processing; the video is erased by default')
 
 
 	parser.add_option_group(general)
@@ -3576,12 +3953,12 @@ def gen_extractors():
 	google_ie = GoogleIE()
 	yahoo_ie = YahooIE()
 	return [
-		youtube_ie,
-		MetacafeIE(youtube_ie),
-		DailymotionIE(),
 		YoutubePlaylistIE(youtube_ie),
 		YoutubeUserIE(youtube_ie),
 		YoutubeSearchIE(youtube_ie),
+		youtube_ie,
+		MetacafeIE(youtube_ie),
+		DailymotionIE(),
 		google_ie,
 		GoogleSearchIE(google_ie),
 		PhotobucketIE(),
@@ -3594,6 +3971,9 @@ def gen_extractors():
 		MyVideoIE(),
 		ComedyCentralIE(),
 		EscapistIE(),
+		CollegeHumorIE(),
+		XVideosIE(),
+        SoundcloudIE(),
 
 		GenericIE()
 	]
@@ -3683,7 +4063,7 @@ def main():
 	except (TypeError, ValueError), err:
 		parser.error(u'invalid playlist end number specified')
 	if opts.extractaudio:
-		if opts.audioformat not in ['best', 'aac', 'mp3']:
+		if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis']:
 			parser.error(u'invalid audio format specified')
 
 	# File downloader
@@ -3691,16 +4071,18 @@ def main():
 		'usenetrc': opts.usenetrc,
 		'username': opts.username,
 		'password': opts.password,
-		'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
+		'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 		'forceurl': opts.geturl,
 		'forcetitle': opts.gettitle,
 		'forcethumbnail': opts.getthumbnail,
 		'forcedescription': opts.getdescription,
 		'forcefilename': opts.getfilename,
+		'forceformat': opts.getformat,
 		'simulate': opts.simulate,
-		'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
+		'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 		'format': opts.format,
 		'format_limit': opts.format_limit,
+		'listformats': opts.listformats,
 		'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
 			or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
 			or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
@@ -3733,7 +4115,7 @@ def main():
 
 	# PostProcessors
 	if opts.extractaudio:
-		fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat))
+		fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
 
 	# Update version
 	if opts.update_self: