transplant 0f6e296a8ec97bce781b722bbaf4cff95642d253

[youtube-dl.git] / youtube_dl / __init__.py

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ffd837c..7772315 100755 (executable)
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -18,12 +18,14 @@ __authors__  = (
        )
 
 __license__ = 'Public Domain'
-__version__ = '2012.01.08b'
+__version__ = '2012.02.27'
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 
+
 import cookielib
 import datetime
+import getpass
 import gzip
 import htmlentitydefs
 import HTMLParser
@@ -31,9 +33,11 @@ import httplib
 import locale
 import math
 import netrc
+import optparse
 import os
 import os.path
 import re
+import shlex
 import socket
 import string
 import subprocess
@@ -896,7 +900,15 @@ class FileDownloader(object):
                # the connection was interrumpted and resuming appears to be
                # possible. This is part of rtmpdump's normal usage, AFAIK.
                basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
-               retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
+               args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
+               if self.params.get('verbose', False):
+                       try:
+                               import pipes
+                               shell_quote = lambda args: ' '.join(map(pipes.quote, args))
+                       except ImportError:
+                               shell_quote = repr
+                       self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
+               retval = subprocess.call(args)
                while retval == 2 or retval == 1:
                        prevsize = os.path.getsize(_encodeFilename(tmpfilename))
                        self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
@@ -1369,10 +1381,9 @@ class YoutubeIE(InfoExtractor):
                        lxml.etree
                except NameError:
                        video_description = u'No description available.'
-                       if self._downloader.params.get('forcedescription', False) or self._downloader.params.get('writedescription', False):
-                               mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
-                               if mobj is not None:
-                                       video_description = mobj.group(1).decode('utf-8')
+                       mobj = re.search(r'<meta name="description" content="(.*?)">', video_webpage)
+                       if mobj is not None:
+                               video_description = mobj.group(1).decode('utf-8')
                else:
                        html_parser = lxml.etree.HTMLParser(encoding='utf-8')
                        vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
@@ -2329,8 +2340,8 @@ class GoogleSearchIE(InfoExtractor):
        """Information Extractor for Google Video search queries."""
        _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
        _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
-       _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
-       _MORE_PAGES_INDICATOR = r'<span>Next</span>'
+       _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
+       _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
        _google_ie = None
        _max_google_results = 1000
        IE_NAME = u'video.google:search'
@@ -2381,12 +2392,11 @@ class GoogleSearchIE(InfoExtractor):
                """Downloads a specified number of results for a query"""
 
                video_ids = []
-               already_seen = set()
-               pagenum = 1
+               pagenum = 0
 
                while True:
                        self.report_download_page(query, pagenum)
-                       result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
+                       result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum*10)
                        request = urllib2.Request(result_url)
                        try:
                                page = urllib2.urlopen(request).read()
@@ -2397,9 +2407,8 @@ class GoogleSearchIE(InfoExtractor):
                        # Extract video identifiers
                        for mobj in re.finditer(self._VIDEO_INDICATOR, page):
                                video_id = mobj.group(1)
-                               if video_id not in already_seen:
+                               if video_id not in video_ids:
                                        video_ids.append(video_id)
-                                       already_seen.add(video_id)
                                        if len(video_ids) == n:
                                                # Specified n videos reached
                                                for id in video_ids:
@@ -4205,11 +4214,6 @@ def updateSelf(downloader, filename):
        downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
 
 def parseOpts():
-       # Deferred imports
-       import getpass
-       import optparse
-       import shlex
-
        def _readOptions(filename_bytes):
                try:
                        optionf = open(filename_bytes)
@@ -4590,6 +4594,7 @@ def _real_main():
                'rejecttitle': opts.rejecttitle,
                'max_downloads': opts.max_downloads,
                'prefer_free_formats': opts.prefer_free_formats,
+               'verbose': opts.verbose,
                })
        for extractor in extractors:
                fd.add_info_extractor(extractor)