[wwwsearch-commits] r29098 - in wwwsearch/mechanize/trunk: mechanize test
jjlee at codespeak.net
jjlee at codespeak.net
Wed Jun 21 22:05:19 CEST 2006
Author: jjlee
Date: Wed Jun 21 22:05:17 2006
New Revision: 29098
Modified:
wwwsearch/mechanize/trunk/mechanize/_opener.py
wwwsearch/mechanize/trunk/mechanize/_urllib2.py
wwwsearch/mechanize/trunk/mechanize/_urllib2_support.py
wwwsearch/mechanize/trunk/test/test_urllib2.py
Log:
Move opener code from _urllib2_support into _opener
Modified: wwwsearch/mechanize/trunk/mechanize/_opener.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_opener.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_opener.py Wed Jun 21 22:05:17 2006
@@ -9,17 +9,22 @@
"""
-import urllib2, bisect, urlparse
-
-from _util import isstringlike
-from _request import Request
-
+import urllib2, bisect, urlparse, httplib, types
+try:
+ import threading as _threading
+except ImportError:
+ import dummy_threading as _threading
try:
set
except NameError:
import sets
set = sets.Set
+import _urllib2_support
+from _util import isstringlike
+from _request import Request
+
+
class OpenerDirector(urllib2.OpenerDirector):
def __init__(self):
urllib2.OpenerDirector.__init__(self)
@@ -237,3 +242,93 @@
raise IOError("incomplete retrieval error",
"got only %d bytes out of %d" % (read,size))
return result
+
+
+class OpenerFactory:
+ """This class's interface is quite likely to change."""
+
+ default_classes = [
+ # handlers
+ urllib2.ProxyHandler,
+ urllib2.UnknownHandler,
+ _urllib2_support.HTTPHandler, # derived from new AbstractHTTPHandler
+ urllib2.HTTPDefaultErrorHandler,
+ _urllib2_support.HTTPRedirectHandler, # bugfixed
+ urllib2.FTPHandler,
+ urllib2.FileHandler,
+ # processors
+ _urllib2_support.HTTPRequestUpgradeProcessor,
+ _urllib2_support.HTTPCookieProcessor,
+ _urllib2_support.HTTPErrorProcessor,
+ ]
+ handlers = []
+ replacement_handlers = []
+
+ def __init__(self, klass=OpenerDirector):
+ self.klass = klass
+
+ def build_opener(self, *handlers):
+ """Create an opener object from a list of handlers and processors.
+
+ The opener will use several default handlers and processors, including
+ support for HTTP and FTP.
+
+ If any of the handlers passed as arguments are subclasses of the
+ default handlers, the default handlers will not be used.
+
+ """
+ opener = self.klass()
+ default_classes = list(self.default_classes)
+ if hasattr(httplib, 'HTTPS'):
+ default_classes.append(_urllib2_support.HTTPSHandler)
+ skip = []
+ for klass in default_classes:
+ for check in handlers:
+ if type(check) == types.ClassType:
+ if issubclass(check, klass):
+ skip.append(klass)
+ elif type(check) == types.InstanceType:
+ if isinstance(check, klass):
+ skip.append(klass)
+ for klass in skip:
+ default_classes.remove(klass)
+
+ for klass in default_classes:
+ opener.add_handler(klass())
+ for h in handlers:
+ if type(h) == types.ClassType:
+ h = h()
+ opener.add_handler(h)
+
+ return opener
+
+
+build_opener = OpenerFactory().build_opener
+
+_opener = None
+urlopen_lock = _threading.Lock()
+def urlopen(url, data=None):
+ global _opener
+ if _opener is None:
+ urlopen_lock.acquire()
+ try:
+ if _opener is None:
+ _opener = build_opener()
+ finally:
+ urlopen_lock.release()
+ return _opener.open(url, data)
+
+def urlretrieve(url, filename=None, reporthook=None, data=None):
+ global _opener
+ if _opener is None:
+ urlopen_lock.acquire()
+ try:
+ if _opener is None:
+ _opener = build_opener()
+ finally:
+ urlopen_lock.release()
+ return _opener.retrieve(url, filename, reporthook, data)
+
+def install_opener(opener):
+ global _opener
+ _opener = opener
Modified: wwwsearch/mechanize/trunk/mechanize/_urllib2.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_urllib2.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_urllib2.py Wed Jun 21 22:05:17 2006
@@ -5,7 +5,9 @@
HTTPError, \
GopherError
# ...and from mechanize
-from _opener import OpenerDirector
+from _opener import OpenerDirector, \
+ build_opener, install_opener, urlopen, \
+ OpenerFactory, urlretrieve
from _auth import \
HTTPPasswordMgr, \
HTTPPasswordMgrWithDefaultRealm, \
@@ -20,8 +22,6 @@
HTTPSClientCertMgr
from _urllib2_support import \
Request, \
- build_opener, install_opener, urlopen, \
- OpenerFactory, urlretrieve, \
RobotExclusionError
# handlers...
Modified: wwwsearch/mechanize/trunk/mechanize/_urllib2_support.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_urllib2_support.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_urllib2_support.py Wed Jun 21 22:05:17 2006
@@ -11,16 +11,11 @@
"""
-import copy, time, tempfile, htmlentitydefs, re, logging, types, \
- socket, urlparse, urllib2, urllib, httplib, sgmllib
+import copy, time, tempfile, htmlentitydefs, re, logging, socket, urlparse, \
+ urllib2, urllib, httplib, sgmllib
from urllib2 import URLError, HTTPError, BaseHandler
from cStringIO import StringIO
-try:
- import threading as _threading
-except ImportError:
- import dummy_threading as _threading
-import _opener
from _request import Request
from _util import isstringlike
from _response import closeable_response, response_seek_wrapper
@@ -648,91 +643,3 @@
return self.do_open(conn_factory, req)
https_request = AbstractHTTPHandler.do_request_
-
-class OpenerFactory:
- """This class's interface is quite likely to change."""
-
- default_classes = [
- # handlers
- urllib2.ProxyHandler,
- urllib2.UnknownHandler,
- HTTPHandler, # from this module (derived from new AbstractHTTPHandler)
- urllib2.HTTPDefaultErrorHandler,
- HTTPRedirectHandler, # from this module (bugfixed)
- urllib2.FTPHandler,
- urllib2.FileHandler,
- # processors
- HTTPRequestUpgradeProcessor,
- HTTPCookieProcessor,
- HTTPErrorProcessor
- ]
- handlers = []
- replacement_handlers = []
-
- def __init__(self, klass=_opener.OpenerDirector):
- self.klass = klass
-
- def build_opener(self, *handlers):
- """Create an opener object from a list of handlers and processors.
-
- The opener will use several default handlers and processors, including
- support for HTTP and FTP.
-
- If any of the handlers passed as arguments are subclasses of the
- default handlers, the default handlers will not be used.
-
- """
- opener = self.klass()
- default_classes = list(self.default_classes)
- if hasattr(httplib, 'HTTPS'):
- default_classes.append(HTTPSHandler)
- skip = []
- for klass in default_classes:
- for check in handlers:
- if type(check) == types.ClassType:
- if issubclass(check, klass):
- skip.append(klass)
- elif type(check) == types.InstanceType:
- if isinstance(check, klass):
- skip.append(klass)
- for klass in skip:
- default_classes.remove(klass)
-
- for klass in default_classes:
- opener.add_handler(klass())
- for h in handlers:
- if type(h) == types.ClassType:
- h = h()
- opener.add_handler(h)
-
- return opener
-
-build_opener = OpenerFactory().build_opener
-
-_opener = None
-urlopen_lock = _threading.Lock()
-def urlopen(url, data=None):
- global _opener
- if _opener is None:
- urlopen_lock.acquire()
- try:
- if _opener is None:
- _opener = build_opener()
- finally:
- urlopen_lock.release()
- return _opener.open(url, data)
-
-def urlretrieve(url, filename=None, reporthook=None, data=None):
- global _opener
- if _opener is None:
- urlopen_lock.acquire()
- try:
- if _opener is None:
- _opener = build_opener()
- finally:
- urlopen_lock.release()
- return _opener.retrieve(url, filename, reporthook, data)
-
-def install_opener(opener):
- global _opener
- _opener = opener
Modified: wwwsearch/mechanize/trunk/test/test_urllib2.py
==============================================================================
--- wwwsearch/mechanize/trunk/test/test_urllib2.py (original)
+++ wwwsearch/mechanize/trunk/test/test_urllib2.py Wed Jun 21 22:05:17 2006
@@ -14,12 +14,12 @@
import mechanize
from mechanize._urllib2_support import Request, AbstractHTTPHandler, \
- build_opener, parse_head, urlopen
+ parse_head
from mechanize import HTTPRedirectHandler, HTTPRequestUpgradeProcessor, \
HTTPEquivProcessor, HTTPRefreshProcessor, SeekableProcessor, \
HTTPCookieProcessor, HTTPRefererProcessor, \
HTTPErrorProcessor, HTTPHandler
-from mechanize import OpenerDirector
+from mechanize import OpenerDirector, build_opener, urlopen
## from logging import getLogger, DEBUG
## l = getLogger("mechanize")
More information about the wwwsearch-commits
mailing list