* 0.2.6 release:
* Don't allow underlying parser errors (e.g. SGMLParseError)
From jjlee at codespeak.net Fri May 25 20:46:26 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Fri, 25 May 2007 20:46:26 +0200 (CEST)
Subject: [wwwsearch-commits] r43646 -
wwwsearch/ClientForm/tag/release/0.2.7-2007-05-24T21:35:32
Message-ID: <20070525184626.2B35F8092@code0.codespeak.net>
Author: jjlee
Date: Fri May 25 20:46:25 2007
New Revision: 43646
Removed:
wwwsearch/ClientForm/tag/release/0.2.7-2007-05-24T21:35:32/
Log:
Remove unused release tag
From jjlee at codespeak.net Fri May 25 20:46:43 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Fri, 25 May 2007 20:46:43 +0200 (CEST)
Subject: [wwwsearch-commits] r43647 -
wwwsearch/ClientForm/tag/release/0.2.7-2007-05-25T19:47:20
Message-ID: <20070525184643.5D3B38092@code0.codespeak.net>
Author: jjlee
Date: Fri May 25 20:46:43 2007
New Revision: 43647
Added:
wwwsearch/ClientForm/tag/release/0.2.7-2007-05-25T19:47:20/
- copied from r43646, wwwsearch/ClientForm/trunk/
Modified:
wwwsearch/ClientForm/tag/release/0.2.7-2007-05-25T19:47:20/setup.cfg
Log:
Tagged ClientForm (trunk -r HEAD, from working copy) release 0.2.7-2007-05-25T19:47:20
Modified: wwwsearch/ClientForm/tag/release/0.2.7-2007-05-25T19:47:20/setup.cfg
==============================================================================
--- wwwsearch/ClientForm/trunk/setup.cfg (original)
+++ wwwsearch/ClientForm/tag/release/0.2.7-2007-05-25T19:47:20/setup.cfg Fri May 25 20:46:43 2007
@@ -1,3 +0,0 @@
-[egg_info]
-tag_build = .dev
-tag_svn_revision = 1
From jjlee at codespeak.net Fri May 25 21:29:17 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Fri, 25 May 2007 21:29:17 +0200 (CEST)
Subject: [wwwsearch-commits] r43651 - wwwsearch/ClientForm/trunk
Message-ID: <20070525192917.B08F88092@code0.codespeak.net>
Author: jjlee
Date: Fri May 25 21:29:16 2007
New Revision: 43651
Modified:
wwwsearch/ClientForm/trunk/test.py
Log:
Revive a bit of accidentally-unreached test code
Modified: wwwsearch/ClientForm/trunk/test.py
==============================================================================
--- wwwsearch/ClientForm/trunk/test.py (original)
+++ wwwsearch/ClientForm/trunk/test.py Fri May 25 21:29:16 2007
@@ -1139,25 +1139,27 @@
def testDisabledRadio(self):
for compat in False, True:
- self._testDisabledCheckbox(compat)
+ self._testDisabledRadio(compat)
def _testDisabledRadio(self, compat):
file = StringIO(
"""""")
hide_deprecations()
forms = ClientForm.ParseFile(file, "http://localhost/",
backwards_compat=compat)
form = forms[0]
control = form.find_control('foo')
- self.assert_(control.disabled)
+
+ # since all items are disabled, .fixup() should not select
+ # anything
+ self.assertEquals(
+ [item.name for item in control.items if item.selected],
+ [],
+ )
+ reset_deprecations()
def testDisabledCheckbox(self):
for compat in False, True:
From jjlee at codespeak.net Mon May 28 17:22:10 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Mon, 28 May 2007 17:22:10 +0200 (CEST)
Subject: [wwwsearch-commits] r43803 - in wwwsearch/mechanize/trunk:
mechanize test
Message-ID: <20070528152210.6107C809C@code0.codespeak.net>
Author: jjlee
Date: Mon May 28 17:22:09 2007
New Revision: 43803
Modified:
wwwsearch/mechanize/trunk/mechanize/_http.py
wwwsearch/mechanize/trunk/test/test_urllib2.py
Log:
Redirected robots.txt fetch no longer results in another attempted robots.txt fetch to check the redirection is allowed! Needs revisiting post-stable release.
Modified: wwwsearch/mechanize/trunk/mechanize/_http.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_http.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_http.py Mon May 28 17:22:09 2007
@@ -79,12 +79,14 @@
# the same.
# XXX really refresh redirections should be visiting; tricky to
# fix, so this will wait until post-stable release
- return Request(newurl,
- headers=req.headers,
- origin_req_host=req.get_origin_req_host(),
- unverifiable=True,
- visit=False,
- )
+ new = Request(newurl,
+ headers=req.headers,
+ origin_req_host=req.get_origin_req_host(),
+ unverifiable=True,
+ visit=False,
+ )
+ new._origin_req = getattr(req, "_origin_req", req)
+ return new
else:
raise HTTPError(req.get_full_url(), code, msg, headers, fp)
@@ -412,6 +414,15 @@
return request
host = request.get_host()
+
+ # robots.txt requests don't need to be allowed by robots.txt :-)
+ origin_req = getattr(request, "_origin_req", None)
+ if (origin_req is not None and
+ origin_req.get_selector() == "/robots.txt" and
+ origin_req.get_host() == host
+ ):
+ return request
+
if host != self._host:
self.rfp = self.rfp_class()
try:
Modified: wwwsearch/mechanize/trunk/test/test_urllib2.py
==============================================================================
--- wwwsearch/mechanize/trunk/test/test_urllib2.py (original)
+++ wwwsearch/mechanize/trunk/test/test_urllib2.py Mon May 28 17:22:09 2007
@@ -806,6 +806,41 @@
h.http_request(req)
self.assert_(rfpc.calls == [])
+ def test_redirected_robots_txt(self):
+ # redirected robots.txt fetch shouldn't result in another attempted
+ # robots.txt fetch to check the redirection is allowed!
+ import mechanize
+ from mechanize import build_opener, HTTPHandler, \
+ HTTPDefaultErrorHandler, HTTPRedirectHandler, \
+ HTTPRobotRulesProcessor
+
+ class MockHTTPHandler(mechanize.BaseHandler):
+ def __init__(self):
+ self.requests = []
+ def http_open(self, req):
+ import mimetools, httplib, copy
+ from StringIO import StringIO
+ self.requests.append(copy.deepcopy(req))
+ if req.get_full_url() == "http://example.com/robots.txt":
+ hdr = "Location: http://example.com/en/robots.txt\r\n\r\n"
+ msg = mimetools.Message(StringIO(hdr))
+ return self.parent.error(
+ "http", req, test_response(), 302, "Blah", msg)
+ else:
+ return test_response("Allow: *", [], req.get_full_url())
+
+ hh = MockHTTPHandler()
+ hdeh = HTTPDefaultErrorHandler()
+ hrh = HTTPRedirectHandler()
+ rh = HTTPRobotRulesProcessor()
+ o = build_test_opener(hh, hdeh, hrh, rh)
+ o.open("http://example.com/")
+ self.assertEqual([req.get_full_url() for req in hh.requests],
+ ["http://example.com/robots.txt",
+ "http://example.com/en/robots.txt",
+ "http://example.com/",
+ ])
+
def test_cookies(self):
cj = MockCookieJar()
h = HTTPCookieProcessor(cj)
From jjlee at codespeak.net Mon May 28 17:34:40 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Mon, 28 May 2007 17:34:40 +0200 (CEST)
Subject: [wwwsearch-commits] r43808 - in wwwsearch/mechanize/trunk:
mechanize test
Message-ID: <20070528153440.57C3680A4@code0.codespeak.net>
Author: jjlee
Date: Mon May 28 17:34:39 2007
New Revision: 43808
Modified:
wwwsearch/mechanize/trunk/mechanize/_rfc3986.py
wwwsearch/mechanize/trunk/test/test_rfc3986.doctest
Log:
Fix exception raised by RFC 3986 implementation with urljoin(base, '/..')
Modified: wwwsearch/mechanize/trunk/mechanize/_rfc3986.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_rfc3986.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_rfc3986.py Mon May 28 17:34:39 2007
@@ -200,7 +200,8 @@
continue
if path == "/..":
path = "/"
- r.pop()
+ if r:
+ r.pop()
continue
# D
if path == ".":
Modified: wwwsearch/mechanize/trunk/test/test_rfc3986.doctest
==============================================================================
--- wwwsearch/mechanize/trunk/test/test_rfc3986.doctest (original)
+++ wwwsearch/mechanize/trunk/test/test_rfc3986.doctest Mon May 28 17:34:39 2007
@@ -59,6 +59,10 @@
'/'
>>> remove_dot_segments("./")
''
+>>> remove_dot_segments("/..")
+'/'
+>>> remove_dot_segments("/../")
+'/'
Examples from RFC 3986 section 5.4
@@ -154,3 +158,11 @@
'http://a/b/c/g#s/../x'
>>> join("http:g")
'http://a/b/c/g'
+
+
+Additional urljoin tests, not taken from RFC:
+
+>>> join("/..")
+'http://a/'
+>>> join("/../")
+'http://a/'
From jjlee at codespeak.net Mon May 28 18:11:08 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Mon, 28 May 2007 18:11:08 +0200 (CEST)
Subject: [wwwsearch-commits] r43811 - wwwsearch/mechanize/trunk/mechanize
Message-ID: <20070528161108.1CBB68099@code0.codespeak.net>
Author: jjlee
Date: Mon May 28 18:11:07 2007
New Revision: 43811
Modified:
wwwsearch/mechanize/trunk/mechanize/_http.py
Log:
Add some debugging code to show URL with HTTPErrors (would be nice to always have this, but don't want to change the precise exception raised right now)
Modified: wwwsearch/mechanize/trunk/mechanize/_http.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_http.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_http.py Mon May 28 18:11:07 2007
@@ -27,6 +27,12 @@
debug = logging.getLogger("mechanize").debug
+# monkeypatch urllib2.HTTPError to show URL
+## def urllib2_str(self):
+## return 'HTTP Error %s: %s (%s)' % (
+## self.code, self.msg, self.geturl())
+## urllib2.HTTPError.__str__ = urllib2_str
+
CHUNK = 1024 # size of chunks fed to HTML HEAD parser, in bytes
DEFAULT_ENCODING = 'latin-1'
From jjlee at codespeak.net Mon May 28 23:06:57 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Mon, 28 May 2007 23:06:57 +0200 (CEST)
Subject: [wwwsearch-commits] r43826 - in wwwsearch/mechanize/trunk: .
test-tools
Message-ID: <20070528210657.442B48095@code0.codespeak.net>
Author: jjlee
Date: Mon May 28 23:06:56 2007
New Revision: 43826
Added:
wwwsearch/mechanize/trunk/test-tools/testprogram.py
wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py
Modified:
wwwsearch/mechanize/trunk/functional_tests.py
wwwsearch/mechanize/trunk/test.py
Log:
Add an -l option to run the functional tests against a local twisted.web2-based server. There are still a few tests that always run against the wwwsearch.sf.net site -- need to extend the local srver a bit to cover these.
Modified: wwwsearch/mechanize/trunk/functional_tests.py
==============================================================================
--- wwwsearch/mechanize/trunk/functional_tests.py (original)
+++ wwwsearch/mechanize/trunk/functional_tests.py Mon May 28 23:06:56 2007
@@ -2,7 +2,9 @@
# These tests access the network.
-import os
+# thanks Moof (aka Giles Antonio Radford) for some of these
+
+import os, sys
from unittest import TestCase
import mechanize
@@ -11,6 +13,14 @@
HTTPHandler, HTTPRefreshProcessor, \
HTTPEquivProcessor, HTTPRedirectHandler, \
HTTPRedirectDebugProcessor, HTTPResponseDebugProcessor
+from mechanize._rfc3986 import urljoin
+
+# XXX
+# document twisted.web2 install (I forgot how I did it -- reinstall!)
+# implement remaining stuff used by functional_tests.py
+# in twisted-localserver.py:
+# - 302 followed by 404 response
+# - helper cgi script for cookies &c.
#from cookielib import CookieJar
#from urllib2 import build_opener, install_opener, urlopen
@@ -20,8 +30,9 @@
## import logging
## logger = logging.getLogger("mechanize")
-## logger.addHandler(logging.StreamHandler())
-## logger.setLevel(logging.DEBUG)
+## logger.addHandler(logging.StreamHandler(sys.stdout))
+## #logger.setLevel(logging.DEBUG)
+## logger.setLevel(logging.INFO)
def sanepathname2url(path):
@@ -39,7 +50,7 @@
self.browser = mechanize.Browser()
def test_simple(self):
- self.browser.open('http://wwwsearch.sourceforge.net/')
+ self.browser.open(self.uri)
self.assertEqual(self.browser.title(), 'Python bits')
# relative URL
self.browser.open('/mechanize/')
@@ -52,7 +63,7 @@
import urllib2
self.assertRaises(
urllib2.HTTPError,
- self.browser.open, "http://wwwsearch.sf.net/doesnotexist"
+ self.browser.open, "http://wwwsearch.sf.net/doesnotexist",
)
def test_reread(self):
@@ -60,7 +71,7 @@
# be true for e.g. mechanize.OpenerDirector when mechanize's own
# handlers are in use, but is guaranteed to be true for
# mechanize.Browser)
- r = self.browser.open('http://wwwsearch.sourceforge.net/')
+ r = self.browser.open(self.uri)
data = r.read()
r.close()
r.seek(0)
@@ -70,12 +81,12 @@
def test_error_recovery(self):
self.assertRaises(OSError, self.browser.open,
'file:///c|thisnoexistyiufheiurgbueirgbue')
- self.browser.open('http://wwwsearch.sourceforge.net/')
+ self.browser.open(self.uri)
self.assertEqual(self.browser.title(), 'Python bits')
def test_redirect(self):
# 301 redirect due to missing final '/'
- r = self.browser.open('http://wwwsearch.sourceforge.net/bits')
+ r = self.browser.open(urljoin(self.uri, "bits"))
self.assertEqual(r.code, 200)
self.assert_("GeneralFAQ.html" in r.read(2048))
@@ -92,7 +103,7 @@
self.assertRaises(mechanize.BrowserStateError, br.back)
test_state(self.browser)
# note this involves a redirect, which should itself be non-visiting
- r = self.browser.open_novisit("http://wwwsearch.sourceforge.net/bits")
+ r = self.browser.open_novisit(urljoin(self.uri, "bits"))
test_state(self.browser)
self.assert_("GeneralFAQ.html" in r.read(2048))
@@ -102,7 +113,7 @@
ua = mechanize.UserAgent()
ua.set_seekable_responses(False)
ua.set_handle_equiv(False)
- response = ua.open('http://wwwsearch.sourceforge.net/')
+ response = ua.open(self.uri)
self.failIf(hasattr(response, "seek"))
data = response.read()
self.assert_("Python bits" in data)
@@ -112,7 +123,7 @@
def test_seek(self):
br = mechanize.Browser()
- r = br.open("http://wwwsearch.sourceforge.net/")
+ r = br.open(self.uri)
html = r.read()
r.seek(0)
self.assertEqual(r.read(), html)
@@ -120,7 +131,7 @@
def test_seekable_response_opener(self):
opener = mechanize.OpenerFactory(
mechanize.SeekableResponseOpener).build_opener()
- r = opener.open("http://wwwsearch.sourceforge.net/bits/cctest2.txt")
+ r = opener.open(urljoin(self.uri, "bits/cctest2.txt"))
r.read()
r.seek(0)
self.assertEqual(r.read(),
@@ -130,11 +141,10 @@
def test_no_seek(self):
# should be possible to turn off UserAgent's .seek() functionality
def check_no_seek(opener):
- r = opener.open(
- "http://wwwsearch.sourceforge.net/bits/cctest2.txt")
+ r = opener.open(urljoin(self.uri, "bits/cctest2.txt"))
self.assert_(not hasattr(r, "seek"))
try:
- opener.open("http://wwwsearch.sourceforge.net/nonexistent")
+ opener.open(urljoin(self.uri, "nonexistent"))
except mechanize.HTTPError, exc:
self.assert_(not hasattr(exc, "seek"))
@@ -154,13 +164,12 @@
# .seek() method, then raised HTTPError exceptions should also have the
# .seek() method
def check(opener, excs_also):
- r = opener.open(
- "http://wwwsearch.sourceforge.net/bits/cctest2.txt")
+ r = opener.open(urljoin(self.uri, "bits/cctest2.txt"))
data = r.read()
r.seek(0)
self.assertEqual(data, r.read(), r.get_data())
try:
- opener.open("http://wwwsearch.sourceforge.net/nonexistent")
+ opener.open(urljoin(self.uri, "nonexistent"))
except mechanize.HTTPError, exc:
data = exc.read()
if excs_also:
@@ -189,7 +198,7 @@
def test_set_response(self):
br = mechanize.Browser()
- r = br.open("http://wwwsearch.sourceforge.net/")
+ r = br.open(self.uri)
html = r.read()
self.assertEqual(br.title(), "Python bits")
@@ -226,7 +235,7 @@
import pickle
b = mechanize.Browser()
- r = b.open("http://wwwsearch.sourceforge.net/bits/cctest2.txt")
+ r = b.open(urljoin(self.uri, "bits/cctest2.txt"))
r.read()
r.close()
@@ -301,11 +310,11 @@
plain_opener = mechanize.build_opener(mechanize.HTTPRobotRulesProcessor)
browser = mechanize.Browser()
for opener in plain_opener, browser:
- r = opener.open("http://wwwsearch.sourceforge.net/robots")
+ r = opener.open(urljoin(self.uri, "robots"))
self.assertEqual(r.code, 200)
self.assertRaises(
mechanize.RobotExclusionError,
- opener.open, "http://wwwsearch.sourceforge.net/norobots")
+ opener.open, urljoin(self.uri, "norobots"))
def test_urlretrieve(self):
url = "http://www.python.org/"
@@ -341,11 +350,10 @@
def test_reload_read_incomplete(self):
from mechanize import Browser
browser = Browser()
- r1 = browser.open(
- "http://wwwsearch.sourceforge.net/bits/mechanize_reload_test.html")
+ r1 = browser.open(urljoin(self.uri, "bits/mechanize_reload_test.html"))
# if we don't do anything and go straight to another page, most of the
# last page's response won't be .read()...
- r2 = browser.open("http://wwwsearch.sourceforge.net/mechanize")
+ r2 = browser.open(urljoin(self.uri, "mechanize"))
self.assert_(len(r1.get_data()) < 4097) # we only .read() a little bit
# ...so if we then go back, .follow_link() for a link near the end (a
# few kb in, past the point that always gets read in HTML files because
@@ -378,5 +386,29 @@
if __name__ == "__main__":
- import unittest
- unittest.main()
+ import sys
+ sys.path.insert(0, "test-tools")
+ import testprogram
+ USAGE_EXAMPLES = """
+Examples:
+ %(progName)s
+ - run all tests
+ %(progName)s functional_tests.SimpleTests
+ - run all 'test*' test methods in class SimpleTests
+ %(progName)s functional_tests.SimpleTests.test_redirect
+ - run SimpleTests.test_redirect
+
+ %(progName)s -l
+ - start a local Twisted HTTP server and run the functional
+ tests against that, rather than against SourceForge
+ (quicker!)
+ Note not all the functional tests use the local server yet
+ -- some currently always access the internet regardless of
+ this option and the --uri option.
+"""
+ prog = testprogram.TestProgram(
+ ["functional_tests"],
+ localServerProcess=testprogram.TwistedServerProcess(),
+ usageExamples=USAGE_EXAMPLES,
+ )
+ result = prog.runTests()
Added: wwwsearch/mechanize/trunk/test-tools/testprogram.py
==============================================================================
--- (empty file)
+++ wwwsearch/mechanize/trunk/test-tools/testprogram.py Mon May 28 23:06:56 2007
@@ -0,0 +1,311 @@
+"""Local server and cgitb support."""
+
+import cgitb
+#cgitb.enable(format="text")
+
+import sys, os, traceback, logging, glob, time
+from unittest import defaultTestLoader, TextTestRunner, TestSuite, TestCase, \
+ _TextTestResult
+
+
+class ServerProcess:
+
+ def __init__(self, filename, name=None):
+ if filename is None:
+ raise ValueError('filename arg must be a string')
+ if name is None:
+ name = filename
+ self.name = os.path.basename(name)
+ self.port = None
+ self.report_hook = lambda msg: None
+ self._filename = filename
+
+ def _get_args(self):
+ """Return list of command line arguments.
+
+ Override me.
+ """
+ return []
+
+ def start(self):
+ self.report_hook("starting (%s)" % (
+ [sys.executable, self._filename]+self._get_args()))
+ self._pid = os.spawnv(
+ os.P_NOWAIT,
+ sys.executable,
+ [sys.executable, self._filename]+self._get_args())
+ self.report_hook("waiting for startup")
+ self._wait_for_startup()
+ self.report_hook("running")
+
+ def _wait_for_startup(self):
+ import socket
+ def connect():
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.settimeout(1.0)
+ try:
+ sock.connect(('127.0.0.1', self.port))
+ finally:
+ sock.close()
+ backoff(connect, (socket.error,))
+
+ def stop(self):
+ """Kill process (forcefully if necessary)."""
+ if os.name == 'nt':
+ kill_windows(self._pid, self.report_hook)
+ else:
+ kill_posix(self._pid, self.report_hook)
+
+def backoff(func, errors,
+ initial_timeout=1., hard_timeout=60., factor=1.2):
+ starttime = time.time()
+ timeout = initial_timeout
+ while time.time() < starttime + hard_timeout - 0.01:
+ try:
+ func()
+ except errors, exc:
+ time.sleep(timeout)
+ timeout *= factor
+ hard_limit = hard_timeout - (time.time() - starttime)
+ timeout = min(timeout, hard_limit)
+ else:
+ break
+
+def kill_windows(handle, report_hook):
+ try:
+ import win32api
+ except ImportError:
+ import ctypes
+ ctypes.windll.kernel32.TerminateProcess(int(handle), -1)
+ else:
+ win32api.TerminateProcess(int(handle), -1)
+
+def kill_posix(pid, report_hook):
+ import signal
+ os.kill(pid, signal.SIGTERM)
+
+ timeout = 10.
+ starttime = time.time()
+ report_hook("waiting for exit")
+ def do_nothing(*args):
+ pass
+ old_handler = signal.signal(signal.SIGCHLD, do_nothing)
+ try:
+ while time.time() < starttime + timeout - 0.01:
+ pid, sts = os.waitpid(pid, os.WNOHANG)
+ if pid != 0:
+ # exited, or error
+ break
+ newtimeout = timeout - (time.time() - starttime) - 1.
+ time.sleep(newtimeout) # wait for signal
+ else:
+ report_hook("forcefully killing")
+ try:
+ os.kill(pid, signal.SIGKILL)
+ except OSError, exc:
+ if exc.errno != errno.ECHILD:
+ raise
+ finally:
+ signal.signal(signal.SIGCHLD, old_handler)
+
+class TwistedServerProcess(ServerProcess):
+
+ def __init__(self, name=None):
+ top_level_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
+ path = os.path.join(top_level_dir, "test-tools/twisted-localserver.py")
+ ServerProcess.__init__(self, path, name)
+
+ def _get_args(self):
+ return [str(self.port)]
+
+
+class CgitbTextResult(_TextTestResult):
+ def _exc_info_to_string(self, err, test):
+ """Converts a sys.exc_info()-style tuple of values into a string."""
+ exctype, value, tb = err
+ # Skip test runner traceback levels
+ while tb and self._is_relevant_tb_level(tb):
+ tb = tb.tb_next
+ if exctype is test.failureException:
+ # Skip assert*() traceback levels
+ length = self._count_relevant_tb_levels(tb)
+ return cgitb.text((exctype, value, tb))
+ return cgitb.text((exctype, value, tb))
+
+class CgitbTextTestRunner(TextTestRunner):
+ def _makeResult(self):
+ return CgitbTextResult(self.stream, self.descriptions, self.verbosity)
+
+def add_uri_attribute_to_test_cases(suite, uri):
+ for test in suite._tests:
+ if isinstance(test, TestCase):
+ test.uri = uri
+ else:
+ try:
+ add_uri_attribute_to_test_cases(test, uri)
+ except AttributeError:
+ pass
+
+
+class TestProgram:
+ """A command-line program that runs a set of tests; this is primarily
+ for making test modules conveniently executable.
+ """
+ USAGE = """\
+Usage: %(progName)s [options] [test] [...]
+
+Note not all the functional tests take note of the --uri argument yet --
+some currently always access the internet regardless of the --uri and
+--run-local-server options.
+
+Options:
+ -l, --run-local-server
+ Run a local Twisted HTTP server for the functional
+ tests. You need Twisted installed for this to work.
+ The server is run on the port given in the --uri
+ option. If --run-local-server is given but no --uri is
+ given, http://127.0.0.1:8000 is used as the base URI.
+ Also, if you're on Windows and don't have pywin32 or
+ ctypes installed, this option won't work, and you'll
+ have to start up test-tools/localserver.py manually.
+ --uri=URL Base URI for functional tests
+ (test.py does not access the network, unless you tell
+ it to run module functional_tests;
+ functional_tests.py does access the network)
+ e.g. --uri=http://127.0.0.1:8000/
+ -h, --help Show this message
+ -v, --verbose Verbose output
+ -q, --quiet Minimal output
+
+The following options are only available through test.py (you can still run the
+functional tests through test.py, just give 'functional_tests' as the module
+name to run):
+
+ -u Skip plain (non-doctest) unittests
+ -d Skip doctests
+ -c Run coverage (requires coverage.py, seems buggy)
+ -t Display tracebacks using cgitb's text mode
+
+"""
+ USAGE_EXAMPLES = """
+Examples:
+ %(progName)s
+ - run all tests
+ %(progName)s test_cookies
+ - run module 'test_cookies'
+ %(progName)s test_cookies.CookieTests
+ - run all 'test*' test methods in test_cookies.CookieTests
+ %(progName)s test_cookies.CookieTests.test_expires
+ - run test_cookies.CookieTests.test_expires
+
+ %(progName)s functional_tests
+ - run the functional tests
+ %(progName)s -l functional_tests
+ - start a local Twisted HTTP server and run the functional
+ tests against that, rather than against SourceForge
+ (quicker!)
+"""
+ def __init__(self, moduleNames, localServerProcess, defaultTest=None,
+ argv=None, testRunner=None, testLoader=defaultTestLoader,
+ defaultUri="http://wwwsearch.sf.net/",
+ usageExamples=USAGE_EXAMPLES,
+ ):
+ self.modules = []
+ for moduleName in moduleNames:
+ module = __import__(moduleName)
+ for part in moduleName.split('.')[1:]:
+ module = getattr(module, part)
+ self.modules.append(module)
+ self.uri = None
+ self._defaultUri = defaultUri
+ if argv is None:
+ argv = sys.argv
+ self.verbosity = 1
+ self.defaultTest = defaultTest
+ self.testRunner = testRunner
+ self.testLoader = testLoader
+ self.progName = os.path.basename(argv[0])
+ self.usageExamples = usageExamples
+ self.runLocalServer = False
+ self.parseArgs(argv)
+ if self.runLocalServer:
+ import urllib
+ from mechanize._rfc3986 import urlsplit
+ authority = urlsplit(self.uri)[1]
+ host, port = urllib.splitport(authority)
+ if port is None:
+ port = "80"
+ try:
+ port = int(port)
+ except:
+ self.usageExit("port in --uri value must be an integer "
+ "(try --uri=http://127.0.0.1:8000/)")
+ self._serverProcess = localServerProcess
+ def report(msg):
+ print "%s: %s" % (localServerProcess.name, msg)
+ localServerProcess.port = port
+ localServerProcess.report_hook = report
+
+ def usageExit(self, msg=None):
+ if msg: print msg
+ print (self.USAGE + self.usageExamples) % self.__dict__
+ sys.exit(2)
+
+ def parseArgs(self, argv):
+ import getopt
+ try:
+ options, args = getopt.getopt(
+ argv[1:],
+ 'hHvql',
+ ['help','verbose','quiet', 'uri=', 'run-local-server'],
+ )
+ uri = None
+ for opt, value in options:
+ if opt in ('-h','-H','--help'):
+ self.usageExit()
+ if opt in ('--uri',):
+ uri = value
+ if opt in ('-q','--quiet'):
+ self.verbosity = 0
+ if opt in ('-v','--verbose'):
+ self.verbosity = 2
+ if opt in ('-l', '--run-local-server'):
+ self.runLocalServer = True
+ if uri is None:
+ if self.runLocalServer:
+ uri = "http://127.0.0.1:8000"
+ else:
+ uri = self._defaultUri
+ self.uri = uri
+ if len(args) == 0 and self.defaultTest is None:
+ suite = TestSuite()
+ for module in self.modules:
+ test = self.testLoader.loadTestsFromModule(module)
+ suite.addTest(test)
+ self.test = suite
+ add_uri_attribute_to_test_cases(self.test, self.uri)
+ return
+ if len(args) > 0:
+ self.testNames = args
+ else:
+ self.testNames = (self.defaultTest,)
+ self.createTests()
+ add_uri_attribute_to_test_cases(self.test, self.uri)
+ except getopt.error, msg:
+ self.usageExit(msg)
+
+ def createTests(self):
+ self.test = self.testLoader.loadTestsFromNames(self.testNames)
+
+ def runTests(self):
+ if self.testRunner is None:
+ self.testRunner = TextTestRunner(verbosity=self.verbosity)
+
+ if self.runLocalServer:
+ self._serverProcess.start()
+ try:
+ result = self.testRunner.run(self.test)
+ finally:
+ if self.runLocalServer:
+ self._serverProcess.stop()
+ return result
Added: wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py
==============================================================================
--- (empty file)
+++ wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py Mon May 28 23:06:56 2007
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+"""
+%prog port
+
+e.g. %prog 8000
+
+Runs a local server to point the mechanize functional tests at. Example:
+
+python test-tools/twisted-localserver.py 8042
+python functional_tests.py --uri=http://localhost:8042/
+
+You need Twisted XXX version to run it:
+
+XXX installation instructions
+"""
+
+import os, sys, re, time
+from twisted.web2 import server, http, resource, channel, \
+ static, http_headers, responsecode
+
+from twisted.internet import reactor
+
+def html(title=None):
+ f = open("README.html", "r")
+ html = f.read()
+ if title is not None:
+ html = re.sub("(.*)", "%s" % title, html)
+ return html
+
+MECHANIZE_HTML = html()
+ROOT_HTML = html("Python bits")
+RELOAD_TEST_HTML = """\
+
+Title
+
+
+near the start
+
+Now some data to prevent HEAD parsing from reading the link near
+the end.
+
+
+%s
+
+near the end
+
+
+
+""" % (("0123456789ABCDEF"*4+"\n")*61)
+
+
+class Page(resource.Resource):
+
+ addSlash = True
+ content_type = http_headers.MimeType("text", "html")
+
+ def render(self, ctx):
+ return http.Response(
+ responsecode.OK,
+ {"content-type": self.content_type},
+ self.text)
+
+def make_page(root, name, text,
+ content_type="text/html"):
+ page = Page()
+ page.text = text
+ base_type, specific_type = content_type.split("/")
+ page.content_type = http_headers.MimeType(base_type, specific_type)
+ setattr(root, "child_"+name, page)
+ return page
+
+def main():
+ root = Page()
+ root.text = ROOT_HTML
+ make_page(root, "mechanize", MECHANIZE_HTML)
+ bits = make_page(root, "robots.txt",
+ "User-Agent: *\nDisallow: /norobots",
+ "text/plain")
+ bits = make_page(root, "robots", "Hello, robots.", "text/plain")
+ bits = make_page(root, "norobots", "Hello, non-robots.", "text/plain")
+ bits = make_page(root, "bits", "GeneralFAQ.html")
+ make_page(bits, "cctest2.txt",
+ "Hello ClientCookie functional test suite.",
+ "text/plain")
+ make_page(bits, "mechanize_reload_test.html", RELOAD_TEST_HTML)
+
+ site = server.Site(root)
+ reactor.listenTCP(int(sys.argv[1]), channel.HTTPFactory(site))
+ reactor.run()
+
+main()
Modified: wwwsearch/mechanize/trunk/test.py
==============================================================================
--- wwwsearch/mechanize/trunk/test.py (original)
+++ wwwsearch/mechanize/trunk/test.py Mon May 28 23:06:56 2007
@@ -8,9 +8,6 @@
"""
-import cgitb
-#cgitb.enable(format="text")
-
# Modules containing tests to run -- a test is anything named *Tests, which
# should be classes deriving from unittest.TestCase.
MODULE_NAMES = ["test_date", "test_browser", "test_response", "test_cookies",
@@ -18,9 +15,7 @@
"test_useragent", "test_html", "test_opener",
]
-import sys, os, traceback, logging, glob
-from unittest import defaultTestLoader, TextTestRunner, TestSuite, TestCase, \
- _TextTestResult
+import sys, os, logging, glob
#level = logging.DEBUG
#level = logging.INFO
@@ -30,109 +25,12 @@
#logging.getLogger("mechanize").addHandler(logging.StreamHandler(sys.stdout))
-class CgitbTextResult(_TextTestResult):
- def _exc_info_to_string(self, err, test):
- """Converts a sys.exc_info()-style tuple of values into a string."""
- exctype, value, tb = err
- # Skip test runner traceback levels
- while tb and self._is_relevant_tb_level(tb):
- tb = tb.tb_next
- if exctype is test.failureException:
- # Skip assert*() traceback levels
- length = self._count_relevant_tb_levels(tb)
- return cgitb.text((exctype, value, tb))
- return cgitb.text((exctype, value, tb))
-
-class CgitbTextTestRunner(TextTestRunner):
- def _makeResult(self):
- return CgitbTextResult(self.stream, self.descriptions, self.verbosity)
-
-
-class TestProgram:
- """A command-line program that runs a set of tests; this is primarily
- for making test modules conveniently executable.
- """
- USAGE = """\
-Usage: %(progName)s [options] [test] [...]
-
-Options:
- -h, --help Show this message
- -v, --verbose Verbose output
- -q, --quiet Minimal output
-
-Examples:
- %(progName)s - run default set of tests
- %(progName)s MyTestSuite - run suite 'MyTestSuite'
- %(progName)s MyTestCase.testSomething - run MyTestCase.testSomething
- %(progName)s MyTestCase - run all 'test*' test methods
- in MyTestCase
-"""
- def __init__(self, moduleNames, defaultTest=None,
- argv=None, testRunner=None, testLoader=defaultTestLoader):
- self.modules = []
- for moduleName in moduleNames:
- module = __import__(moduleName)
- for part in moduleName.split('.')[1:]:
- module = getattr(module, part)
- self.modules.append(module)
- if argv is None:
- argv = sys.argv
- self.verbosity = 1
- self.defaultTest = defaultTest
- self.testRunner = testRunner
- self.testLoader = testLoader
- self.progName = os.path.basename(argv[0])
- self.parseArgs(argv)
-
- def usageExit(self, msg=None):
- if msg: print msg
- print self.USAGE % self.__dict__
- sys.exit(2)
-
- def parseArgs(self, argv):
- import getopt
- try:
- options, args = getopt.getopt(argv[1:], 'hHvq',
- ['help','verbose','quiet'])
- for opt, value in options:
- if opt in ('-h','-H','--help'):
- self.usageExit()
- if opt in ('-q','--quiet'):
- self.verbosity = 0
- if opt in ('-v','--verbose'):
- self.verbosity = 2
- if len(args) == 0 and self.defaultTest is None:
- suite = TestSuite()
- for module in self.modules:
- test = self.testLoader.loadTestsFromModule(module)
- suite.addTest(test)
- self.test = suite
- return
- if len(args) > 0:
- self.testNames = args
- else:
- self.testNames = (self.defaultTest,)
- self.createTests()
- except getopt.error, msg:
- self.usageExit(msg)
-
- def createTests(self):
- self.test = self.testLoader.loadTestsFromNames(self.testNames)
-
- def runTests(self):
- if self.testRunner is None:
- self.testRunner = TextTestRunner(verbosity=self.verbosity)
- result = self.testRunner.run(self.test)
- return result
-
-
if __name__ == "__main__":
-## sys.path.insert(0, '/home/john/comp/dev/rl/jjlee/lib/python')
-## import jjl
-## import __builtin__
-## __builtin__.jjl = jjl
+ # XXX
+ # temporary stop-gap to run doctests &c.
+ # should switch to nose or something
- # XXX temporary stop-gap to run doctests
+ top_level_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
# XXXX coverage output seems incorrect ATM
run_coverage = "-c" in sys.argv
@@ -159,6 +57,7 @@
# that renamed module.
sys.path.insert(0, "test-tools")
import doctest
+ import testprogram
import coverage
if run_coverage:
@@ -176,7 +75,8 @@
if run_doctests:
# run .doctest files needing special support
common_globs = {"mechanize": mechanize}
- pm_doctest_filename = os.path.join("test", "test_password_manager.doctest")
+ pm_doctest_filename = os.path.join(
+ "test", "test_password_manager.doctest")
for globs in [
{"mgr_class": mechanize.HTTPPasswordMgr},
{"mgr_class": mechanize.HTTPProxyPasswordMgr},
@@ -217,8 +117,12 @@
sys.path.insert(0, test_path)
test_runner = None
if use_cgitb:
- test_runner = CgitbTextTestRunner()
- prog = TestProgram(MODULE_NAMES, testRunner=test_runner)
+ test_runner = testprogram.CgitbTextTestRunner()
+ prog = testprogram.TestProgram(
+ MODULE_NAMES,
+ testRunner=test_runner,
+ localServerProcess=testprogram.TwistedServerProcess(),
+ )
result = prog.runTests()
if run_coverage:
From jjlee at codespeak.net Thu May 31 00:37:14 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Thu, 31 May 2007 00:37:14 +0200 (CEST)
Subject: [wwwsearch-commits] r43927 - in wwwsearch/mechanize/trunk: .
test-tools
Message-ID: <20070530223714.0F1738080@code0.codespeak.net>
Author: jjlee
Date: Thu May 31 00:37:13 2007
New Revision: 43927
Modified:
wwwsearch/mechanize/trunk/functional_tests.py
wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py
Log:
A bit more work on local twisted server for functional tests: Add 302 redirection page to get rid of one of the hardcoded wwwsearch URLs. Fix some issues in local twisted server.
Modified: wwwsearch/mechanize/trunk/functional_tests.py
==============================================================================
--- wwwsearch/mechanize/trunk/functional_tests.py (original)
+++ wwwsearch/mechanize/trunk/functional_tests.py Thu May 31 00:37:13 2007
@@ -57,13 +57,13 @@
self.assertEqual(self.browser.title(), 'mechanize')
def test_302_and_404(self):
- # the combination of 302 (caused by use of "sf.net") and 404 has caused
- # problems in the past due to accidental double-wrapping of the error
- # response
+ # the combination of 302 and 404 (/redirected is configured to redirect
+ # to a non-existent URL /nonexistent) has caused problems in the past
+ # due to accidental double-wrapping of the error response
import urllib2
self.assertRaises(
urllib2.HTTPError,
- self.browser.open, "http://wwwsearch.sf.net/doesnotexist",
+ self.browser.open, urljoin(self.uri, "/redirected"),
)
def test_reread(self):
Modified: wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py
==============================================================================
--- wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py (original)
+++ wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py Thu May 31 00:37:13 2007
@@ -60,29 +60,45 @@
{"content-type": self.content_type},
self.text)
-def make_page(root, name, text,
- content_type="text/html"):
+def _make_page(root, name, text,
+ content_type="text/html",
+ leaf=False):
page = Page()
page.text = text
base_type, specific_type = content_type.split("/")
page.content_type = http_headers.MimeType(base_type, specific_type)
+ page.addSlash = not leaf
setattr(root, "child_"+name, page)
return page
+def make_page(root, name, text,
+ content_type="text/html"):
+ return _make_page(root, name, text, content_type, leaf=False)
+
+def make_leaf_page(root, name, text,
+ content_type="text/html"):
+ return _make_page(root, name, text, content_type, leaf=True)
+
+def make_redirect(root, name, location_relative_ref):
+ redirect = resource.RedirectResource(path=location_relative_ref)
+ setattr(root, "child_"+name, redirect)
+ return redirect
+
def main():
root = Page()
root.text = ROOT_HTML
make_page(root, "mechanize", MECHANIZE_HTML)
- bits = make_page(root, "robots.txt",
- "User-Agent: *\nDisallow: /norobots",
- "text/plain")
- bits = make_page(root, "robots", "Hello, robots.", "text/plain")
- bits = make_page(root, "norobots", "Hello, non-robots.", "text/plain")
+ make_leaf_page(root, "robots.txt",
+ "User-Agent: *\nDisallow: /norobots",
+ "text/plain")
+ make_leaf_page(root, "robots", "Hello, robots.", "text/plain")
+ make_leaf_page(root, "norobots", "Hello, non-robots.", "text/plain")
bits = make_page(root, "bits", "GeneralFAQ.html")
- make_page(bits, "cctest2.txt",
- "Hello ClientCookie functional test suite.",
- "text/plain")
- make_page(bits, "mechanize_reload_test.html", RELOAD_TEST_HTML)
+ make_leaf_page(bits, "cctest2.txt",
+ "Hello ClientCookie functional test suite.",
+ "text/plain")
+ make_leaf_page(bits, "mechanize_reload_test.html", RELOAD_TEST_HTML)
+ make_redirect(root, "redirected", "/doesnotexist")
site = server.Site(root)
reactor.listenTCP(int(sys.argv[1]), channel.HTTPFactory(site))
From jjlee at codespeak.net Thu May 31 01:13:21 2007
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Thu, 31 May 2007 01:13:21 +0200 (CEST)
Subject: [wwwsearch-commits] r43928 - in wwwsearch/mechanize/trunk: .
test-tools
Message-ID: <20070530231321.640438093@code0.codespeak.net>
Author: jjlee
Date: Thu May 31 01:13:20 2007
New Revision: 43928
Modified:
wwwsearch/mechanize/trunk/functional_tests.py
wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py
Log:
Get rid of the remaining hard-coded URLs in functional tests :-)
Modified: wwwsearch/mechanize/trunk/functional_tests.py
==============================================================================
--- wwwsearch/mechanize/trunk/functional_tests.py (original)
+++ wwwsearch/mechanize/trunk/functional_tests.py Thu May 31 01:13:20 2007
@@ -256,8 +256,8 @@
def test_referer(self):
br = mechanize.Browser()
- referer = "http://wwwsearch.sourceforge.net/bits/referertest.html"
- info = "http://wwwsearch.sourceforge.net/cgi-bin/cookietest.cgi"
+ referer = urljoin(self.uri, "bits/referertest.html")
+ info = urljoin(self.uri, "/cgi-bin/cookietest.cgi")
r = br.open(info)
self.assert_(referer not in r.get_data())
@@ -286,8 +286,7 @@
try:
install_opener(o)
try:
- r = urlopen(
- "http://wwwsearch.sourceforge.net/cgi-bin/cookietest.cgi")
+ r = urlopen(urljoin(self.uri, "/cgi-bin/cookietest.cgi"))
except urllib2.URLError, e:
#print e.read()
raise
@@ -317,7 +316,7 @@
opener.open, urljoin(self.uri, "norobots"))
def test_urlretrieve(self):
- url = "http://www.python.org/"
+ url = urljoin(self.uri, "/mechanize/")
test_filename = "python.html"
def check_retrieve(opener, filename, headers):
self.assertEqual(headers.get('Content-Type'), 'text/html')
Modified: wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py
==============================================================================
--- wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py (original)
+++ wwwsearch/mechanize/trunk/test-tools/twisted-localserver.py Thu May 31 01:13:20 2007
@@ -16,7 +16,7 @@
import os, sys, re, time
from twisted.web2 import server, http, resource, channel, \
- static, http_headers, responsecode
+ static, http_headers, responsecode, twcgi
from twisted.internet import reactor
@@ -47,6 +47,16 @@