[wwwsearch-commits] r21729 - in wwwsearch/ClientCookie/trunk: .
ClientCookie test
jjlee at codespeak.net
jjlee at codespeak.net
Thu Jan 5 23:46:18 CET 2006
Author: jjlee
Date: Thu Jan 5 23:46:16 2006
New Revision: 21729
Modified:
wwwsearch/ClientCookie/trunk/ClientCookie/_Util.py
wwwsearch/ClientCookie/trunk/doc.html.in
wwwsearch/ClientCookie/trunk/test/test_misc.py
wwwsearch/ClientCookie/trunk/test/test_urllib2.py
Log:
Add .set_data() method to response_seek_wrapper; Support .code and .msg on response_seek_wrapper instances
Modified: wwwsearch/ClientCookie/trunk/ClientCookie/_Util.py
==============================================================================
--- wwwsearch/ClientCookie/trunk/ClientCookie/_Util.py (original)
+++ wwwsearch/ClientCookie/trunk/ClientCookie/_Util.py Thu Jan 5 23:46:16 2006
@@ -564,8 +564,7 @@
class response_seek_wrapper(seek_wrapper):
"""Avoids unnecessarily clobbering urllib.addinfourl methods on .close().
- After .close():
- , the following methods are supported:
+ After .close(), the following methods are supported:
.read()
.readline()
@@ -577,12 +576,19 @@
.__iter__()
.next()
- Also supports pickling.
+ and the following attributes are supported.
+
+ .code
+ .msg
+
+ Also supports pickling (but the stdlib does something to prevent it, IIRC).
"""
def __init__(self, wrapped):
seek_wrapper.__init__(self, wrapped)
+ self.msg = wrapped.msg
+ self.code = wrapped.code
self._headers = self.wrapped.info()
def close(self):
@@ -598,6 +604,14 @@
cpy._headers = self.wrapped.info().copy()
return cpy
+ def set_data(self, data):
+ self.seek(0)
+ self.read()
+ self.close()
+ cache = self._seek_wrapper__cache = StringIO()
+ cache.write(data)
+ self.seek(0)
+
def __getstate__(self):
# There are three obvious options here:
# 1. truncate
Modified: wwwsearch/ClientCookie/trunk/doc.html.in
==============================================================================
--- wwwsearch/ClientCookie/trunk/doc.html.in (original)
+++ wwwsearch/ClientCookie/trunk/doc.html.in Thu Jan 5 23:46:16 2006
@@ -349,7 +349,8 @@
<dd><p>This makes ClientCookie's response objects <code>seek()</code>able.
Seeking is done lazily (ie. the response object only reads from the socket as
necessary, rather than slurping in all the data before the response is returned
-to you). XXX only works for HTTP ATM, I think
+to you). XXX only works for HTTP ATM, I think, and also doesn't work for
+HTTPError exceptions...
<dt><code>HTTPRefererProcessor</code>
@@ -652,6 +653,31 @@
form, though the supported date/time formats may change.
+<a name="badhtml"></a>
+<h2>Dealing with bad HTML</h2>
+
+<p>XXX Intro
+
+<p>XXX Test me
+
+@{colorize("""\
+import copy
+import ClientCookie
+class CommentCleanProcessor(ClientCookie.BaseProcessor):
+ def http_response(self, request, response):
+ if not hasattr(response, "seek"):
+ response = ClientCookie.response_seek_wrapper(response)
+ response.seek(0)
+ new_response = copy.copy(response)
+ new_response.set_data(
+ re.sub("<!-([^-]*)->", "<!--\\1-->", response.read()))
+ return new_response
+ https_response = http_response
+""")}
+
+<p>XXX TidyProcessor: mxTidy? tidylib? tidy?
+
+
<a name="standards"></a>
<h2>Note about cookie standards</h2>
Modified: wwwsearch/ClientCookie/trunk/test/test_misc.py
==============================================================================
--- wwwsearch/ClientCookie/trunk/test/test_misc.py (original)
+++ wwwsearch/ClientCookie/trunk/test/test_misc.py Thu Jan 5 23:46:16 2006
@@ -1,5 +1,6 @@
"""Miscellaneous pyunit tests."""
+import copy
import cStringIO, string
from unittest import TestCase
@@ -27,11 +28,17 @@
class TestUnSeekableResponse(TestUnSeekable):
def __init__(self, text, headers):
TestUnSeekable.__init__(self, text)
+ self.code = 200
+ self.msg = "OK"
self.headers = headers
+ self.url = "http://example.com/"
def info(self):
return self.headers
+ def close(self):
+ pass
+
class SeekableTests(TestCase):
@@ -42,6 +49,8 @@
dog.
"""
+ text_lines = map(lambda l: l+"\n", string.split(text, "\n")[:-1])
+
def testSeekable(self):
try:
from exceptions import StopIteration
@@ -49,20 +58,45 @@
from ClientCookie._ClientCookie import StopIteration
from ClientCookie._Util import seek_wrapper
text = self.text
- text_lines = map(lambda l: l+"\n", string.split(text, "\n")[:-1])
+ text_lines = self.text_lines
+
+ for ii in range(1, 6):
+ fh = TestUnSeekable(text)
+ sfh = seek_wrapper(fh)
+ test = getattr(self, "_test%d" % ii)
+ test(sfh)
+
+ # copies have independent seek positions
fh = TestUnSeekable(text)
sfh = seek_wrapper(fh)
+ self._testCopy(sfh)
+
+ def _testCopy(self, sfh):
+ sfh2 = copy.copy(sfh)
+ sfh.read(10)
+ text = self.text
+ self.assertEqual(sfh2.read(10), text[:10])
+ sfh2.seek(5)
+ self.assertEqual(sfh.read(10), text[10:20])
+ self.assertEqual(sfh2.read(10), text[5:15])
+ sfh.seek(0)
+ sfh2.seek(0)
+ return sfh2
+
+ def _test1(self, sfh):
+ text = self.text
+ text_lines = self.text_lines
assert sfh.read(10) == text[:10] # calls fh.read
- assert fh.log[-1] == ("read", 10)
+ assert sfh.log[-1] == ("read", 10) # .log delegated to fh
sfh.seek(0) # doesn't call fh.seek
assert sfh.read(10) == text[:10] # doesn't call fh.read
- assert len(fh.log) == 1
+ assert len(sfh.log) == 1
sfh.seek(0)
assert sfh.read(5) == text[:5] # read only part of cached data
- assert len(fh.log) == 1
+ assert len(sfh.log) == 1
sfh.seek(0)
assert sfh.read(25) == text[:25] # calls fh.read
- assert fh.log[1] == ("read", 15)
+ assert sfh.log[1] == ("read", 15)
lines = []
sfh.seek(-1, 1)
while 1:
@@ -70,7 +104,7 @@
if l == "": break
lines.append(l)
assert lines == ["s over the lazy\n"]+text_lines[2:]
- assert fh.log[2:] == [("readline", -1)]*5
+ assert sfh.log[2:] == [("readline", -1)]*5
sfh.seek(0)
lines = []
while 1:
@@ -79,8 +113,8 @@
lines.append(l)
assert lines == text_lines
- fh = TestUnSeekable(text)
- sfh = seek_wrapper(fh)
+ def _test2(self, sfh):
+ text = self.text
sfh.read(5)
sfh.seek(0)
assert sfh.read() == text
@@ -94,17 +128,19 @@
assert sfh.readline(5) == "The q"
assert sfh.readline() == "uick brown fox\n"
- fh = TestUnSeekable(text)
- sfh = seek_wrapper(fh)
+ def _test3(self, sfh):
+ text = self.text
+ text_lines = self.text_lines
sfh.read(25)
sfh.seek(-1, 1)
self.assertEqual(sfh.readlines(), ["s over the lazy\n"]+text_lines[2:])
- nr_logs = len(fh.log)
+ nr_logs = len(sfh.log)
sfh.seek(0)
assert sfh.readlines() == text_lines
- fh = TestUnSeekable(text)
- sfh = seek_wrapper(fh)
+ def _test4(self, sfh):
+ text = self.text
+ text_lines = self.text_lines
count = 0
limit = 10
while count < limit:
@@ -117,8 +153,8 @@
else:
assert False, "StopIteration not raised"
- fh = TestUnSeekable(text)
- sfh = seek_wrapper(fh)
+ def _test5(self, sfh):
+ text = self.text
sfh.read(10)
sfh.seek(5)
self.assert_(sfh.invariant())
@@ -127,27 +163,9 @@
sfh.seek(0)
self.assertEqual(sfh.read(), text)
- # copies have independent seek positions
- fh = TestUnSeekable(text)
- sfh = seek_wrapper(fh)
- self._testCopy(sfh)
-
- def _testCopy(self, sfh):
- import copy
- sfh2 = copy.copy(sfh)
- sfh.read(10)
- text = self.text
- self.assertEqual(sfh2.read(10), text[:10])
- sfh2.seek(5)
- self.assertEqual(sfh.read(10), text[10:20])
- self.assertEqual(sfh2.read(10), text[5:15])
- sfh.seek(0)
- sfh2.seek(0)
- return sfh2
-
def testResponseSeekWrapper(self):
- hdrs = {"Content-type": "text/html"}
from ClientCookie import response_seek_wrapper
+ hdrs = {"Content-type": "text/html"}
r = TestUnSeekableResponse(self.text, hdrs)
rsw = response_seek_wrapper(r)
rsw2 = self._testCopy(rsw)
@@ -155,6 +173,32 @@
self.assertEqual(rsw.info(), rsw2.info())
self.assert_(rsw.info() is not rsw2.info())
+ def testSetResponseData(self):
+ from ClientCookie import response_seek_wrapper
+ r = TestUnSeekableResponse(self.text, {'blah': 'yawn'})
+ rsw = response_seek_wrapper(r)
+ rsw.set_data("""\
+A Seeming somwhat more than View;
+ That doth instruct the Mind
+ In Things that ly behind,
+""")
+ self.assertEqual(rsw.read(9), "A Seeming")
+ self.assertEqual(rsw.read(13), " somwhat more")
+ rsw.seek(0)
+ self.assertEqual(rsw.read(9), "A Seeming")
+ self.assertEqual(rsw.readline(), " somwhat more than View;\n")
+ rsw.seek(0)
+ self.assertEqual(rsw.readline(), "A Seeming somwhat more than View;\n")
+ rsw.seek(-1, 1)
+ self.assertEqual(rsw.read(7), "\n That")
+
+ r = TestUnSeekableResponse(self.text, {'blah': 'yawn'})
+ rsw = response_seek_wrapper(r)
+ rsw.set_data(self.text)
+ self._test2(rsw)
+ rsw.seek(0)
+ self._test4(rsw)
+
if __name__ == "__main__":
import unittest
Modified: wwwsearch/ClientCookie/trunk/test/test_urllib2.py
==============================================================================
--- wwwsearch/ClientCookie/trunk/test/test_urllib2.py (original)
+++ wwwsearch/ClientCookie/trunk/test/test_urllib2.py Thu Jan 5 23:46:16 2006
@@ -665,6 +665,8 @@
req = urllib2.Request("http://example.com/")
class MockUnseekableResponse:
+ code = 200
+ msg = "OK"
def info(self): pass
r = MockUnseekableResponse()
newr = h.http_response(req, r)
More information about the wwwsearch-commits
mailing list