From jjlee at codespeak.net Wed Jul 5 22:18:03 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Wed, 5 Jul 2006 22:18:03 +0200 (CEST)
Subject: [wwwsearch-commits] r29654 - wwwsearch/mechanize/trunk/mechanize
Message-ID: <20060705201803.27FD1100A7@code0.codespeak.net>
Author: jjlee
Date: Wed Jul 5 22:18:02 2006
New Revision: 29654
Added:
wwwsearch/mechanize/trunk/mechanize/_seek.py
Log:
Add missing module _seek.py
Added: wwwsearch/mechanize/trunk/mechanize/_seek.py
==============================================================================
--- (empty file)
+++ wwwsearch/mechanize/trunk/mechanize/_seek.py Wed Jul 5 22:18:02 2006
@@ -0,0 +1,11 @@
+from urllib2 import BaseHandler
+from _response import response_seek_wrapper
+
+
+class SeekableProcessor(BaseHandler):
+ """Make responses seekable."""
+
+ def any_response(self, request, response):
+ if not hasattr(response, "seek"):
+ return response_seek_wrapper(response)
+ return response
From jjlee at codespeak.net Wed Jul 5 22:18:32 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Wed, 5 Jul 2006 22:18:32 +0200 (CEST)
Subject: [wwwsearch-commits] r29655 - wwwsearch/mechanize/trunk
Message-ID: <20060705201832.420C8100AC@code0.codespeak.net>
Author: jjlee
Date: Wed Jul 5 22:18:31 2006
New Revision: 29655
Modified:
wwwsearch/mechanize/trunk/0.1-changes.txt
Log:
Explain how to get a sequence from iterable return values
Modified: wwwsearch/mechanize/trunk/0.1-changes.txt
==============================================================================
--- wwwsearch/mechanize/trunk/0.1-changes.txt (original)
+++ wwwsearch/mechanize/trunk/0.1-changes.txt Wed Jul 5 22:18:31 2006
@@ -31,7 +31,7 @@
- .forms() and .links() now both return iterators (in fact, generators),
not sequences (not really an interface change: these were always
documented to return iterables, but it will no doubt break some client
- code).
+ code). Use e.g. list(browser.forms()) if you want a list.
- .links no longer raises LinkNotFoundError (was accidental -- only
.click_link() / .find_link() should raise this).
From jjlee at codespeak.net Wed Jul 5 22:54:42 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Wed, 5 Jul 2006 22:54:42 +0200 (CEST)
Subject: [wwwsearch-commits] r29657 - in wwwsearch/mechanize/trunk:
mechanize test
Message-ID: <20060705205442.6BE44100AF@code0.codespeak.net>
Author: jjlee
Date: Wed Jul 5 22:54:41 2006
New Revision: 29657
Modified:
wwwsearch/mechanize/trunk/mechanize/_http.py
wwwsearch/mechanize/trunk/test/test_urllib2.py
Log:
Fix UnboundLocalError for Refresh with URL but no '=' (titus)
Modified: wwwsearch/mechanize/trunk/mechanize/_http.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_http.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_http.py Wed Jul 5 22:54:41 2006
@@ -433,9 +433,10 @@
if ii != -1:
pause, newurl_spec = float(refresh[:ii]), refresh[ii+1:]
jj = newurl_spec.find("=")
+ key = None
if jj != -1:
key, newurl = newurl_spec[:jj], newurl_spec[jj+1:]
- if key.strip().lower() != "url":
+ if key is None or key.strip().lower() != "url":
debug("bad Refresh header: %r" % refresh)
return response
else:
Modified: wwwsearch/mechanize/trunk/test/test_urllib2.py
==============================================================================
--- wwwsearch/mechanize/trunk/test/test_urllib2.py (original)
+++ wwwsearch/mechanize/trunk/test/test_urllib2.py Wed Jul 5 22:54:41 2006
@@ -778,14 +778,20 @@
# XXX test processor constructor optional args
h = HTTPRefreshProcessor(max_time=None, honor_time=False)
- for val in ['0; url="http://example.com/foo/"', "2"]:
+ for val, valid in [
+ ('0; url="http://example.com/foo/"', True),
+ ("2", True),
+ # in the past, this failed with UnboundLocalError
+ ('0; "http://example.com/foo/"', False),
+ ]:
o = h.parent = MockOpener()
req = Request("http://example.com/")
headers = MockHeaders({"refresh": val})
r = MockResponse(200, "OK", headers, "")
newr = h.http_response(req, r)
- self.assertEqual(o.proto, "http")
- self.assertEqual(o.args, (req, r, "refresh", "OK", headers))
+ if valid:
+ self.assertEqual(o.proto, "http")
+ self.assertEqual(o.args, (req, r, "refresh", "OK", headers))
def test_redirect(self):
from_url = "http://example.com/a.html"
From jjlee at codespeak.net Sun Jul 9 14:35:54 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Sun, 9 Jul 2006 14:35:54 +0200 (CEST)
Subject: [wwwsearch-commits] r29894 - wwwsearch/mechanize/trunk
Message-ID: <20060709123554.8C1FA1007B@code0.codespeak.net>
Author: jjlee
Date: Sun Jul 9 14:35:52 2006
New Revision: 29894
Modified:
wwwsearch/mechanize/trunk/README.html.in
Log:
HTML fixes
Modified: wwwsearch/mechanize/trunk/README.html.in
==============================================================================
--- wwwsearch/mechanize/trunk/README.html.in (original)
+++ wwwsearch/mechanize/trunk/README.html.in Sun Jul 9 14:35:52 2006
@@ -171,7 +171,7 @@
""")}
-so anything you would normally import from urllib2 can
+so anything you would normally import from urllib2 can
(and should, by preference, to insulate you from future changes) be
imported from mechanize instead. In many cases if you import an
object from mechanize it will be the very same object you would get if
@@ -333,7 +333,8 @@
Make EncodingFinder public, I guess (but probably
improve it first). (For example: support Mark Pilgrim's universal
encoding detector?)
- Add two-way links between BeautifulSoup & ClientForm object models.
+ Add two-way links between BeautifulSoup & ClientForm object
+ models.
In 0.2: switch to Python unicode strings everywhere appropriate
(HTTP level should still use byte strings, of course).
clean_url(): test browser behaviour. I think
From jjlee at codespeak.net Sun Jul 9 14:37:52 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Sun, 9 Jul 2006 14:37:52 +0200 (CEST)
Subject: [wwwsearch-commits] r29895 - wwwsearch/mechanize/trunk/mechanize
Message-ID: <20060709123752.1AE361007B@code0.codespeak.net>
Author: jjlee
Date: Sun Jul 9 14:37:50 2006
New Revision: 29895
Modified:
wwwsearch/mechanize/trunk/mechanize/_html.py
Log:
Simplify CachingGeneratorFunction implementation a bit
Modified: wwwsearch/mechanize/trunk/mechanize/_html.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_html.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_html.py Sun Jul 9 14:37:50 2006
@@ -40,19 +40,15 @@
def __init__(self, iterable):
self._cache = []
-
# wrap iterable to make it non-restartable (otherwise, repeated
# __call__ would give incorrect results)
- def make_gen():
- for item in iterable:
- yield item
- self._generator = make_gen()
+ self._iterator = iter(iterable)
def __call__(self):
cache = self._cache
for item in cache:
yield item
- for item in self._generator:
+ for item in self._iterator:
cache.append(item)
yield item
From jjlee at codespeak.net Sun Jul 9 15:35:44 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Sun, 9 Jul 2006 15:35:44 +0200 (CEST)
Subject: [wwwsearch-commits] r29900 - wwwsearch/mechanize/trunk/mechanize
Message-ID: <20060709133544.D93BD1007E@code0.codespeak.net>
Author: jjlee
Date: Sun Jul 9 15:35:44 2006
New Revision: 29900
Modified:
wwwsearch/mechanize/trunk/mechanize/_html.py
Log:
Fix RobustTitleFactory (titus)
Modified: wwwsearch/mechanize/trunk/mechanize/_html.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_html.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_html.py Sun Jul 9 15:35:44 2006
@@ -413,7 +413,7 @@
self._bs = soup
self._encoding = encoding
- def title(soup):
+ def title(self):
import BeautifulSoup
title = self._bs.first("title")
if title == BeautifulSoup.Null:
From jjlee at codespeak.net Wed Jul 12 00:04:18 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Wed, 12 Jul 2006 00:04:18 +0200 (CEST)
Subject: [wwwsearch-commits] r29964 - wwwsearch/mechanize/trunk
Message-ID: <20060711220418.499B21007E@code0.codespeak.net>
Author: jjlee
Date: Wed Jul 12 00:04:17 2006
New Revision: 29964
Modified:
wwwsearch/mechanize/trunk/README.html.in
wwwsearch/mechanize/trunk/setup.py
Log:
Declare a setuptools dependency on BeautifulSoup version 2
Modified: wwwsearch/mechanize/trunk/README.html.in
==============================================================================
--- wwwsearch/mechanize/trunk/README.html.in (original)
+++ wwwsearch/mechanize/trunk/README.html.in Wed Jul 12 00:04:17 2006
@@ -296,8 +296,6 @@
integrate docstring and non-docstring docs.
Note BeautifulSoup 3.0 doesn't work yet.
- Add an "extra" to setup.py for BeautifulSoup (i.e. a declared
- feature depending on BeautifulSoup)?
Document use of BeautifulSoup (RobustFactory).
Document means of processing response on ad-hoc basis with
.set_response() - e.g. to fix bad encoding in Content-type header or
@@ -571,7 +569,14 @@
Which version of Python do I need?
2.3 or above.
What else do I need?
- mechanize depends on ClientForm.
+
mechanize depends on ClientForm. The
+ setup.py script also declares a dependency on BeautifulSoup,
+ but there is no true dependency: the declaration is there only to avoid
+ confusing people who don't realise that mechanize is not compatible with
+ BeautifulSoup version 3 -- only BeautifulSoup version 2 is currently
+ supported. A future version of mechanize will support BeautifulSoup
+ version 3.
The versions of those required modules are listed in the
setup.py for mechanize (included with the download). The
dependencies are automatically fetched by =0.2.2, ==dev"]
+INSTALL_REQUIRES = [
+ "ClientForm>=0.2.2, ==dev",
+ # there's no dependency on BeautifulSoup, but people get confused when
+ # BeautifulSoup 3 doesn't work
+ "BeautifulSoup>=2.1.1, <3.0",
+ ]
NAME = "mechanize"
PACKAGE = True
LICENSE = "BSD" # or ZPL 2.1
From jjlee at codespeak.net Sat Jul 29 17:02:21 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Sat, 29 Jul 2006 17:02:21 +0200 (CEST)
Subject: [wwwsearch-commits] r30731 - in wwwsearch/mechanize/trunk: .
mechanize
Message-ID: <20060729150221.9D18D1007B@code0.codespeak.net>
Author: jjlee
Date: Sat Jul 29 17:02:19 2006
New Revision: 30731
Modified:
wwwsearch/mechanize/trunk/mechanize/_http.py
wwwsearch/mechanize/trunk/test.py
Log:
Fix bug with quoted META Refresh URL (nilton.volpato at gmail.com); Clean up test.py a bit
Modified: wwwsearch/mechanize/trunk/mechanize/_http.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_http.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_http.py Sat Jul 29 17:02:19 2006
@@ -400,6 +400,42 @@
https_request = http_request
https_response = http_response
+
+def clean_refresh_url(url):
+ # e.g. Firefox 1.5 does (something like) this
+ if ((url.startswith('"') and url.endswith('"')) or
+ (url.startswith("'") and url.endswith("'"))):
+ return url[1:-1]
+ return url
+
+def parse_refresh_header(refresh):
+ """
+ >>> parse_refresh_header("1; url=http://example.com/")
+ (1.0, 'http://example.com/')
+ >>> parse_refresh_header("1; url='http://example.com/'")
+ (1.0, 'http://example.com/')
+ >>> parse_refresh_header("1")
+ (1.0, None)
+ >>> parse_refresh_header("blah")
+ Traceback (most recent call last):
+ ValueError: invalid literal for float(): blah
+
+ """
+
+ ii = refresh.find(";")
+ if ii != -1:
+ pause, newurl_spec = float(refresh[:ii]), refresh[ii+1:]
+ jj = newurl_spec.find("=")
+ key = None
+ if jj != -1:
+ key, newurl = newurl_spec[:jj], newurl_spec[jj+1:]
+ newurl = clean_refresh_url(newurl)
+ if key is None or key.strip().lower() != "url":
+ raise ValueError()
+ else:
+ pause, newurl = float(refresh), None
+ return pause, newurl
+
class HTTPRefreshProcessor(BaseHandler):
"""Perform HTTP Refresh redirections.
@@ -429,18 +465,13 @@
if code == 200 and hdrs.has_key("refresh"):
refresh = hdrs.getheaders("refresh")[0]
- ii = refresh.find(";")
- if ii != -1:
- pause, newurl_spec = float(refresh[:ii]), refresh[ii+1:]
- jj = newurl_spec.find("=")
- key = None
- if jj != -1:
- key, newurl = newurl_spec[:jj], newurl_spec[jj+1:]
- if key is None or key.strip().lower() != "url":
- debug("bad Refresh header: %r" % refresh)
- return response
- else:
- pause, newurl = float(refresh), response.geturl()
+ try:
+ pause, newurl = parse_refresh_header(refresh)
+ except ValueError:
+ debug("bad Refresh header: %r" % refresh)
+ return response
+ if newurl is None:
+ newurl = response.geturl()
if (self.max_time is None) or (pause <= self.max_time):
if pause > 1E-3 and self.honor_time:
time.sleep(pause)
Modified: wwwsearch/mechanize/trunk/test.py
==============================================================================
--- wwwsearch/mechanize/trunk/test.py (original)
+++ wwwsearch/mechanize/trunk/test.py Sat Jul 29 17:02:19 2006
@@ -14,7 +14,7 @@
"test_headers", "test_urllib2", "test_pullparser",
]
-import sys, os, traceback, logging
+import sys, os, traceback, logging, glob
from unittest import defaultTestLoader, TextTestRunner, TestSuite, TestCase
level = logging.DEBUG
@@ -108,6 +108,8 @@
## __builtin__.jjl = jjl
# XXX temporary stop-gap to run doctests
+
+ # import local copy of Python 2.5 doctest
assert os.path.isdir("test")
sys.path.insert(0, "test")
# needed for recent doctest / linecache -- this is only for testing
@@ -118,29 +120,43 @@
# that renamed module.
sys.path.insert(0, "test-tools")
import doctest
+
import mechanize
+
+ # run .doctest files needing special support
common_globs = {"mechanize": mechanize}
+ pm_doctest_filename = os.path.join("test", "test_password_manager.doctest")
for globs in [
{"mgr_class": mechanize.HTTPPasswordMgr},
{"mgr_class": mechanize.HTTPProxyPasswordMgr},
]:
globs.update(common_globs)
doctest.testfile(
- os.path.join("test", "test_password_manager.doctest"),
+ pm_doctest_filename,
#os.path.join("test", "test_scratch.doctest"),
globs=globs,
)
-
- doctest.testfile(os.path.join("test", "test_rfc3986.doctest"))
- doctest.testfile(os.path.join("test", "test_request.doctest"))
- doctest.testfile(os.path.join("test", "test_history.doctest"))
- doctest.testfile(os.path.join("test", "test_html.doctest"))
- from mechanize import _headersutil, _auth, _clientcookie, _pullparser
+
+ # run .doctest files
+ special_doctests = [pm_doctest_filename,
+ os.path.join("test", "test_scratch.doctest"),
+ ]
+ doctest_files = glob.glob(os.path.join("test", "*.doctest"))
+ for dt in special_doctests:
+ doctest_files.remove(dt)
+ for df in doctest_files:
+ doctest.testfile(df)
+
+ # run doctests in docstrings
+ from mechanize import _headersutil, _auth, _clientcookie, _pullparser, \
+ _http
doctest.testmod(_headersutil)
doctest.testmod(_auth)
doctest.testmod(_clientcookie)
doctest.testmod(_pullparser)
+ doctest.testmod(_http)
+ # run vanilla unittest tests
import unittest
test_path = os.path.join(os.path.dirname(sys.argv[0]), "test")
sys.path.insert(0, test_path)
From jjlee at codespeak.net Thu Aug 10 00:01:04 2006
From: jjlee at codespeak.net (jjlee at codespeak.net)
Date: Thu, 10 Aug 2006 00:01:04 +0200 (CEST)
Subject: [wwwsearch-commits] r31210 - in wwwsearch/mechanize/trunk: docs
mechanize test
Message-ID: <20060809220104.8BC9310060@code0.codespeak.net>
Author: jjlee
Date: Thu Aug 10 00:01:03 2006
New Revision: 31210
Added:
wwwsearch/mechanize/trunk/docs/
Modified:
wwwsearch/mechanize/trunk/mechanize/_http.py
wwwsearch/mechanize/trunk/test/test_urllib2.py
Log:
Fix HTTP-EQUIV with no content attribute case (Pratik Dam ); Fix assumption that httplib.HTTPMessage treats dict-style __setitem__ as append rather than set (where on earth did I get that from?)
Modified: wwwsearch/mechanize/trunk/mechanize/_http.py
==============================================================================
--- wwwsearch/mechanize/trunk/mechanize/_http.py (original)
+++ wwwsearch/mechanize/trunk/mechanize/_http.py Thu Aug 10 00:01:03 2006
@@ -148,7 +148,7 @@
http_equiv = self.unescape_attr_if_required(value)
elif key == "content":
content = self.unescape_attr_if_required(value)
- if http_equiv is not None:
+ if http_equiv is not None and content is not None:
self.http_equiv.append((http_equiv, content))
def end_head(self):
@@ -280,9 +280,9 @@
def http_response(self, request, response):
if not hasattr(response, "seek"):
response = response_seek_wrapper(response)
- headers = response.info()
+ http_message = response.info()
url = response.geturl()
- ct_hdrs = response.info().getheaders("content-type")
+ ct_hdrs = http_message.getheaders("content-type")
if is_html(ct_hdrs, url, self._allow_xhtml):
try:
try:
@@ -294,8 +294,11 @@
pass
else:
for hdr, val in html_headers:
- # rfc822.Message interprets this as appending, not clobbering
- headers[hdr] = val
+ # add a header
+ http_message.dict[hdr.lower()] = val
+ text = hdr + ": " + val
+ for line in text.split("\n"):
+ http_message.headers.append(line + "\n")
return response
https_response = http_response
Modified: wwwsearch/mechanize/trunk/test/test_urllib2.py
==============================================================================
--- wwwsearch/mechanize/trunk/test/test_urllib2.py (original)
+++ wwwsearch/mechanize/trunk/test/test_urllib2.py Thu Aug 10 00:01:03 2006
@@ -9,7 +9,7 @@
# ProxyHandler, CustomProxy, CustomProxyHandler (I don't use a proxy)
# GopherHandler (haven't used gopher for a decade or so...)
-import unittest, StringIO, os, sys, UserDict
+import unittest, StringIO, os, sys, UserDict, httplib
import mechanize
@@ -36,10 +36,18 @@
def readline(self, count=None): pass
def close(self): pass
-class MockHeaders(dict):
- def getheaders(self, name):
- name = name.lower()
- return [v for k, v in self.iteritems() if name == k.lower()]
+def http_message(mapping):
+ """
+ >>> http_message({"Content-Type": "text/html"}).items()
+ [('content-type', 'text/html')]
+
+ """
+ f = []
+ for kv in mapping.items():
+ f.append("%s: %s" % kv)
+ f.append("")
+ msg = httplib.HTTPMessage(StringIO.StringIO("\r\n".join(f)))
+ return msg
class MockResponse(StringIO.StringIO):
def __init__(self, code, msg, headers, data, url=None):
@@ -764,15 +772,19 @@
req = Request("http://example.com/")
r = MockResponse(
200, "OK",
- MockHeaders({"Foo": "Bar", "Content-type": "text/html"}),
+ http_message({"Foo": "Bar",
+ "Content-type": "text/html",
+ "Refresh": "blah"}),
''
''
- ''
+ '