[wwwsearch-commits] r27549 - in wwwsearch/mechanize/trunk: attic mechanize test
jjlee at codespeak.net
jjlee at codespeak.net
Sun May 21 21:32:20 CEST 2006
Author: jjlee
Date: Sun May 21 21:32:06 2006
New Revision: 27549
Added:
wwwsearch/mechanize/trunk/mechanize/_clientcookie.py
- copied, changed from r27546, wwwsearch/mechanize/trunk/mechanize/_ClientCookie.py
wwwsearch/mechanize/trunk/mechanize/_headersutil.py
- copied, changed from r27015, wwwsearch/mechanize/trunk/mechanize/_HeadersUtil.py
wwwsearch/mechanize/trunk/mechanize/_mozillacookiejar.py
- copied, changed from r27546, wwwsearch/mechanize/trunk/mechanize/_MozillaCookieJar.py
wwwsearch/mechanize/trunk/mechanize/_msiecookiejar.py
- copied, changed from r27546, wwwsearch/mechanize/trunk/mechanize/_MSIECookieJar.py
wwwsearch/mechanize/trunk/mechanize/_opener.py
- copied, changed from r27019, wwwsearch/mechanize/trunk/mechanize/_Opener.py
wwwsearch/mechanize/trunk/mechanize/_request.py
- copied, changed from r27015, wwwsearch/mechanize/trunk/mechanize/_Request.py
wwwsearch/mechanize/trunk/mechanize/_util.py
- copied unchanged from r27015, wwwsearch/mechanize/trunk/mechanize/_Util.py
Removed:
wwwsearch/mechanize/trunk/mechanize/_ClientCookie.py
wwwsearch/mechanize/trunk/mechanize/_HeadersUtil.py
wwwsearch/mechanize/trunk/mechanize/_MSIECookieJar.py
wwwsearch/mechanize/trunk/mechanize/_MozillaCookieJar.py
wwwsearch/mechanize/trunk/mechanize/_Opener.py
wwwsearch/mechanize/trunk/mechanize/_Request.py
wwwsearch/mechanize/trunk/mechanize/_Util.py
Modified:
wwwsearch/mechanize/trunk/attic/BSDDBCookieJar.py
wwwsearch/mechanize/trunk/attic/MSIEDBCookieJar.py
wwwsearch/mechanize/trunk/mechanize/__init__.py
wwwsearch/mechanize/trunk/mechanize/_gzip.py
wwwsearch/mechanize/trunk/mechanize/_html.py
wwwsearch/mechanize/trunk/mechanize/_mechanize.py
wwwsearch/mechanize/trunk/mechanize/_urllib2.py
wwwsearch/mechanize/trunk/mechanize/_urllib2_support.py
wwwsearch/mechanize/trunk/mechanize/_useragent.py
wwwsearch/mechanize/trunk/test/test_cookies.py
wwwsearch/mechanize/trunk/test/test_date.py
wwwsearch/mechanize/trunk/test/test_headers.py
wwwsearch/mechanize/trunk/test/test_mechanize.py
wwwsearch/mechanize/trunk/test/test_misc.py
wwwsearch/mechanize/trunk/test/test_urllib2.py
Log:
Rename modules to be all lower case
Modified: wwwsearch/mechanize/trunk/attic/BSDDBCookieJar.py
==============================================================================
--- wwwsearch/mechanize/trunk/attic/BSDDBCookieJar.py (original)
+++ wwwsearch/mechanize/trunk/attic/BSDDBCookieJar.py Sun May 21 21:32:06 2006
@@ -15,7 +15,7 @@
from mechanize import CookieJar
# this is importing from a private module: don't do this in your own code!
-from _ClientCookie import MappingIterator
+from _clientcookie import MappingIterator
import bsddb
import cPickle as pickle
Modified: wwwsearch/mechanize/trunk/attic/MSIEDBCookieJar.py
==============================================================================
--- wwwsearch/mechanize/trunk/attic/MSIEDBCookieJar.py (original)
+++ wwwsearch/mechanize/trunk/attic/MSIEDBCookieJar.py Sun May 21 21:32:06 2006
@@ -16,7 +16,7 @@
"""
from ClientCookie import MSIEBase, CookieJar
-from _Util import time2netscape
+from _util import time2netscape
def set_cookie_hdr_from_cookie(cookie):
params = []
Deleted: /wwwsearch/mechanize/trunk/mechanize/_ClientCookie.py
==============================================================================
--- /wwwsearch/mechanize/trunk/mechanize/_ClientCookie.py Sun May 21 21:32:06 2006
+++ (empty file)
@@ -1,1656 +0,0 @@
-"""HTTP cookie handling for web clients, plus some other stuff.
-
-This module originally developed from my port of Gisle Aas' Perl module
-HTTP::Cookies, from the libwww-perl library.
-
-Docstrings, comments and debug strings in this code refer to the
-attributes of the HTTP cookie system as cookie-attributes, to distinguish
-them clearly from Python attributes.
-
- CookieJar____
- / \ \
- FileCookieJar \ \
- / | \ \ \
- MozillaCookieJar | LWPCookieJar \ \
- | | \
- | ---MSIEBase | \
- | / | | \
- | / MSIEDBCookieJar BSDDBCookieJar
- |/
- MSIECookieJar
-
-Comments to John J Lee <jjl at pobox.com>.
-
-
-Copyright 2002-2006 John J Lee <jjl at pobox.com>
-Copyright 1997-1999 Gisle Aas (original libwww-perl code)
-Copyright 2002-2003 Johnny Lee (original MSIE Perl code)
-
-This code is free software; you can redistribute it and/or modify it
-under the terms of the BSD or ZPL 2.1 licenses (see the file
-COPYING.txt included with the distribution).
-
-"""
-
-import sys, re, urlparse, string, copy, time, struct, urllib, types, logging
-try:
- import threading
- _threading = threading; del threading
-except ImportError:
- import dummy_threading
- _threading = dummy_threading; del dummy_threading
-import httplib # only for the default HTTP port
-
-MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
- "instance initialised with one)")
-DEFAULT_HTTP_PORT = str(httplib.HTTP_PORT)
-
-from _HeadersUtil import split_header_words, parse_ns_headers
-from _Util import startswith, endswith, isstringlike, getheaders
-
-debug = logging.getLogger("mechanize.cookies").debug
-
-
-def reraise_unmasked_exceptions(unmasked=()):
- # There are a few catch-all except: statements in this module, for
- # catching input that's bad in unexpected ways.
- # This function re-raises some exceptions we don't want to trap.
- import mechanize, warnings
- if not mechanize.USE_BARE_EXCEPT:
- raise
- unmasked = unmasked + (KeyboardInterrupt, SystemExit, MemoryError)
- etype = sys.exc_info()[0]
- if issubclass(etype, unmasked):
- raise
- # swallowed an exception
- import traceback, StringIO
- f = StringIO.StringIO()
- traceback.print_exc(None, f)
- msg = f.getvalue()
- warnings.warn("mechanize bug!\n%s" % msg, stacklevel=2)
-
-
-IPV4_RE = re.compile(r"\.\d+$")
-def is_HDN(text):
- """Return True if text is a host domain name."""
- # XXX
- # This may well be wrong. Which RFC is HDN defined in, if any (for
- # the purposes of RFC 2965)?
- # For the current implementation, what about IPv6? Remember to look
- # at other uses of IPV4_RE also, if change this.
- return not (IPV4_RE.search(text) or
- text == "" or
- text[0] == "." or text[-1] == ".")
-
-def domain_match(A, B):
- """Return True if domain A domain-matches domain B, according to RFC 2965.
-
- A and B may be host domain names or IP addresses.
-
- RFC 2965, section 1:
-
- Host names can be specified either as an IP address or a HDN string.
- Sometimes we compare one host name with another. (Such comparisons SHALL
- be case-insensitive.) Host A's name domain-matches host B's if
-
- * their host name strings string-compare equal; or
-
- * A is a HDN string and has the form NB, where N is a non-empty
- name string, B has the form .B', and B' is a HDN string. (So,
- x.y.com domain-matches .Y.com but not Y.com.)
-
- Note that domain-match is not a commutative operation: a.b.c.com
- domain-matches .c.com, but not the reverse.
-
- """
- # Note that, if A or B are IP addresses, the only relevant part of the
- # definition of the domain-match algorithm is the direct string-compare.
- A = string.lower(A)
- B = string.lower(B)
- if A == B:
- return True
- if not is_HDN(A):
- return False
- i = string.rfind(A, B)
- has_form_nb = not (i == -1 or i == 0)
- return (
- has_form_nb and
- startswith(B, ".") and
- is_HDN(B[1:])
- )
-
-def liberal_is_HDN(text):
- """Return True if text is a sort-of-like a host domain name.
-
- For accepting/blocking domains.
-
- """
- return not IPV4_RE.search(text)
-
-def user_domain_match(A, B):
- """For blocking/accepting domains.
-
- A and B may be host domain names or IP addresses.
-
- """
- A = string.lower(A)
- B = string.lower(B)
- if not (liberal_is_HDN(A) and liberal_is_HDN(B)):
- if A == B:
- # equal IP addresses
- return True
- return False
- initial_dot = startswith(B, ".")
- if initial_dot and endswith(A, B):
- return True
- if not initial_dot and A == B:
- return True
- return False
-
-cut_port_re = re.compile(r":\d+$")
-def request_host(request):
- """Return request-host, as defined by RFC 2965.
-
- Variation from RFC: returned value is lowercased, for convenient
- comparison.
-
- """
- url = request.get_full_url()
- host = urlparse.urlparse(url)[1]
- if host == "":
- host = request.get_header("Host", "")
-
- # remove port, if present
- host = cut_port_re.sub("", host, 1)
- return string.lower(host)
-
-def eff_request_host(request):
- """Return a tuple (request-host, effective request-host name).
-
- As defined by RFC 2965, except both are lowercased.
-
- """
- erhn = req_host = request_host(request)
- if string.find(req_host, ".") == -1 and not IPV4_RE.search(req_host):
- erhn = req_host + ".local"
- return req_host, erhn
-
-def request_path(request):
- """request-URI, as defined by RFC 2965."""
- url = request.get_full_url()
- #scheme, netloc, path, parameters, query, frag = urlparse.urlparse(url)
- #req_path = escape_path(string.join(urlparse.urlparse(url)[2:], ""))
- path, parameters, query, frag = urlparse.urlparse(url)[2:]
- if parameters:
- path = "%s;%s" % (path, parameters)
- path = escape_path(path)
- req_path = urlparse.urlunparse(("", "", path, "", query, frag))
- if not startswith(req_path, "/"):
- # fix bad RFC 2396 absoluteURI
- req_path = "/"+req_path
- return req_path
-
-def request_port(request):
- host = request.get_host()
- i = string.find(host, ':')
- if i >= 0:
- port = host[i+1:]
- try:
- int(port)
- except ValueError:
- debug("nonnumeric port: '%s'", port)
- return None
- else:
- port = DEFAULT_HTTP_PORT
- return port
-
-# Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't
-# need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).
-HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"
-ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")
-def uppercase_escaped_char(match):
- return "%%%s" % string.upper(match.group(1))
-def escape_path(path):
- """Escape any invalid characters in HTTP URL, and uppercase all escapes."""
- # There's no knowing what character encoding was used to create URLs
- # containing %-escapes, but since we have to pick one to escape invalid
- # path characters, we pick UTF-8, as recommended in the HTML 4.0
- # specification:
- # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
- # And here, kind of: draft-fielding-uri-rfc2396bis-03
- # (And in draft IRI specification: draft-duerst-iri-05)
- # (And here, for new URI schemes: RFC 2718)
- if isinstance(path, types.UnicodeType):
- path = path.encode("utf-8")
- path = urllib.quote(path, HTTP_PATH_SAFE)
- path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
- return path
-
-def reach(h):
- """Return reach of host h, as defined by RFC 2965, section 1.
-
- The reach R of a host name H is defined as follows:
-
- * If
-
- - H is the host domain name of a host; and,
-
- - H has the form A.B; and
-
- - A has no embedded (that is, interior) dots; and
-
- - B has at least one embedded dot, or B is the string "local".
- then the reach of H is .B.
-
- * Otherwise, the reach of H is H.
-
- >>> reach("www.acme.com")
- '.acme.com'
- >>> reach("acme.com")
- 'acme.com'
- >>> reach("acme.local")
- '.local'
-
- """
- i = string.find(h, ".")
- if i >= 0:
- #a = h[:i] # this line is only here to show what a is
- b = h[i+1:]
- i = string.find(b, ".")
- if is_HDN(h) and (i >= 0 or b == "local"):
- return "."+b
- return h
-
-def is_third_party(request):
- """
-
- RFC 2965, section 3.3.6:
-
- An unverifiable transaction is to a third-party host if its request-
- host U does not domain-match the reach R of the request-host O in the
- origin transaction.
-
- """
- req_host = request_host(request)
- # the origin request's request-host was stuffed into request by
- # _urllib2_support.AbstractHTTPHandler
- return not domain_match(req_host, reach(request.origin_req_host))
-
-
-class Cookie:
- """HTTP Cookie.
-
- This class represents both Netscape and RFC 2965 cookies.
-
- This is deliberately a very simple class. It just holds attributes. It's
- possible to construct Cookie instances that don't comply with the cookie
- standards. CookieJar.make_cookies is the factory function for Cookie
- objects -- it deals with cookie parsing, supplying defaults, and
- normalising to the representation used in this class. CookiePolicy is
- responsible for checking them to see whether they should be accepted from
- and returned to the server.
-
- version: integer;
- name: string;
- value: string (may be None);
- port: string; None indicates no attribute was supplied (eg. "Port", rather
- than eg. "Port=80"); otherwise, a port string (eg. "80") or a port list
- string (eg. "80,8080")
- port_specified: boolean; true if a value was supplied with the Port
- cookie-attribute
- domain: string;
- domain_specified: boolean; true if Domain was explicitly set
- domain_initial_dot: boolean; true if Domain as set in HTTP header by server
- started with a dot (yes, this really is necessary!)
- path: string;
- path_specified: boolean; true if Path was explicitly set
- secure: boolean; true if should only be returned over secure connection
- expires: integer; seconds since epoch (RFC 2965 cookies should calculate
- this value from the Max-Age attribute)
- discard: boolean, true if this is a session cookie; (if no expires value,
- this should be true)
- comment: string;
- comment_url: string;
- rfc2109: boolean; true if cookie arrived in a Set-Cookie: (not
- Set-Cookie2:) header, but had a version cookie-attribute of 1
- rest: mapping of other cookie-attributes
-
- Note that the port may be present in the headers, but unspecified ("Port"
- rather than"Port=80", for example); if this is the case, port is None.
-
- """
-
- def __init__(self, version, name, value,
- port, port_specified,
- domain, domain_specified, domain_initial_dot,
- path, path_specified,
- secure,
- expires,
- discard,
- comment,
- comment_url,
- rest,
- rfc2109=False,
- ):
-
- if version is not None: version = int(version)
- if expires is not None: expires = int(expires)
- if port is None and port_specified is True:
- raise ValueError("if port is None, port_specified must be false")
-
- self.version = version
- self.name = name
- self.value = value
- self.port = port
- self.port_specified = port_specified
- # normalise case, as per RFC 2965 section 3.3.3
- self.domain = string.lower(domain)
- self.domain_specified = domain_specified
- # Sigh. We need to know whether the domain given in the
- # cookie-attribute had an initial dot, in order to follow RFC 2965
- # (as clarified in draft errata). Needed for the returned $Domain
- # value.
- self.domain_initial_dot = domain_initial_dot
- self.path = path
- self.path_specified = path_specified
- self.secure = secure
- self.expires = expires
- self.discard = discard
- self.comment = comment
- self.comment_url = comment_url
- self.rfc2109 = rfc2109
-
- self._rest = copy.copy(rest)
-
- def has_nonstandard_attr(self, name):
- return self._rest.has_key(name)
- def get_nonstandard_attr(self, name, default=None):
- return self._rest.get(name, default)
- def set_nonstandard_attr(self, name, value):
- self._rest[name] = value
- def nonstandard_attr_keys(self):
- return self._rest.keys()
-
- def is_expired(self, now=None):
- if now is None: now = time.time()
- return (self.expires is not None) and (self.expires <= now)
-
- def __str__(self):
- if self.port is None: p = ""
- else: p = ":"+self.port
- limit = self.domain + p + self.path
- if self.value is not None:
- namevalue = "%s=%s" % (self.name, self.value)
- else:
- namevalue = self.name
- return "<Cookie %s for %s>" % (namevalue, limit)
-
- def __repr__(self):
- args = []
- for name in ["version", "name", "value",
- "port", "port_specified",
- "domain", "domain_specified", "domain_initial_dot",
- "path", "path_specified",
- "secure", "expires", "discard", "comment", "comment_url",
- ]:
- attr = getattr(self, name)
- args.append("%s=%s" % (name, repr(attr)))
- args.append("rest=%s" % repr(self._rest))
- args.append("rfc2109=%s" % repr(self.rfc2109))
- return "Cookie(%s)" % string.join(args, ", ")
-
-
-class CookiePolicy:
- """Defines which cookies get accepted from and returned to server.
-
- May also modify cookies.
-
- The subclass DefaultCookiePolicy defines the standard rules for Netscape
- and RFC 2965 cookies -- override that if you want a customised policy.
-
- As well as implementing set_ok and return_ok, implementations of this
- interface must also supply the following attributes, indicating which
- protocols should be used, and how. These can be read and set at any time,
- though whether that makes complete sense from the protocol point of view is
- doubtful.
-
- Public attributes:
-
- netscape: implement netscape protocol
- rfc2965: implement RFC 2965 protocol
- rfc2109_as_netscape:
- WARNING: This argument will change or go away if is not accepted into
- the Python standard library in this form!
- If true, treat RFC 2109 cookies as though they were Netscape cookies. The
- default is for this attribute to be None, which means treat 2109 cookies
- as RFC 2965 cookies unless RFC 2965 handling is switched off (which it is,
- by default), and as Netscape cookies otherwise.
- hide_cookie2: don't add Cookie2 header to requests (the presence of
- this header indicates to the server that we understand RFC 2965
- cookies)
-
- """
- def set_ok(self, cookie, request):
- """Return true if (and only if) cookie should be accepted from server.
-
- Currently, pre-expired cookies never get this far -- the CookieJar
- class deletes such cookies itself.
-
- cookie: mechanize.Cookie object
- request: object implementing the interface defined by
- CookieJar.extract_cookies.__doc__
-
- """
- raise NotImplementedError()
-
- def return_ok(self, cookie, request):
- """Return true if (and only if) cookie should be returned to server.
-
- cookie: mechanize.Cookie object
- request: object implementing the interface defined by
- CookieJar.add_cookie_header.__doc__
-
- """
- raise NotImplementedError()
-
- def domain_return_ok(self, domain, request):
- """Return false if cookies should not be returned, given cookie domain.
-
- This is here as an optimization, to remove the need for checking every
- cookie with a particular domain (which may involve reading many files).
- The default implementations of domain_return_ok and path_return_ok
- (return True) leave all the work to return_ok.
-
- If domain_return_ok returns true for the cookie domain, path_return_ok
- is called for the cookie path. Otherwise, path_return_ok and return_ok
- are never called for that cookie domain. If path_return_ok returns
- true, return_ok is called with the Cookie object itself for a full
- check. Otherwise, return_ok is never called for that cookie path.
-
- Note that domain_return_ok is called for every *cookie* domain, not
- just for the *request* domain. For example, the function might be
- called with both ".acme.com" and "www.acme.com" if the request domain is
- "www.acme.com". The same goes for path_return_ok.
-
- For argument documentation, see the docstring for return_ok.
-
- """
- return True
-
- def path_return_ok(self, path, request):
- """Return false if cookies should not be returned, given cookie path.
-
- See the docstring for domain_return_ok.
-
- """
- return True
-
-
-class DefaultCookiePolicy(CookiePolicy):
- """Implements the standard rules for accepting and returning cookies.
-
- Both RFC 2965 and Netscape cookies are covered. RFC 2965 handling is
- switched off by default.
-
- The easiest way to provide your own policy is to override this class and
- call its methods in your overriden implementations before adding your own
- additional checks.
-
- import mechanize
- class MyCookiePolicy(mechanize.DefaultCookiePolicy):
- def set_ok(self, cookie, request):
- if not mechanize.DefaultCookiePolicy.set_ok(
- self, cookie, request):
- return False
- if i_dont_want_to_store_this_cookie():
- return False
- return True
-
- In addition to the features required to implement the CookiePolicy
- interface, this class allows you to block and allow domains from setting
- and receiving cookies. There are also some strictness switches that allow
- you to tighten up the rather loose Netscape protocol rules a little bit (at
- the cost of blocking some benign cookies).
-
- A domain blacklist and whitelist is provided (both off by default). Only
- domains not in the blacklist and present in the whitelist (if the whitelist
- is active) participate in cookie setting and returning. Use the
- blocked_domains constructor argument, and blocked_domains and
- set_blocked_domains methods (and the corresponding argument and methods for
- allowed_domains). If you set a whitelist, you can turn it off again by
- setting it to None.
-
- Domains in block or allow lists that do not start with a dot must
- string-compare equal. For example, "acme.com" matches a blacklist entry of
- "acme.com", but "www.acme.com" does not. Domains that do start with a dot
- are matched by more specific domains too. For example, both "www.acme.com"
- and "www.munitions.acme.com" match ".acme.com" (but "acme.com" itself does
- not). IP addresses are an exception, and must match exactly. For example,
- if blocked_domains contains "192.168.1.2" and ".168.1.2" 192.168.1.2 is
- blocked, but 193.168.1.2 is not.
-
- Additional Public Attributes:
-
- General strictness switches
-
- strict_domain: don't allow sites to set two-component domains with
- country-code top-level domains like .co.uk, .gov.uk, .co.nz. etc.
- This is far from perfect and isn't guaranteed to work!
-
- RFC 2965 protocol strictness switches
-
- strict_rfc2965_unverifiable: follow RFC 2965 rules on unverifiable
- transactions (usually, an unverifiable transaction is one resulting from
- a redirect or an image hosted on another site); if this is false, cookies
- are NEVER blocked on the basis of verifiability
-
- Netscape protocol strictness switches
-
- strict_ns_unverifiable: apply RFC 2965 rules on unverifiable transactions
- even to Netscape cookies
- strict_ns_domain: flags indicating how strict to be with domain-matching
- rules for Netscape cookies:
- DomainStrictNoDots: when setting cookies, host prefix must not contain a
- dot (eg. www.foo.bar.com can't set a cookie for .bar.com, because
- www.foo contains a dot)
- DomainStrictNonDomain: cookies that did not explicitly specify a Domain
- cookie-attribute can only be returned to a domain that string-compares
- equal to the domain that set the cookie (eg. rockets.acme.com won't
- be returned cookies from acme.com that had no Domain cookie-attribute)
- DomainRFC2965Match: when setting cookies, require a full RFC 2965
- domain-match
- DomainLiberal and DomainStrict are the most useful combinations of the
- above flags, for convenience
- strict_ns_set_initial_dollar: ignore cookies in Set-Cookie: headers that
- have names starting with '$'
- strict_ns_set_path: don't allow setting cookies whose path doesn't
- path-match request URI
-
- """
-
- DomainStrictNoDots = 1
- DomainStrictNonDomain = 2
- DomainRFC2965Match = 4
-
- DomainLiberal = 0
- DomainStrict = DomainStrictNoDots|DomainStrictNonDomain
-
- def __init__(self,
- blocked_domains=None, allowed_domains=None,
- netscape=True, rfc2965=False,
- # WARNING: this argument will change or go away if is not
- # accepted into the Python standard library in this form!
- # default, ie. treat 2109 as netscape iff not rfc2965
- rfc2109_as_netscape=None,
- hide_cookie2=False,
- strict_domain=False,
- strict_rfc2965_unverifiable=True,
- strict_ns_unverifiable=False,
- strict_ns_domain=DomainLiberal,
- strict_ns_set_initial_dollar=False,
- strict_ns_set_path=False,
- ):
- """
- Constructor arguments should be used as keyword arguments only.
-
- blocked_domains: sequence of domain names that we never accept cookies
- from, nor return cookies to
- allowed_domains: if not None, this is a sequence of the only domains
- for which we accept and return cookies
-
- For other arguments, see CookiePolicy.__doc__ and
- DefaultCookiePolicy.__doc__..
-
- """
- self.netscape = netscape
- self.rfc2965 = rfc2965
- self.rfc2109_as_netscape = rfc2109_as_netscape
- self.hide_cookie2 = hide_cookie2
- self.strict_domain = strict_domain
- self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
- self.strict_ns_unverifiable = strict_ns_unverifiable
- self.strict_ns_domain = strict_ns_domain
- self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
- self.strict_ns_set_path = strict_ns_set_path
-
- if blocked_domains is not None:
- self._blocked_domains = tuple(blocked_domains)
- else:
- self._blocked_domains = ()
-
- if allowed_domains is not None:
- allowed_domains = tuple(allowed_domains)
- self._allowed_domains = allowed_domains
-
- def blocked_domains(self):
- """Return the sequence of blocked domains (as a tuple)."""
- return self._blocked_domains
- def set_blocked_domains(self, blocked_domains):
- """Set the sequence of blocked domains."""
- self._blocked_domains = tuple(blocked_domains)
-
- def is_blocked(self, domain):
- for blocked_domain in self._blocked_domains:
- if user_domain_match(domain, blocked_domain):
- return True
- return False
-
- def allowed_domains(self):
- """Return None, or the sequence of allowed domains (as a tuple)."""
- return self._allowed_domains
- def set_allowed_domains(self, allowed_domains):
- """Set the sequence of allowed domains, or None."""
- if allowed_domains is not None:
- allowed_domains = tuple(allowed_domains)
- self._allowed_domains = allowed_domains
-
- def is_not_allowed(self, domain):
- if self._allowed_domains is None:
- return False
- for allowed_domain in self._allowed_domains:
- if user_domain_match(domain, allowed_domain):
- return False
- return True
-
- def set_ok(self, cookie, request):
- """
- If you override set_ok, be sure to call this method. If it returns
- false, so should your subclass (assuming your subclass wants to be more
- strict about which cookies to accept).
-
- """
- debug(" - checking cookie %s", cookie)
-
- assert cookie.name is not None
-
- for n in "version", "verifiability", "name", "path", "domain", "port":
- fn_name = "set_ok_"+n
- fn = getattr(self, fn_name)
- if not fn(cookie, request):
- return False
-
- return True
-
- def set_ok_version(self, cookie, request):
- if cookie.version is None:
- # Version is always set to 0 by parse_ns_headers if it's a Netscape
- # cookie, so this must be an invalid RFC 2965 cookie.
- debug(" Set-Cookie2 without version attribute (%s)", cookie)
- return False
- if cookie.version > 0 and not self.rfc2965:
- debug(" RFC 2965 cookies are switched off")
- return False
- elif cookie.version == 0 and not self.netscape:
- debug(" Netscape cookies are switched off")
- return False
- return True
-
- def set_ok_verifiability(self, cookie, request):
- if request.unverifiable and is_third_party(request):
- if cookie.version > 0 and self.strict_rfc2965_unverifiable:
- debug(" third-party RFC 2965 cookie during "
- "unverifiable transaction")
- return False
- elif cookie.version == 0 and self.strict_ns_unverifiable:
- debug(" third-party Netscape cookie during "
- "unverifiable transaction")
- return False
- return True
-
- def set_ok_name(self, cookie, request):
- # Try and stop servers setting V0 cookies designed to hack other
- # servers that know both V0 and V1 protocols.
- if (cookie.version == 0 and self.strict_ns_set_initial_dollar and
- startswith(cookie.name, "$")):
- debug(" illegal name (starts with '$'): '%s'", cookie.name)
- return False
- return True
-
- def set_ok_path(self, cookie, request):
- if cookie.path_specified:
- req_path = request_path(request)
- if ((cookie.version > 0 or
- (cookie.version == 0 and self.strict_ns_set_path)) and
- not startswith(req_path, cookie.path)):
- debug(" path attribute %s is not a prefix of request "
- "path %s", cookie.path, req_path)
- return False
- return True
-
- def set_ok_countrycode_domain(self, cookie, request):
- """Return False if explicit cookie domain is not acceptable.
-
- Called by set_ok_domain, for convenience of overriding by
- subclasses.
-
- """
- if cookie.domain_specified and self.strict_domain:
- domain = cookie.domain
- # since domain was specified, we know that:
- assert domain.startswith(".")
- if string.count(domain, ".") == 2:
- # domain like .foo.bar
- i = string.rfind(domain, ".")
- tld = domain[i+1:]
- sld = domain[1:i]
- if (string.lower(sld) in [
- "co", "ac",
- "com", "edu", "org", "net", "gov", "mil", "int",
- "aero", "biz", "cat", "coop", "info", "jobs", "mobi",
- "museum", "name", "pro", "travel",
- ] and
- len(tld) == 2):
- # domain like .co.uk
- return False
- return True
-
- def set_ok_domain(self, cookie, request):
- if self.is_blocked(cookie.domain):
- debug(" domain %s is in user block-list", cookie.domain)
- return False
- if self.is_not_allowed(cookie.domain):
- debug(" domain %s is not in user allow-list", cookie.domain)
- return False
- if not self.set_ok_countrycode_domain(cookie, request):
- debug(" country-code second level domain %s", cookie.domain)
- return False
- if cookie.domain_specified:
- req_host, erhn = eff_request_host(request)
- domain = cookie.domain
- if startswith(domain, "."):
- undotted_domain = domain[1:]
- else:
- undotted_domain = domain
- embedded_dots = (string.find(undotted_domain, ".") >= 0)
- if not embedded_dots and domain != ".local":
- debug(" non-local domain %s contains no embedded dot",
- domain)
- return False
- if cookie.version == 0:
- if (not endswith(erhn, domain) and
- (not startswith(erhn, ".") and
- not endswith("."+erhn, domain))):
- debug(" effective request-host %s (even with added "
- "initial dot) does not end end with %s",
- erhn, domain)
- return False
- if (cookie.version > 0 or
- (self.strict_ns_domain & self.DomainRFC2965Match)):
- if not domain_match(erhn, domain):
- debug(" effective request-host %s does not domain-match "
- "%s", erhn, domain)
- return False
- if (cookie.version > 0 or
- (self.strict_ns_domain & self.DomainStrictNoDots)):
- host_prefix = req_host[:-len(domain)]
- if (string.find(host_prefix, ".") >= 0 and
- not IPV4_RE.search(req_host)):
- debug(" host prefix %s for domain %s contains a dot",
- host_prefix, domain)
- return False
- return True
-
- def set_ok_port(self, cookie, request):
- if cookie.port_specified:
- req_port = request_port(request)
- if req_port is None:
- req_port = "80"
- else:
- req_port = str(req_port)
- for p in string.split(cookie.port, ","):
- try:
- int(p)
- except ValueError:
- debug(" bad port %s (not numeric)", p)
- return False
- if p == req_port:
- break
- else:
- debug(" request port (%s) not found in %s",
- req_port, cookie.port)
- return False
- return True
-
- def return_ok(self, cookie, request):
- """
- If you override return_ok, be sure to call this method. If it returns
- false, so should your subclass (assuming your subclass wants to be more
- strict about which cookies to return).
-
- """
- # Path has already been checked by path_return_ok, and domain blocking
- # done by domain_return_ok.
- debug(" - checking cookie %s", cookie)
-
- for n in "version", "verifiability", "secure", "expires", "port", "domain":
- fn_name = "return_ok_"+n
- fn = getattr(self, fn_name)
- if not fn(cookie, request):
- return False
- return True
-
- def return_ok_version(self, cookie, request):
- if cookie.version > 0 and not self.rfc2965:
- debug(" RFC 2965 cookies are switched off")
- return False
- elif cookie.version == 0 and not self.netscape:
- debug(" Netscape cookies are switched off")
- return False
- return True
-
- def return_ok_verifiability(self, cookie, request):
- if request.unverifiable and is_third_party(request):
- if cookie.version > 0 and self.strict_rfc2965_unverifiable:
- debug(" third-party RFC 2965 cookie during unverifiable "
- "transaction")
- return False
- elif cookie.version == 0 and self.strict_ns_unverifiable:
- debug(" third-party Netscape cookie during unverifiable "
- "transaction")
- return False
- return True
-
- def return_ok_secure(self, cookie, request):
- if cookie.secure and request.get_type() != "https":
- debug(" secure cookie with non-secure request")
- return False
- return True
-
- def return_ok_expires(self, cookie, request):
- if cookie.is_expired(self._now):
- debug(" cookie expired")
- return False
- return True
-
- def return_ok_port(self, cookie, request):
- if cookie.port:
- req_port = request_port(request)
- if req_port is None:
- req_port = "80"
- for p in string.split(cookie.port, ","):
- if p == req_port:
- break
- else:
- debug(" request port %s does not match cookie port %s",
- req_port, cookie.port)
- return False
- return True
-
- def return_ok_domain(self, cookie, request):
- req_host, erhn = eff_request_host(request)
- domain = cookie.domain
-
- # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
- if (cookie.version == 0 and
- (self.strict_ns_domain & self.DomainStrictNonDomain) and
- not cookie.domain_specified and domain != erhn):
- debug(" cookie with unspecified domain does not string-compare "
- "equal to request domain")
- return False
-
- if cookie.version > 0 and not domain_match(erhn, domain):
- debug(" effective request-host name %s does not domain-match "
- "RFC 2965 cookie domain %s", erhn, domain)
- return False
- if cookie.version == 0 and not endswith("."+erhn, domain):
- debug(" request-host %s does not match Netscape cookie domain "
- "%s", req_host, domain)
- return False
- return True
-
- def domain_return_ok(self, domain, request):
- # Liberal check of domain. This is here as an optimization to avoid
- # having to load lots of MSIE cookie files unless necessary.
-
- # Munge req_host and erhn to always start with a dot, so as to err on
- # the side of letting cookies through.
- dotted_req_host, dotted_erhn = eff_request_host(request)
- if not startswith(dotted_req_host, "."):
- dotted_req_host = "."+dotted_req_host
- if not startswith(dotted_erhn, "."):
- dotted_erhn = "."+dotted_erhn
- if not (endswith(dotted_req_host, domain) or
- endswith(dotted_erhn, domain)):
- #debug(" request domain %s does not match cookie domain %s",
- # req_host, domain)
- return False
-
- if self.is_blocked(domain):
- debug(" domain %s is in user block-list", domain)
- return False
- if self.is_not_allowed(domain):
- debug(" domain %s is not in user allow-list", domain)
- return False
-
- return True
-
- def path_return_ok(self, path, request):
- debug("- checking cookie path=%s", path)
- req_path = request_path(request)
- if not startswith(req_path, path):
- debug(" %s does not path-match %s", req_path, path)
- return False
- return True
-
-
-def vals_sorted_by_key(adict):
- keys = adict.keys()
- keys.sort()
- return map(adict.get, keys)
-
-class MappingIterator:
- """Iterates over nested mapping, depth-first, in sorted order by key."""
- def __init__(self, mapping):
- self._s = [(vals_sorted_by_key(mapping), 0, None)] # LIFO stack
-
- def __iter__(self): return self
-
- def next(self):
- # this is hairy because of lack of generators
- while 1:
- try:
- vals, i, prev_item = self._s.pop()
- except IndexError:
- raise StopIteration()
- if i < len(vals):
- item = vals[i]
- i = i + 1
- self._s.append((vals, i, prev_item))
- try:
- item.items
- except AttributeError:
- # non-mapping
- break
- else:
- # mapping
- self._s.append((vals_sorted_by_key(item), 0, item))
- continue
- return item
-
-
-# Used as second parameter to dict.get method, to distinguish absent
-# dict key from one with a None value.
-class Absent: pass
-
-class CookieJar:
- """Collection of HTTP cookies.
-
- You may not need to know about this class: try mechanize.urlopen().
-
- The major methods are extract_cookies and add_cookie_header; these are all
- you are likely to need.
-
- CookieJar supports the iterator protocol:
-
- for cookie in cookiejar:
- # do something with cookie
-
- Methods:
-
- add_cookie_header(request)
- extract_cookies(response, request)
- make_cookies(response, request)
- set_cookie_if_ok(cookie, request)
- set_cookie(cookie)
- clear_session_cookies()
- clear_expired_cookies()
- clear(domain=None, path=None, name=None)
-
- Public attributes
-
- policy: CookiePolicy object
-
- """
-
- non_word_re = re.compile(r"\W")
- quote_re = re.compile(r"([\"\\])")
- strict_domain_re = re.compile(r"\.?[^.]*")
- domain_re = re.compile(r"[^.]*")
- dots_re = re.compile(r"^\.+")
-
- def __init__(self, policy=None):
- """
- See CookieJar.__doc__ for argument documentation.
-
- """
- if policy is None:
- policy = DefaultCookiePolicy()
- self._policy = policy
-
- self._cookies = {}
-
- # for __getitem__ iteration in pre-2.2 Pythons
- self._prev_getitem_index = 0
-
- def set_policy(self, policy):
- self._policy = policy
-
- def _cookies_for_domain(self, domain, request):
- cookies = []
- if not self._policy.domain_return_ok(domain, request):
- return []
- debug("Checking %s for cookies to return", domain)
- cookies_by_path = self._cookies[domain]
- for path in cookies_by_path.keys():
- if not self._policy.path_return_ok(path, request):
- continue
- cookies_by_name = cookies_by_path[path]
- for cookie in cookies_by_name.values():
- if not self._policy.return_ok(cookie, request):
- debug(" not returning cookie")
- continue
- debug(" it's a match")
- cookies.append(cookie)
- return cookies
-
- def _cookies_for_request(self, request):
- """Return a list of cookies to be returned to server."""
- cookies = []
- for domain in self._cookies.keys():
- cookies.extend(self._cookies_for_domain(domain, request))
- return cookies
-
- def _cookie_attrs(self, cookies):
- """Return a list of cookie-attributes to be returned to server.
-
- like ['foo="bar"; $Path="/"', ...]
-
- The $Version attribute is also added when appropriate (currently only
- once per request).
-
- """
- # add cookies in order of most specific (ie. longest) path first
- def decreasing_size(a, b): return cmp(len(b.path), len(a.path))
- cookies.sort(decreasing_size)
-
- version_set = False
-
- attrs = []
- for cookie in cookies:
- # set version of Cookie header
- # XXX
- # What should it be if multiple matching Set-Cookie headers have
- # different versions themselves?
- # Answer: there is no answer; was supposed to be settled by
- # RFC 2965 errata, but that may never appear...
- version = cookie.version
- if not version_set:
- version_set = True
- if version > 0:
- attrs.append("$Version=%s" % version)
-
- # quote cookie value if necessary
- # (not for Netscape protocol, which already has any quotes
- # intact, due to the poorly-specified Netscape Cookie: syntax)
- if ((cookie.value is not None) and
- self.non_word_re.search(cookie.value) and version > 0):
- value = self.quote_re.sub(r"\\\1", cookie.value)
- else:
- value = cookie.value
-
- # add cookie-attributes to be returned in Cookie header
- if cookie.value is None:
- attrs.append(cookie.name)
- else:
- attrs.append("%s=%s" % (cookie.name, value))
- if version > 0:
- if cookie.path_specified:
- attrs.append('$Path="%s"' % cookie.path)
- if startswith(cookie.domain, "."):
- domain = cookie.domain
- if (not cookie.domain_initial_dot and
- startswith(domain, ".")):
- domain = domain[1:]
- attrs.append('$Domain="%s"' % domain)
- if cookie.port is not None:
- p = "$Port"
- if cookie.port_specified:
- p = p + ('="%s"' % cookie.port)
- attrs.append(p)
-
- return attrs
-
- def add_cookie_header(self, request):
- """Add correct Cookie: header to request (urllib2.Request object).
-
- The Cookie2 header is also added unless policy.hide_cookie2 is true.
-
- The request object (usually a urllib2.Request instance) must support
- the methods get_full_url, get_host, get_type, has_header, get_header,
- header_items and add_unredirected_header, as documented by urllib2, and
- the port attribute (the port number). Actually,
- RequestUpgradeProcessor will automatically upgrade your Request object
- to one with has_header, get_header, header_items and
- add_unredirected_header, if it lacks those methods, for compatibility
- with pre-2.4 versions of urllib2.
-
- """
- debug("add_cookie_header")
- self._policy._now = self._now = int(time.time())
-
- req_host, erhn = eff_request_host(request)
- strict_non_domain = (
- self._policy.strict_ns_domain & self._policy.DomainStrictNonDomain)
-
- cookies = self._cookies_for_request(request)
-
- attrs = self._cookie_attrs(cookies)
- if attrs:
- if not request.has_header("Cookie"):
- request.add_unredirected_header(
- "Cookie", string.join(attrs, "; "))
-
- # if necessary, advertise that we know RFC 2965
- if self._policy.rfc2965 and not self._policy.hide_cookie2:
- for cookie in cookies:
- if cookie.version != 1 and not request.has_header("Cookie2"):
- request.add_unredirected_header("Cookie2", '$Version="1"')
- break
-
- self.clear_expired_cookies()
-
- def _normalized_cookie_tuples(self, attrs_set):
- """Return list of tuples containing normalised cookie information.
-
- attrs_set is the list of lists of key,value pairs extracted from
- the Set-Cookie or Set-Cookie2 headers.
-
- Tuples are name, value, standard, rest, where name and value are the
- cookie name and value, standard is a dictionary containing the standard
- cookie-attributes (discard, secure, version, expires or max-age,
- domain, path and port) and rest is a dictionary containing the rest of
- the cookie-attributes.
-
- """
- cookie_tuples = []
-
- boolean_attrs = "discard", "secure"
- value_attrs = ("version",
- "expires", "max-age",
- "domain", "path", "port",
- "comment", "commenturl")
-
- for cookie_attrs in attrs_set:
- name, value = cookie_attrs[0]
-
- # Build dictionary of standard cookie-attributes (standard) and
- # dictionary of other cookie-attributes (rest).
-
- # Note: expiry time is normalised to seconds since epoch. V0
- # cookies should have the Expires cookie-attribute, and V1 cookies
- # should have Max-Age, but since V1 includes RFC 2109 cookies (and
- # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we
- # accept either (but prefer Max-Age).
- max_age_set = False
-
- bad_cookie = False
-
- standard = {}
- rest = {}
- for k, v in cookie_attrs[1:]:
- lc = string.lower(k)
- # don't lose case distinction for unknown fields
- if lc in value_attrs or lc in boolean_attrs:
- k = lc
- if k in boolean_attrs and v is None:
- # boolean cookie-attribute is present, but has no value
- # (like "discard", rather than "port=80")
- v = True
- if standard.has_key(k):
- # only first value is significant
- continue
- if k == "domain":
- if v is None:
- debug(" missing value for domain attribute")
- bad_cookie = True
- break
- # RFC 2965 section 3.3.3
- v = string.lower(v)
- if k == "expires":
- if max_age_set:
- # Prefer max-age to expires (like Mozilla)
- continue
- if v is None:
- debug(" missing or invalid value for expires "
- "attribute: treating as session cookie")
- continue
- if k == "max-age":
- max_age_set = True
- try:
- v = int(v)
- except ValueError:
- debug(" missing or invalid (non-numeric) value for "
- "max-age attribute")
- bad_cookie = True
- break
- # convert RFC 2965 Max-Age to seconds since epoch
- # XXX Strictly you're supposed to follow RFC 2616
- # age-calculation rules. Remember that zero Max-Age is a
- # is a request to discard (old and new) cookie, though.
- k = "expires"
- v = self._now + v
- if (k in value_attrs) or (k in boolean_attrs):
- if (v is None and
- k not in ["port", "comment", "commenturl"]):
- debug(" missing value for %s attribute" % k)
- bad_cookie = True
- break
- standard[k] = v
- else:
- rest[k] = v
-
- if bad_cookie:
- continue
-
- cookie_tuples.append((name, value, standard, rest))
-
- return cookie_tuples
-
- def _cookie_from_cookie_tuple(self, tup, request):
- # standard is dict of standard cookie-attributes, rest is dict of the
- # rest of them
- name, value, standard, rest = tup
-
- domain = standard.get("domain", Absent)
- path = standard.get("path", Absent)
- port = standard.get("port", Absent)
- expires = standard.get("expires", Absent)
-
- # set the easy defaults
- version = standard.get("version", None)
- if version is not None: version = int(version)
- secure = standard.get("secure", False)
- # (discard is also set if expires is Absent)
- discard = standard.get("discard", False)
- comment = standard.get("comment", None)
- comment_url = standard.get("commenturl", None)
-
- # set default path
- if path is not Absent and path != "":
- path_specified = True
- path = escape_path(path)
- else:
- path_specified = False
- path = request_path(request)
- i = string.rfind(path, "/")
- if i != -1:
- if version == 0:
- # Netscape spec parts company from reality here
- path = path[:i]
- else:
- path = path[:i+1]
- if len(path) == 0: path = "/"
-
- # set default domain
- domain_specified = domain is not Absent
- # but first we have to remember whether it starts with a dot
- domain_initial_dot = False
- if domain_specified:
- domain_initial_dot = bool(startswith(domain, "."))
- if domain is Absent:
- req_host, erhn = eff_request_host(request)
- domain = erhn
- elif not startswith(domain, "."):
- domain = "."+domain
-
- # set default port
- port_specified = False
- if port is not Absent:
- if port is None:
- # Port attr present, but has no value: default to request port.
- # Cookie should then only be sent back on that port.
- port = request_port(request)
- else:
- port_specified = True
- port = re.sub(r"\s+", "", port)
- else:
- # No port attr present. Cookie can be sent back on any port.
- port = None
-
- # set default expires and discard
- if expires is Absent:
- expires = None
- discard = True
- elif expires <= self._now:
- # Expiry date in past is request to delete cookie. This can't be
- # in DefaultCookiePolicy, because can't delete cookies there.
- try:
- self.clear(domain, path, name)
- except KeyError:
- pass
- debug("Expiring cookie, domain='%s', path='%s', name='%s'",
- domain, path, name)
- return None
-
- return Cookie(version,
- name, value,
- port, port_specified,
- domain, domain_specified, domain_initial_dot,
- path, path_specified,
- secure,
- expires,
- discard,
- comment,
- comment_url,
- rest)
-
- def _cookies_from_attrs_set(self, attrs_set, request):
- cookie_tuples = self._normalized_cookie_tuples(attrs_set)
-
- cookies = []
- for tup in cookie_tuples:
- cookie = self._cookie_from_cookie_tuple(tup, request)
- if cookie: cookies.append(cookie)
- return cookies
-
- def _process_rfc2109_cookies(self, cookies):
- if self._policy.rfc2109_as_netscape is None:
- rfc2109_as_netscape = not self._policy.rfc2965
- else:
- rfc2109_as_netscape = self._policy.rfc2109_as_netscape
- for cookie in cookies:
- if cookie.version == 1:
- cookie.rfc2109 = True
- if rfc2109_as_netscape:
- # treat 2109 cookies as Netscape cookies rather than
- # as RFC2965 cookies
- cookie.version = 0
-
- def make_cookies(self, response, request):
- """Return sequence of Cookie objects extracted from response object.
-
- See extract_cookies.__doc__ for the interfaces required of the
- response and request arguments.
-
- """
- # get cookie-attributes for RFC 2965 and Netscape protocols
- headers = response.info()
- rfc2965_hdrs = getheaders(headers, "Set-Cookie2")
- ns_hdrs = getheaders(headers, "Set-Cookie")
-
- rfc2965 = self._policy.rfc2965
- netscape = self._policy.netscape
-
- if ((not rfc2965_hdrs and not ns_hdrs) or
- (not ns_hdrs and not rfc2965) or
- (not rfc2965_hdrs and not netscape) or
- (not netscape and not rfc2965)):
- return [] # no relevant cookie headers: quick exit
-
- try:
- cookies = self._cookies_from_attrs_set(
- split_header_words(rfc2965_hdrs), request)
- except:
- reraise_unmasked_exceptions()
- cookies = []
-
- if ns_hdrs and netscape:
- try:
- # RFC 2109 and Netscape cookies
- ns_cookies = self._cookies_from_attrs_set(
- parse_ns_headers(ns_hdrs), request)
- except:
- reraise_unmasked_exceptions()
- ns_cookies = []
- self._process_rfc2109_cookies(ns_cookies)
-
- # Look for Netscape cookies (from Set-Cookie headers) that match
- # corresponding RFC 2965 cookies (from Set-Cookie2 headers).
- # For each match, keep the RFC 2965 cookie and ignore the Netscape
- # cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are
- # bundled in with the Netscape cookies for this purpose, which is
- # reasonable behaviour.
- if rfc2965:
- lookup = {}
- for cookie in cookies:
- lookup[(cookie.domain, cookie.path, cookie.name)] = None
-
- def no_matching_rfc2965(ns_cookie, lookup=lookup):
- key = ns_cookie.domain, ns_cookie.path, ns_cookie.name
- return not lookup.has_key(key)
- ns_cookies = filter(no_matching_rfc2965, ns_cookies)
-
- if ns_cookies:
- cookies.extend(ns_cookies)
-
- return cookies
-
- def set_cookie_if_ok(self, cookie, request):
- """Set a cookie if policy says it's OK to do so.
-
- cookie: mechanize.Cookie instance
- request: see extract_cookies.__doc__ for the required interface
-
- """
- self._policy._now = self._now = int(time.time())
-
- if self._policy.set_ok(cookie, request):
- self.set_cookie(cookie)
-
- def set_cookie(self, cookie):
- """Set a cookie, without checking whether or not it should be set.
-
- cookie: mechanize.Cookie instance
- """
- c = self._cookies
- if not c.has_key(cookie.domain): c[cookie.domain] = {}
- c2 = c[cookie.domain]
- if not c2.has_key(cookie.path): c2[cookie.path] = {}
- c3 = c2[cookie.path]
- c3[cookie.name] = cookie
-
- def extract_cookies(self, response, request):
- """Extract cookies from response, where allowable given the request.
-
- Look for allowable Set-Cookie: and Set-Cookie2: headers in the response
- object passed as argument. Any of these headers that are found are
- used to update the state of the object (subject to the policy.set_ok
- method's approval).
-
- The response object (usually be the result of a call to
- mechanize.urlopen, or similar) should support an info method, which
- returns a mimetools.Message object (in fact, the 'mimetools.Message
- object' may be any object that provides a getallmatchingheaders
- method).
-
- The request object (usually a urllib2.Request instance) must support
- the methods get_full_url and get_host, as documented by urllib2, and
- the port attribute (the port number). The request is used to set
- default values for cookie-attributes as well as for checking that the
- cookie is OK to be set.
-
- """
- debug("extract_cookies: %s", response.info())
- self._policy._now = self._now = int(time.time())
-
- for cookie in self.make_cookies(response, request):
- if self._policy.set_ok(cookie, request):
- debug(" setting cookie: %s", cookie)
- self.set_cookie(cookie)
-
- def clear(self, domain=None, path=None, name=None):
- """Clear some cookies.
-
- Invoking this method without arguments will clear all cookies. If
- given a single argument, only cookies belonging to that domain will be
- removed. If given two arguments, cookies belonging to the specified
- path within that domain are removed. If given three arguments, then
- the cookie with the specified name, path and domain is removed.
-
- Raises KeyError if no matching cookie exists.
-
- """
- if name is not None:
- if (domain is None) or (path is None):
- raise ValueError(
- "domain and path must be given to remove a cookie by name")
- del self._cookies[domain][path][name]
- elif path is not None:
- if domain is None:
- raise ValueError(
- "domain must be given to remove cookies by path")
- del self._cookies[domain][path]
- elif domain is not None:
- del self._cookies[domain]
- else:
- self._cookies = {}
-
- def clear_session_cookies(self):
- """Discard all session cookies.
-
- Discards all cookies held by object which had either no Max-Age or
- Expires cookie-attribute or an explicit Discard cookie-attribute, or
- which otherwise have ended up with a true discard attribute. For
- interactive browsers, the end of a session usually corresponds to
- closing the browser window.
-
- Note that the save method won't save session cookies anyway, unless you
- ask otherwise by passing a true ignore_discard argument.
-
- """
- for cookie in self:
- if cookie.discard:
- self.clear(cookie.domain, cookie.path, cookie.name)
-
- def clear_expired_cookies(self):
- """Discard all expired cookies.
-
- You probably don't need to call this method: expired cookies are never
- sent back to the server (provided you're using DefaultCookiePolicy),
- this method is called by CookieJar itself every so often, and the save
- method won't save expired cookies anyway (unless you ask otherwise by
- passing a true ignore_expires argument).
-
- """
- now = time.time()
- for cookie in self:
- if cookie.is_expired(now):
- self.clear(cookie.domain, cookie.path, cookie.name)
-
- def __getitem__(self, i):
- if i == 0:
- self._getitem_iterator = self.__iter__()
- elif self._prev_getitem_index != i-1: raise IndexError(
- "CookieJar.__getitem__ only supports sequential iteration")
- self._prev_getitem_index = i
- try:
- return self._getitem_iterator.next()
- except StopIteration:
- raise IndexError()
-
- def __iter__(self):
- return MappingIterator(self._cookies)
-
- def __len__(self):
- """Return number of contained cookies."""
- i = 0
- for cookie in self: i = i + 1
- return i
-
- def __repr__(self):
- r = []
- for cookie in self: r.append(repr(cookie))
- return "<%s[%s]>" % (self.__class__, string.join(r, ", "))
-
- def __str__(self):
- r = []
- for cookie in self: r.append(str(cookie))
- return "<%s[%s]>" % (self.__class__, string.join(r, ", "))
-
-
-class LoadError(Exception): pass
-
-class FileCookieJar(CookieJar):
- """CookieJar that can be loaded from and saved to a file.
-
- Additional methods
-
- save(filename=None, ignore_discard=False, ignore_expires=False)
- load(filename=None, ignore_discard=False, ignore_expires=False)
- revert(filename=None, ignore_discard=False, ignore_expires=False)
-
- Additional public attributes
-
- filename: filename for loading and saving cookies
-
- Additional public readable attributes
-
- delayload: request that cookies are lazily loaded from disk; this is only
- a hint since this only affects performance, not behaviour (unless the
- cookies on disk are changing); a CookieJar object may ignore it (in fact,
- only MSIECookieJar lazily loads cookies at the moment)
-
- """
-
- def __init__(self, filename=None, delayload=False, policy=None):
- """
- See FileCookieJar.__doc__ for argument documentation.
-
- Cookies are NOT loaded from the named file until either the load or
- revert method is called.
-
- """
- CookieJar.__init__(self, policy)
- if filename is not None and not isstringlike(filename):
- raise ValueError("filename must be string-like")
- self.filename = filename
- self.delayload = bool(delayload)
-
- def save(self, filename=None, ignore_discard=False, ignore_expires=False):
- """Save cookies to a file.
-
- filename: name of file in which to save cookies
- ignore_discard: save even cookies set to be discarded
- ignore_expires: save even cookies that have expired
-
- The file is overwritten if it already exists, thus wiping all its
- cookies. Saved cookies can be restored later using the load or revert
- methods. If filename is not specified, self.filename is used; if
- self.filename is None, ValueError is raised.
-
- """
- raise NotImplementedError()
-
- def load(self, filename=None, ignore_discard=False, ignore_expires=False):
- """Load cookies from a file.
-
- Old cookies are kept unless overwritten by newly loaded ones.
-
- Arguments are as for .save().
-
- If filename is not specified, self.filename is used; if self.filename
- is None, ValueError is raised. The named file must be in the format
- understood by the class, or LoadError will be raised. This format will
- be identical to that written by the save method, unless the load format
- is not sufficiently well understood (as is the case for MSIECookieJar).
-
- """
- if filename is None:
- if self.filename is not None: filename = self.filename
- else: raise ValueError(MISSING_FILENAME_TEXT)
-
- f = open(filename)
- try:
- self._really_load(f, filename, ignore_discard, ignore_expires)
- finally:
- f.close()
-
- def revert(self, filename=None,
- ignore_discard=False, ignore_expires=False):
- """Clear all cookies and reload cookies from a saved file.
-
- Raises LoadError (or IOError) if reversion is not successful; the
- object's state will not be altered if this happens.
-
- """
- if filename is None:
- if self.filename is not None: filename = self.filename
- else: raise ValueError(MISSING_FILENAME_TEXT)
-
- old_state = copy.deepcopy(self._cookies)
- self._cookies = {}
- try:
- self.load(filename, ignore_discard, ignore_expires)
- except (LoadError, IOError):
- self._cookies = old_state
- raise
Deleted: /wwwsearch/mechanize/trunk/mechanize/_HeadersUtil.py
==============================================================================
--- /wwwsearch/mechanize/trunk/mechanize/_HeadersUtil.py Sun May 21 21:32:06 2006
+++ (empty file)
@@ -1,225 +0,0 @@
-"""Utility functions for HTTP header value parsing and construction.
-
-Copyright 1997-1998, Gisle Aas
-Copyright 2002-2006, John J. Lee
-
-This code is free software; you can redistribute it and/or modify it
-under the terms of the BSD or ZPL 2.1 licenses (see the file
-COPYING.txt included with the distribution).
-
-"""
-
-import os, re, string, urlparse
-from types import StringType
-from types import UnicodeType
-STRING_TYPES = StringType, UnicodeType
-
-from _Util import startswith, endswith, http2time
-
-def is_html(ct_headers, url, allow_xhtml=False):
- """
- ct_headers: Sequence of Content-Type headers
- url: Response URL
-
- """
- if not ct_headers:
- # guess
- ext = os.path.splitext(urlparse.urlparse(url)[2])[1]
- html_exts = [".htm", ".html"]
- if allow_xhtml:
- html_exts += [".xhtml"]
- return ext in html_exts
- # use first header
- ct = split_header_words(ct_headers)[0][0][0]
- html_types = ["text/html"]
- if allow_xhtml:
- html_types += [
- "text/xhtml", "text/xml",
- "application/xml", "application/xhtml+xml",
- ]
- return ct in html_types
-
-def unmatched(match):
- """Return unmatched part of re.Match object."""
- start, end = match.span(0)
- return match.string[:start]+match.string[end:]
-
-token_re = re.compile(r"^\s*([^=\s;,]+)")
-quoted_value_re = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"")
-value_re = re.compile(r"^\s*=\s*([^\s;,]*)")
-escape_re = re.compile(r"\\(.)")
-def split_header_words(header_values):
- r"""Parse header values into a list of lists containing key,value pairs.
-
- The function knows how to deal with ",", ";" and "=" as well as quoted
- values after "=". A list of space separated tokens are parsed as if they
- were separated by ";".
-
- If the header_values passed as argument contains multiple values, then they
- are treated as if they were a single value separated by comma ",".
-
- This means that this function is useful for parsing header fields that
- follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
- the requirement for tokens).
-
- headers = #header
- header = (token | parameter) *( [";"] (token | parameter))
-
- token = 1*<any CHAR except CTLs or separators>
- separators = "(" | ")" | "<" | ">" | "@"
- | "," | ";" | ":" | "\" | <">
- | "/" | "[" | "]" | "?" | "="
- | "{" | "}" | SP | HT
-
- quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
- qdtext = <any TEXT except <">>
- quoted-pair = "\" CHAR
-
- parameter = attribute "=" value
- attribute = token
- value = token | quoted-string
-
- Each header is represented by a list of key/value pairs. The value for a
- simple token (not part of a parameter) is None. Syntactically incorrect
- headers will not necessarily be parsed as you would want.
-
- This is easier to describe with some examples:
-
- >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
- [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
- >>> split_header_words(['text/html; charset="iso-8859-1"'])
- [[('text/html', None), ('charset', 'iso-8859-1')]]
- >>> split_header_words([r'Basic realm="\"foo\bar\""'])
- [[('Basic', None), ('realm', '"foobar"')]]
-
- """
- assert type(header_values) not in STRING_TYPES
- result = []
- for text in header_values:
- orig_text = text
- pairs = []
- while text:
- m = token_re.search(text)
- if m:
- text = unmatched(m)
- name = m.group(1)
- m = quoted_value_re.search(text)
- if m: # quoted value
- text = unmatched(m)
- value = m.group(1)
- value = escape_re.sub(r"\1", value)
- else:
- m = value_re.search(text)
- if m: # unquoted value
- text = unmatched(m)
- value = m.group(1)
- value = string.rstrip(value)
- else:
- # no value, a lone token
- value = None
- pairs.append((name, value))
- elif startswith(string.lstrip(text), ","):
- # concatenated headers, as per RFC 2616 section 4.2
- text = string.lstrip(text)[1:]
- if pairs: result.append(pairs)
- pairs = []
- else:
- # skip junk
- non_junk, nr_junk_chars = re.subn("^[=\s;]*", "", text)
- assert nr_junk_chars > 0, (
- "split_header_words bug: '%s', '%s', %s" %
- (orig_text, text, pairs))
- text = non_junk
- if pairs: result.append(pairs)
- return result
-
-join_escape_re = re.compile(r"([\"\\])")
-def join_header_words(lists):
- """Do the inverse of the conversion done by split_header_words.
-
- Takes a list of lists of (key, value) pairs and produces a single header
- value. Attribute values are quoted if needed.
-
- >>> join_header_words([[("text/plain", None), ("charset", "iso-8859/1")]])
- 'text/plain; charset="iso-8859/1"'
- >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859/1")]])
- 'text/plain, charset="iso-8859/1"'
-
- """
- headers = []
- for pairs in lists:
- attr = []
- for k, v in pairs:
- if v is not None:
- if not re.search(r"^\w+$", v):
- v = join_escape_re.sub(r"\\\1", v) # escape " and \
- v = '"%s"' % v
- if k is None: # Netscape cookies may have no name
- k = v
- else:
- k = "%s=%s" % (k, v)
- attr.append(k)
- if attr: headers.append(string.join(attr, "; "))
- return string.join(headers, ", ")
-
-def parse_ns_headers(ns_headers):
- """Ad-hoc parser for Netscape protocol cookie-attributes.
-
- The old Netscape cookie format for Set-Cookie can for instance contain
- an unquoted "," in the expires field, so we have to use this ad-hoc
- parser instead of split_header_words.
-
- XXX This may not make the best possible effort to parse all the crap
- that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
- parser is probably better, so could do worse than following that if
- this ever gives any trouble.
-
- Currently, this is also used for parsing RFC 2109 cookies.
-
- """
- known_attrs = ("expires", "domain", "path", "secure",
- # RFC 2109 attrs (may turn up in Netscape cookies, too)
- "port", "max-age")
-
- result = []
- for ns_header in ns_headers:
- pairs = []
- version_set = False
- params = re.split(r";\s*", ns_header)
- for ii in range(len(params)):
- param = params[ii]
- param = string.rstrip(param)
- if param == "": continue
- if "=" not in param:
- k, v = param, None
- else:
- k, v = re.split(r"\s*=\s*", param, 1)
- k = string.lstrip(k)
- if ii != 0:
- lc = string.lower(k)
- if lc in known_attrs:
- k = lc
- if k == "version":
- # This is an RFC 2109 cookie.
- version_set = True
- if k == "expires":
- # convert expires date to seconds since epoch
- if startswith(v, '"'): v = v[1:]
- if endswith(v, '"'): v = v[:-1]
- v = http2time(v) # None if invalid
- pairs.append((k, v))
-
- if pairs:
- if not version_set:
- pairs.append(("version", "0"))
- result.append(pairs)
-
- return result
-
-
-def _test():
- import doctest, _HeadersUtil
- return doctest.testmod(_HeadersUtil)
-
-if __name__ == "__main__":
- _test()
Deleted: /wwwsearch/mechanize/trunk/mechanize/_MSIECookieJar.py
==============================================================================
--- /wwwsearch/mechanize/trunk/mechanize/_MSIECookieJar.py Sun May 21 21:32:06 2006
+++ (empty file)
@@ -1,388 +0,0 @@
-"""Microsoft Internet Explorer cookie loading on Windows.
-
-Copyright 2002-2003 Johnny Lee <typo_pl at hotmail.com> (MSIE Perl code)
-Copyright 2002-2006 John J Lee <jjl at pobox.com> (The Python port)
-
-This code is free software; you can redistribute it and/or modify it
-under the terms of the BSD or ZPL 2.1 licenses (see the file
-COPYING.txt included with the distribution).
-
-"""
-
-# XXX names and comments are not great here
-
-import os, re, string, time, struct, logging
-if os.name == "nt":
- import _winreg
-
-from _ClientCookie import FileCookieJar, CookieJar, Cookie, \
- MISSING_FILENAME_TEXT, LoadError
-from _Util import startswith
-
-debug = logging.getLogger("mechanize").debug
-
-
-def regload(path, leaf):
- key = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, path, 0,
- _winreg.KEY_ALL_ACCESS)
- try:
- value = _winreg.QueryValueEx(key, leaf)[0]
- except WindowsError:
- value = None
- return value
-
-WIN32_EPOCH = 0x019db1ded53e8000L # 1970 Jan 01 00:00:00 in Win32 FILETIME
-
-def epoch_time_offset_from_win32_filetime(filetime):
- """Convert from win32 filetime to seconds-since-epoch value.
-
- MSIE stores create and expire times as Win32 FILETIME, which is 64
- bits of 100 nanosecond intervals since Jan 01 1601.
-
- mechanize expects time in 32-bit value expressed in seconds since the
- epoch (Jan 01 1970).
-
- """
- if filetime < WIN32_EPOCH:
- raise ValueError("filetime (%d) is before epoch (%d)" %
- (filetime, WIN32_EPOCH))
-
- return divmod((filetime - WIN32_EPOCH), 10000000L)[0]
-
-def binary_to_char(c): return "%02X" % ord(c)
-def binary_to_str(d): return string.join(map(binary_to_char, list(d)), "")
-
-class MSIEBase:
- magic_re = re.compile(r"Client UrlCache MMF Ver \d\.\d.*")
- padding = "\x0d\xf0\xad\x0b"
-
- msie_domain_re = re.compile(r"^([^/]+)(/.*)$")
- cookie_re = re.compile("Cookie\:.+\@([\x21-\xFF]+).*?"
- "(.+\@[\x21-\xFF]+\.txt)")
-
- # path under HKEY_CURRENT_USER from which to get location of index.dat
- reg_path = r"software\microsoft\windows" \
- r"\currentversion\explorer\shell folders"
- reg_key = "Cookies"
-
- def __init__(self):
- self._delayload_domains = {}
-
- def _delayload_domain(self, domain):
- # if necessary, lazily load cookies for this domain
- delayload_info = self._delayload_domains.get(domain)
- if delayload_info is not None:
- cookie_file, ignore_discard, ignore_expires = delayload_info
- try:
- self.load_cookie_data(cookie_file,
- ignore_discard, ignore_expires)
- except (LoadError, IOError):
- debug("error reading cookie file, skipping: %s", cookie_file)
- else:
- del self._delayload_domains[domain]
-
- def _load_cookies_from_file(self, filename):
- debug("Loading MSIE cookies file: %s", filename)
- cookies = []
-
- cookies_fh = open(filename)
-
- try:
- while 1:
- key = cookies_fh.readline()
- if key == "": break
-
- rl = cookies_fh.readline
- def getlong(rl=rl): return long(rl().rstrip())
- def getstr(rl=rl): return rl().rstrip()
-
- key = key.rstrip()
- value = getstr()
- domain_path = getstr()
- flags = getlong() # 0x2000 bit is for secure I think
- lo_expire = getlong()
- hi_expire = getlong()
- lo_create = getlong()
- hi_create = getlong()
- sep = getstr()
-
- if "" in (key, value, domain_path, flags, hi_expire, lo_expire,
- hi_create, lo_create, sep) or (sep != "*"):
- break
-
- m = self.msie_domain_re.search(domain_path)
- if m:
- domain = m.group(1)
- path = m.group(2)
-
- cookies.append({"KEY": key, "VALUE": value, "DOMAIN": domain,
- "PATH": path, "FLAGS": flags, "HIXP": hi_expire,
- "LOXP": lo_expire, "HICREATE": hi_create,
- "LOCREATE": lo_create})
- finally:
- cookies_fh.close()
-
- return cookies
-
- def load_cookie_data(self, filename,
- ignore_discard=False, ignore_expires=False):
- """Load cookies from file containing actual cookie data.
-
- Old cookies are kept unless overwritten by newly loaded ones.
-
- You should not call this method if the delayload attribute is set.
-
- I think each of these files contain all cookies for one user, domain,
- and path.
-
- filename: file containing cookies -- usually found in a file like
- C:\WINNT\Profiles\joe\Cookies\joe at blah[1].txt
-
- """
- now = int(time.time())
-
- cookie_data = self._load_cookies_from_file(filename)
-
- for cookie in cookie_data:
- flags = cookie["FLAGS"]
- secure = ((flags & 0x2000) != 0)
- filetime = (cookie["HIXP"] << 32) + cookie["LOXP"]
- expires = epoch_time_offset_from_win32_filetime(filetime)
- if expires < now:
- discard = True
- else:
- discard = False
- domain = cookie["DOMAIN"]
- initial_dot = startswith(domain, ".")
- if initial_dot:
- domain_specified = True
- else:
- # MSIE 5 does not record whether the domain cookie-attribute
- # was specified.
- # Assuming it wasn't is conservative, because with strict
- # domain matching this will match less frequently; with regular
- # Netscape tail-matching, this will match at exactly the same
- # times that domain_specified = True would. It also means we
- # don't have to prepend a dot to achieve consistency with our
- # own & Mozilla's domain-munging scheme.
- domain_specified = False
-
- # assume path_specified is false
- # XXX is there other stuff in here? -- eg. comment, commentURL?
- c = Cookie(0,
- cookie["KEY"], cookie["VALUE"],
- None, False,
- domain, domain_specified, initial_dot,
- cookie["PATH"], False,
- secure,
- expires,
- discard,
- None,
- None,
- {"flags": flags})
- if not ignore_discard and c.discard:
- continue
- if not ignore_expires and c.is_expired(now):
- continue
- CookieJar.set_cookie(self, c)
-
- def load_from_registry(self, ignore_discard=False, ignore_expires=False,
- username=None):
- """
- username: only required on win9x
-
- """
- cookies_dir = regload(self.reg_path, self.reg_key)
- filename = os.path.normpath(os.path.join(cookies_dir, "INDEX.DAT"))
- self.load(filename, ignore_discard, ignore_expires, username)
-
- def _really_load(self, index, filename, ignore_discard, ignore_expires,
- username):
- now = int(time.time())
-
- if username is None:
- username = string.lower(os.environ['USERNAME'])
-
- cookie_dir = os.path.dirname(filename)
-
- data = index.read(256)
- if len(data) != 256:
- raise LoadError("%s file is too short" % filename)
-
- # Cookies' index.dat file starts with 32 bytes of signature
- # followed by an offset to the first record, stored as a little-
- # endian DWORD.
- sig, size, data = data[:32], data[32:36], data[36:]
- size = struct.unpack("<L", size)[0]
-
- # check that sig is valid
- if not self.magic_re.match(sig) or size != 0x4000:
- raise LoadError("%s ['%s' %s] does not seem to contain cookies" %
- (str(filename), sig, size))
-
- # skip to start of first record
- index.seek(size, 0)
-
- sector = 128 # size of sector in bytes
-
- while 1:
- data = ""
-
- # Cookies are usually in two contiguous sectors, so read in two
- # sectors and adjust if not a Cookie.
- to_read = 2 * sector
- d = index.read(to_read)
- if len(d) != to_read:
- break
- data = data + d
-
- # Each record starts with a 4-byte signature and a count
- # (little-endian DWORD) of sectors for the record.
- sig, size, data = data[:4], data[4:8], data[8:]
- size = struct.unpack("<L", size)[0]
-
- to_read = (size - 2) * sector
-
-## from urllib import quote
-## print "data", quote(data)
-## print "sig", quote(sig)
-## print "size in sectors", size
-## print "size in bytes", size*sector
-## print "size in units of 16 bytes", (size*sector) / 16
-## print "size to read in bytes", to_read
-## print
-
- if sig != "URL ":
- assert (sig in ("HASH", "LEAK",
- self.padding, "\x00\x00\x00\x00"),
- "unrecognized MSIE index.dat record: %s" %
- binary_to_str(sig))
- if sig == "\x00\x00\x00\x00":
- # assume we've got all the cookies, and stop
- break
- if sig == self.padding:
- continue
- # skip the rest of this record
- assert to_read >= 0
- if size != 2:
- assert to_read != 0
- index.seek(to_read, 1)
- continue
-
- # read in rest of record if necessary
- if size > 2:
- more_data = index.read(to_read)
- if len(more_data) != to_read: break
- data = data + more_data
-
- cookie_re = ("Cookie\:%s\@([\x21-\xFF]+).*?" % username +
- "(%s\@[\x21-\xFF]+\.txt)" % username)
- m = re.search(cookie_re, data, re.I)
- if m:
- cookie_file = os.path.join(cookie_dir, m.group(2))
- if not self.delayload:
- try:
- self.load_cookie_data(cookie_file,
- ignore_discard, ignore_expires)
- except (LoadError, IOError):
- debug("error reading cookie file, skipping: %s",
- cookie_file)
- else:
- domain = m.group(1)
- i = domain.find("/")
- if i != -1:
- domain = domain[:i]
-
- self._delayload_domains[domain] = (
- cookie_file, ignore_discard, ignore_expires)
-
-
-class MSIECookieJar(MSIEBase, FileCookieJar):
- """FileCookieJar that reads from the Windows MSIE cookies database.
-
- MSIECookieJar can read the cookie files of Microsoft Internet Explorer
- (MSIE) for Windows version 5 on Windows NT and version 6 on Windows XP and
- Windows 98. Other configurations may also work, but are untested. Saving
- cookies in MSIE format is NOT supported. If you save cookies, they'll be
- in the usual Set-Cookie3 format, which you can read back in using an
- instance of the plain old CookieJar class. Don't save using the same
- filename that you loaded cookies from, because you may succeed in
- clobbering your MSIE cookies index file!
-
- You should be able to have LWP share Internet Explorer's cookies like
- this (note you need to supply a username to load_from_registry if you're on
- Windows 9x or Windows ME):
-
- cj = MSIECookieJar(delayload=1)
- # find cookies index file in registry and load cookies from it
- cj.load_from_registry()
- opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cj))
- response = opener.open("http://example.com/")
-
- Iterating over a delayloaded MSIECookieJar instance will not cause any
- cookies to be read from disk. To force reading of all cookies from disk,
- call read_all_cookies. Note that the following methods iterate over self:
- clear_temporary_cookies, clear_expired_cookies, __len__, __repr__, __str__
- and as_string.
-
- Additional methods:
-
- load_from_registry(ignore_discard=False, ignore_expires=False,
- username=None)
- load_cookie_data(filename, ignore_discard=False, ignore_expires=False)
- read_all_cookies()
-
- """
- def __init__(self, filename=None, delayload=False, policy=None):
- MSIEBase.__init__(self)
- FileCookieJar.__init__(self, filename, delayload, policy)
-
- def set_cookie(self, cookie):
- if self.delayload:
- self._delayload_domain(cookie.domain)
- CookieJar.set_cookie(self, cookie)
-
- def _cookies_for_request(self, request):
- """Return a list of cookies to be returned to server."""
- domains = self._cookies.copy()
- domains.update(self._delayload_domains)
- domains = domains.keys()
-
- cookies = []
- for domain in domains:
- cookies.extend(self._cookies_for_domain(domain, request))
- return cookies
-
- def _cookies_for_domain(self, domain, request):
- if not self._policy.domain_return_ok(domain, request):
- return []
- debug("Checking %s for cookies to return", domain)
- if self.delayload:
- self._delayload_domain(domain)
- return CookieJar._cookies_for_domain(self, domain, request)
-
- def read_all_cookies(self):
- """Eagerly read in all cookies."""
- if self.delayload:
- for domain in self._delayload_domains.keys():
- self._delayload_domain(domain)
-
- def load(self, filename, ignore_discard=False, ignore_expires=False,
- username=None):
- """Load cookies from an MSIE 'index.dat' cookies index file.
-
- filename: full path to cookie index file
- username: only required on win9x
-
- """
- if filename is None:
- if self.filename is not None: filename = self.filename
- else: raise ValueError(MISSING_FILENAME_TEXT)
-
- index = open(filename, "rb")
-
- try:
- self._really_load(index, filename, ignore_discard, ignore_expires,
- username)
- finally:
- index.close()
Deleted: /wwwsearch/mechanize/trunk/mechanize/_MozillaCookieJar.py
==============================================================================
--- /wwwsearch/mechanize/trunk/mechanize/_MozillaCookieJar.py Sun May 21 21:32:06 2006
+++ (empty file)
@@ -1,160 +0,0 @@
-"""Mozilla / Netscape cookie loading / saving.
-
-Copyright 2002-2006 John J Lee <jjl at pobox.com>
-Copyright 1997-1999 Gisle Aas (original libwww-perl code)
-
-This code is free software; you can redistribute it and/or modify it
-under the terms of the BSD or ZPL 2.1 licenses (see the file
-COPYING.txt included with the distribution).
-
-"""
-
-import re, string, time, logging
-
-from _ClientCookie import reraise_unmasked_exceptions, FileCookieJar, Cookie, \
- MISSING_FILENAME_TEXT, LoadError
-from _Util import startswith, endswith
-debug = logging.getLogger("ClientCookie").debug
-
-
-class MozillaCookieJar(FileCookieJar):
- """
-
- WARNING: you may want to backup your browser's cookies file if you use
- this class to save cookies. I *think* it works, but there have been
- bugs in the past!
-
- This class differs from CookieJar only in the format it uses to save and
- load cookies to and from a file. This class uses the Mozilla/Netscape
- `cookies.txt' format. lynx uses this file format, too.
-
- Don't expect cookies saved while the browser is running to be noticed by
- the browser (in fact, Mozilla on unix will overwrite your saved cookies if
- you change them on disk while it's running; on Windows, you probably can't
- save at all while the browser is running).
-
- Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
- Netscape cookies on saving.
-
- In particular, the cookie version and port number information is lost,
- together with information about whether or not Path, Port and Discard were
- specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
- domain as set in the HTTP header started with a dot (yes, I'm aware some
- domains in Netscape files start with a dot and some don't -- trust me, you
- really don't want to know any more about this).
-
- Note that though Mozilla and Netscape use the same format, they use
- slightly different headers. The class saves cookies using the Netscape
- header by default (Mozilla can cope with that).
-
- """
- magic_re = "#( Netscape)? HTTP Cookie File"
- header = """\
- # Netscape HTTP Cookie File
- # http://www.netscape.com/newsref/std/cookie_spec.html
- # This is a generated file! Do not edit.
-
-"""
-
- def _really_load(self, f, filename, ignore_discard, ignore_expires):
- now = time.time()
-
- magic = f.readline()
- if not re.search(self.magic_re, magic):
- f.close()
- raise LoadError(
- "%s does not look like a Netscape format cookies file" %
- filename)
-
- try:
- while 1:
- line = f.readline()
- if line == "": break
-
- # last field may be absent, so keep any trailing tab
- if endswith(line, "\n"): line = line[:-1]
-
- # skip comments and blank lines XXX what is $ for?
- if (startswith(string.strip(line), "#") or
- startswith(string.strip(line), "$") or
- string.strip(line) == ""):
- continue
-
- domain, domain_specified, path, secure, expires, name, value = \
- string.split(line, "\t")
- secure = (secure == "TRUE")
- domain_specified = (domain_specified == "TRUE")
- if name == "":
- name = value
- value = None
-
- initial_dot = startswith(domain, ".")
- assert domain_specified == initial_dot
-
- discard = False
- if expires == "":
- expires = None
- discard = True
-
- # assume path_specified is false
- c = Cookie(0, name, value,
- None, False,
- domain, domain_specified, initial_dot,
- path, False,
- secure,
- expires,
- discard,
- None,
- None,
- {})
- if not ignore_discard and c.discard:
- continue
- if not ignore_expires and c.is_expired(now):
- continue
- self.set_cookie(c)
-
- except:
- reraise_unmasked_exceptions((IOError,))
- raise LoadError("invalid Netscape format file %s: %s" %
- (filename, line))
-
- def save(self, filename=None, ignore_discard=False, ignore_expires=False):
- if filename is None:
- if self.filename is not None: filename = self.filename
- else: raise ValueError(MISSING_FILENAME_TEXT)
-
- f = open(filename, "w")
- try:
- debug("Saving Netscape cookies.txt file")
- f.write(self.header)
- now = time.time()
- for cookie in self:
- if not ignore_discard and cookie.discard:
- debug(" Not saving %s: marked for discard", cookie.name)
- continue
- if not ignore_expires and cookie.is_expired(now):
- debug(" Not saving %s: expired", cookie.name)
- continue
- if cookie.secure: secure = "TRUE"
- else: secure = "FALSE"
- if startswith(cookie.domain, "."): initial_dot = "TRUE"
- else: initial_dot = "FALSE"
- if cookie.expires is not None:
- expires = str(cookie.expires)
- else:
- expires = ""
- if cookie.value is None:
- # cookies.txt regards 'Set-Cookie: foo' as a cookie
- # with no name, whereas cookielib regards it as a
- # cookie with no value.
- name = ""
- value = cookie.name
- else:
- name = cookie.name
- value = cookie.value
- f.write(
- string.join([cookie.domain, initial_dot, cookie.path,
- secure, expires, name, value], "\t")+
- "\n")
- finally:
- f.close()
Deleted: /wwwsearch/mechanize/trunk/mechanize/_Opener.py
==============================================================================
--- /wwwsearch/mechanize/trunk/mechanize/_Opener.py Sun May 21 21:32:06 2006
+++ (empty file)
@@ -1,267 +0,0 @@
-"""Integration with Python standard library module urllib2: OpenerDirector
-class.
-
-Copyright 2004-2006 John J Lee <jjl at pobox.com>
-
-This code is free software; you can redistribute it and/or modify it
-under the terms of the BSD or ZPL 2.1 licenses (see the file
-COPYING.txt included with the distribution).
-
-"""
-
-import urllib2, string, bisect, urlparse
-
-from _Util import startswith, isstringlike
-from _Request import Request
-
-try:
- set
-except NameError:
- import sets
- set = sets.Set
-
-def methnames(obj):
- """Return method names of class instance.
-
- dir(obj) doesn't work across Python versions, this does.
-
- """
- return methnames_of_instance_as_dict(obj).keys()
-
-def methnames_of_instance_as_dict(inst):
- names = {}
- names.update(methnames_of_class_as_dict(inst.__class__))
- for methname in dir(inst):
- candidate = getattr(inst, methname)
- if callable(candidate):
- names[methname] = None
- return names
-
-def methnames_of_class_as_dict(klass):
- names = {}
- for methname in dir(klass):
- candidate = getattr(klass, methname)
- if callable(candidate):
- names[methname] = None
- for baseclass in klass.__bases__:
- names.update(methnames_of_class_as_dict(baseclass))
- return names
-
-
-class OpenerDirector(urllib2.OpenerDirector):
- def __init__(self):
- urllib2.OpenerDirector.__init__(self)
- # really none of these are (sanely) public -- the lack of initial
- # underscore on some is just due to following urllib2
- self.process_response = {}
- self.process_request = {}
- self._any_request = {}
- self._any_response = {}
- self._handler_index_valid = True
-
- def add_handler(self, handler):
- if handler in self.handlers:
- return
- # XXX why does self.handlers need to be sorted?
- bisect.insort(self.handlers, handler)
- handler.add_parent(self)
- self._handler_index_valid = False
-
- def _maybe_reindex_handlers(self):
- if self._handler_index_valid:
- return
-
- handle_error = {}
- handle_open = {}
- process_request = {}
- process_response = {}
- any_request = set()
- any_response = set()
- unwanted = []
-
- for handler in self.handlers:
- added = False
- for meth in methnames(handler):
- if meth in ["redirect_request", "do_open", "proxy_open"]:
- # oops, coincidental match
- continue
-
- if meth == "any_request":
- any_request.add(handler)
- added = True
- continue
- elif meth == "any_response":
- any_response.add(handler)
- added = True
- continue
-
- ii = meth.find("_")
- scheme = meth[:ii]
- condition = meth[ii+1:]
-
- if startswith(condition, "error"):
- jj = string.find(meth[ii+1:], "_") + ii + 1
- kind = meth[jj+1:]
- try:
- kind = int(kind)
- except ValueError:
- pass
- lookup = handle_error.setdefault(scheme, {})
- elif condition == "open":
- kind = scheme
- lookup = handle_open
- elif condition == "request":
- kind = scheme
- lookup = process_request
- elif condition == "response":
- kind = scheme
- lookup = process_response
- else:
- continue
-
- lookup.setdefault(kind, set()).add(handler)
- added = True
-
- if not added:
- unwanted.append(handler)
-
- for handler in unwanted:
- self.handlers.remove(handler)
-
- # sort indexed methods
- # XXX could be cleaned up
- for lookup in [process_request, process_response]:
- for scheme, handlers in lookup.iteritems():
- lookup[scheme] = handlers
- for scheme, lookup in handle_error.iteritems():
- for code, handlers in lookup.iteritems():
- handlers = list(handlers)
- handlers.sort()
- lookup[code] = handlers
- for scheme, handlers in handle_open.iteritems():
- handlers = list(handlers)
- handlers.sort()
- handle_open[scheme] = handlers
-
- # cache the indexes
- self.handle_error = handle_error
- self.handle_open = handle_open
- self.process_request = process_request
- self.process_response = process_response
- self._any_request = any_request
- self._any_response = any_response
-
- def _request(self, url_or_req, data):
- if isstringlike(url_or_req):
- req = Request(url_or_req, data)
- else:
- # already a urllib2.Request or mechanize.Request instance
- req = url_or_req
- if data is not None:
- req.add_data(data)
- return req
-
- def open(self, fullurl, data=None):
- req = self._request(fullurl, data)
- req_scheme = req.get_type()
-
- self._maybe_reindex_handlers()
-
- # pre-process request
- # XXX should we allow a Processor to change the URL scheme
- # of the request?
- request_processors = set(self.process_request.get(req_scheme, []))
- request_processors.update(self._any_request)
- request_processors = list(request_processors)
- request_processors.sort()
- for processor in request_processors:
- for meth_name in ["any_request", req_scheme+"_request"]:
- meth = getattr(processor, meth_name, None)
- if meth:
- req = meth(req)
-
- # In Python >= 2.4, .open() supports processors already, so we must
- # call ._open() instead.
- urlopen = getattr(urllib2.OpenerDirector, "_open",
- urllib2.OpenerDirector.open)
- response = urlopen(self, req, data)
-
- # post-process response
- response_processors = set(self.process_response.get(req_scheme, []))
- response_processors.update(self._any_response)
- response_processors = list(response_processors)
- response_processors.sort()
- for processor in response_processors:
- for meth_name in ["any_response", req_scheme+"_response"]:
- meth = getattr(processor, meth_name, None)
- if meth:
- response = meth(req, response)
-
- return response
-
- def error(self, proto, *args):
- if proto in ['http', 'https']:
- # XXX http[s] protocols are special-cased
- dict = self.handle_error['http'] # https is not different than http
- proto = args[2] # YUCK!
- meth_name = 'http_error_%s' % proto
- http_err = 1
- orig_args = args
- else:
- dict = self.handle_error
- meth_name = proto + '_error'
- http_err = 0
- args = (dict, proto, meth_name) + args
- result = apply(self._call_chain, args)
- if result:
- return result
-
- if http_err:
- args = (dict, 'default', 'http_error_default') + orig_args
- return apply(self._call_chain, args)
-
- def retrieve(self, fullurl, filename=None, reporthook=None, data=None):
- """Returns (filename, headers).
-
- For remote objects, the default filename will refer to a temporary
- file.
-
- """
- req = self._request(fullurl, data)
- type_ = req.get_type()
- fp = self.open(req)
- headers = fp.info()
- if filename is None and type == 'file':
- return url2pathname(req.get_selector()), headers
- if filename:
- tfp = open(filename, 'wb')
- else:
- path = urlparse(fullurl)[2]
- suffix = os.path.splitext(path)[1]
- tfp = tempfile.TemporaryFile("wb", suffix=suffix)
- result = filename, headers
- bs = 1024*8
- size = -1
- read = 0
- blocknum = 1
- if reporthook:
- if headers.has_key("content-length"):
- size = int(headers["Content-Length"])
- reporthook(0, bs, size)
- while 1:
- block = fp.read(bs)
- read += len(block)
- if reporthook:
- reporthook(blocknum, bs, size)
- blocknum = blocknum + 1
- if not block:
- break
- tfp.write(block)
- fp.close()
- tfp.close()
- del fp
- del tfp
- if size>=0 and read<size:
- raise IOError("incomplete retrieval error",
- "got only %d bytes out of %d" % (read,size))
- return result
Deleted: /wwwsearch/mechanize/trunk/mechanize/_Request.py
==============================================================================
--- /wwwsearch/mechanize/trunk/mechanize/_Request.py Sun May 21 21:32:06 2006
+++ (empty file)
@@ -1,68 +0,0 @@
-"""Integration with Python standard library module urllib2: Request class.
-
-Copyright 2004-2006 John J Lee <jjl at pobox.com>
-
-This code is free software; you can redistribute it and/or modify it
-under the terms of the BSD or ZPL 2.1 licenses (see the file
-COPYING.txt included with the distribution).
-
-"""
-
-import urllib2, string
-
-from _ClientCookie import request_host
-
-
-class Request(urllib2.Request):
- def __init__(self, url, data=None, headers={},
- origin_req_host=None, unverifiable=False):
- urllib2.Request.__init__(self, url, data, headers)
- self.unredirected_hdrs = {}
-
- # All the terminology below comes from RFC 2965.
- self.unverifiable = unverifiable
- # Set request-host of origin transaction.
- # The origin request-host is needed in order to decide whether
- # unverifiable sub-requests (automatic redirects, images embedded
- # in HTML, etc.) are to third-party hosts. If they are, the
- # resulting transactions might need to be conducted with cookies
- # turned off.
- if origin_req_host is None:
- origin_req_host = request_host(self)
- self.origin_req_host = origin_req_host
-
- def get_origin_req_host(self):
- return self.origin_req_host
-
- def is_unverifiable(self):
- return self.unverifiable
-
- def add_unredirected_header(self, key, val):
- """Add a header that will not be added to a redirected request."""
- self.unredirected_hdrs[string.capitalize(key)] = val
-
- def has_header(self, header_name):
- """True iff request has named header (regular or unredirected)."""
- if (self.headers.has_key(header_name) or
- self.unredirected_hdrs.has_key(header_name)):
- return True
- return False
-
- def get_header(self, header_name, default=None):
- return self.headers.get(
- header_name,
- self.unredirected_hdrs.get(header_name, default))
-
- def header_items(self):
- hdrs = self.unredirected_hdrs.copy()
- hdrs.update(self.headers)
- return hdrs.items()
-
- def __str__(self):
- return "<Request for %s>" % self.get_full_url()
-
- def get_method(self):
- if self.has_data():
- return "POST"
- else:
- return "GET"
Deleted: /wwwsearch/mechanize/trunk/mechanize/_Util.py
==============================================================================
--- /wwwsearch/mechanize/trunk/mechanize/_Util.py Sun May 21 21:32:06 2006
+++ (empty file)
@@ -1,629 +0,0 @@
-"""Python backwards-compat., date/time routines, seekable file object wrapper.
-
- Copyright 2002-2006 John J Lee <jjl at pobox.com>
-
-This code is free software; you can redistribute it and/or modify it
-under the terms of the BSD or ZPL 2.1 licenses (see the file
-COPYING.txt included with the distribution).
-
-"""
-
-import re, string, time, copy, urllib
-from types import TupleType
-from cStringIO import StringIO
-
-def startswith(string, initial):
- if len(initial) > len(string): return False
- return string[:len(initial)] == initial
-
-def endswith(string, final):
- if len(final) > len(string): return False
- return string[-len(final):] == final
-
-def isstringlike(x):
- try: x+""
- except: return False
- else: return True
-
-SPACE_DICT = {}
-for c in string.whitespace:
- SPACE_DICT[c] = None
-del c
-def isspace(string):
- for c in string:
- if not SPACE_DICT.has_key(c): return False
- return True
-
-## def caller():
-## try:
-## raise SyntaxError
-## except:
-## import sys
-## return sys.exc_traceback.tb_frame.f_back.f_back.f_code.co_name
-
-
-# this is here rather than in _HeadersUtil as it's just for
-# compatibility with old Python versions, rather than entirely new code
-def getheaders(msg, name):
- """Get all values for a header.
-
- This returns a list of values for headers given more than once; each
- value in the result list is stripped in the same way as the result of
- getheader(). If the header is not given, return an empty list.
- """
- result = []
- current = ''
- have_header = 0
- for s in msg.getallmatchingheaders(name):
- if isspace(s[0]):
- if current:
- current = "%s\n %s" % (current, string.strip(s))
- else:
- current = string.strip(s)
- else:
- if have_header:
- result.append(current)
- current = string.strip(s[string.find(s, ":") + 1:])
- have_header = 1
- if have_header:
- result.append(current)
- return result
-
-from calendar import timegm
-
-# Date/time conversion routines for formats used by the HTTP protocol.
-
-EPOCH = 1970
-def my_timegm(tt):
- year, month, mday, hour, min, sec = tt[:6]
- if ((year >= EPOCH) and (1 <= month <= 12) and (1 <= mday <= 31) and
- (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
- return timegm(tt)
- else:
- return None
-
-days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
-months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
-months_lower = []
-for month in months: months_lower.append(string.lower(month))
-
-
-def time2isoz(t=None):
- """Return a string representing time in seconds since epoch, t.
-
- If the function is called without an argument, it will use the current
- time.
-
- The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
- representing Universal Time (UTC, aka GMT). An example of this format is:
-
- 1994-11-24 08:49:37Z
-
- """
- if t is None: t = time.time()
- year, mon, mday, hour, min, sec = time.gmtime(t)[:6]
- return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
- year, mon, mday, hour, min, sec)
-
-def time2netscape(t=None):
- """Return a string representing time in seconds since epoch, t.
-
- If the function is called without an argument, it will use the current
- time.
-
- The format of the returned string is like this:
-
- Wed, DD-Mon-YYYY HH:MM:SS GMT
-
- """
- if t is None: t = time.time()
- year, mon, mday, hour, min, sec, wday = time.gmtime(t)[:7]
- return "%s %02d-%s-%04d %02d:%02d:%02d GMT" % (
- days[wday], mday, months[mon-1], year, hour, min, sec)
-
-
-UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}
-
-timezone_re = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$")
-def offset_from_tz_string(tz):
- offset = None
- if UTC_ZONES.has_key(tz):
- offset = 0
- else:
- m = timezone_re.search(tz)
- if m:
- offset = 3600 * int(m.group(2))
- if m.group(3):
- offset = offset + 60 * int(m.group(3))
- if m.group(1) == '-':
- offset = -offset
- return offset
-
-def _str2time(day, mon, yr, hr, min, sec, tz):
- # translate month name to number
- # month numbers start with 1 (January)
- try:
- mon = months_lower.index(string.lower(mon))+1
- except ValueError:
- # maybe it's already a number
- try:
- imon = int(mon)
- except ValueError:
- return None
- if 1 <= imon <= 12:
- mon = imon
- else:
- return None
-
- # make sure clock elements are defined
- if hr is None: hr = 0
- if min is None: min = 0
- if sec is None: sec = 0
-
- yr = int(yr)
- day = int(day)
- hr = int(hr)
- min = int(min)
- sec = int(sec)
-
- if yr < 1000:
- # find "obvious" year
- cur_yr = time.localtime(time.time())[0]
- m = cur_yr % 100
- tmp = yr
- yr = yr + cur_yr - m
- m = m - tmp
- if abs(m) > 50:
- if m > 0: yr = yr + 100
- else: yr = yr - 100
-
- # convert UTC time tuple to seconds since epoch (not timezone-adjusted)
- t = my_timegm((yr, mon, day, hr, min, sec, tz))
-
- if t is not None:
- # adjust time using timezone string, to get absolute time since epoch
- if tz is None:
- tz = "UTC"
- tz = string.upper(tz)
- offset = offset_from_tz_string(tz)
- if offset is None:
- return None
- t = t - offset
-
- return t
-
-
-strict_re = re.compile(r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) (\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$")
-wkday_re = re.compile(
- r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I)
-loose_http_re = re.compile(
- r"""^
- (\d\d?) # day
- (?:\s+|[-\/])
- (\w+) # month
- (?:\s+|[-\/])
- (\d+) # year
- (?:
- (?:\s+|:) # separator before clock
- (\d\d?):(\d\d) # hour:min
- (?::(\d\d))? # optional seconds
- )? # optional clock
- \s*
- ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
- \s*
- (?:\(\w+\))? # ASCII representation of timezone in parens.
- \s*$""", re.X)
-def http2time(text):
- """Returns time in seconds since epoch of time represented by a string.
-
- Return value is an integer.
-
- None is returned if the format of str is unrecognized, the time is outside
- the representable range, or the timezone string is not recognized. If the
- string contains no timezone, UTC is assumed.
-
- The timezone in the string may be numerical (like "-0800" or "+0100") or a
- string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
- timezone strings equivalent to UTC (zero offset) are known to the function.
-
- The function loosely parses the following formats:
-
- Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
- Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
- Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
- 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
- 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
- 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
-
- The parser ignores leading and trailing whitespace. The time may be
- absent.
-
- If the year is given with only 2 digits, the function will select the
- century that makes the year closest to the current date.
-
- """
- # fast exit for strictly conforming string
- m = strict_re.search(text)
- if m:
- g = m.groups()
- mon = months_lower.index(string.lower(g[1])) + 1
- tt = (int(g[2]), mon, int(g[0]),
- int(g[3]), int(g[4]), float(g[5]))
- return my_timegm(tt)
-
- # No, we need some messy parsing...
-
- # clean up
- text = string.lstrip(text)
- text = wkday_re.sub("", text, 1) # Useless weekday
-
- # tz is time zone specifier string
- day, mon, yr, hr, min, sec, tz = [None]*7
-
- # loose regexp parse
- m = loose_http_re.search(text)
- if m is not None:
- day, mon, yr, hr, min, sec, tz = m.groups()
- else:
- return None # bad format
-
- return _str2time(day, mon, yr, hr, min, sec, tz)
-
-
-iso_re = re.compile(
- """^
- (\d{4}) # year
- [-\/]?
- (\d\d?) # numerical month
- [-\/]?
- (\d\d?) # day
- (?:
- (?:\s+|[-:Tt]) # separator before clock
- (\d\d?):?(\d\d) # hour:min
- (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
- )? # optional clock
- \s*
- ([-+]?\d\d?:?(:?\d\d)?
- |Z|z)? # timezone (Z is "zero meridian", i.e. GMT)
- \s*$""", re.X)
-def iso2time(text):
- """
- As for http2time, but parses the ISO 8601 formats:
-
- 1994-02-03 14:15:29 -0100 -- ISO 8601 format
- 1994