From ianb at codespeak.net  Tue Jul  1 19:07:33 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  1 Jul 2008 19:07:33 +0200 (CEST)
Subject: [z3-checkins] r56206 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk: . deliverance
	deliverance/tests deliverance/tests/example-files
Message-ID: <20080701170733.9C2A616A0F8@codespeak.net>

Author: ianb
Date: Tue Jul  1 19:07:32 2008
New Revision: 56206

Added:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/index.html   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example.py   (contents, props changed)
Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py
Log:
Added an example setup, and a bunch of fixes and tweaks to work that out

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	Tue Jul  1 19:07:32 2008
@@ -8,20 +8,34 @@
 
 import logging
 from lxml.etree import tostring, _Element
+from tempita import HTMLTemplate, html_quote, html
+
+NOTIFY = (logging.INFO + logging.WARN) / 2
+
+logging.addLevelName(NOTIFY, 'NOTIFY')
 
 class SavingLogger(object):
     """
     Logger that saves all its messages locally.
     """
-    def __init__(self, request, description=True):
+    def __init__(self, request, middleware, description=None):
         self.messages = []
+        if description is None:
+            description = 'deliv_log' in request.GET
         if description:
             self.descriptions = []
             self.describe = self.add_description
         else:
+            self.descriptions = None
             self.describe = None
+        self.middleware = middleware
+        self.request = request
+        # This is writable:
+        self.theme_url = None
+
     def add_description(self, msg):
         self.descriptions.append(msg)
+
     def message(self, level, el, msg, *args, **kw):
         if args:
             msg = msg % args
@@ -30,23 +44,104 @@
         self.messages.append((level, el, msg))
         return msg
     def debug(self, el, msg, *args, **kw):
-        self.message(logging.DEBUG, el, msg, *args, **kw)
+        return self.message(logging.DEBUG, el, msg, *args, **kw)
     def info(self, el, msg, *args, **kw):
-        self.message(logging.INFO, el, msg, *args, **kw)
+        return self.message(logging.INFO, el, msg, *args, **kw)
     def notify(self, el, msg, *args, **kw):
-        self.message(logging.INFO+1, el, msg, *args, **kw)
+        return self.message(logging.NOTIFY, el, msg, *args, **kw)
     def warn(self, el, msg, *args, **kw):
-        self.message(logging.WARN, el, msg, *args, **kw)
+        return self.message(logging.WARN, el, msg, *args, **kw)
     warning = warn
     def error(self, el, msg, *args, **kw):
-        self.message(logging.ERROR, el, msg, *args, **kw)
+        return self.message(logging.ERROR, el, msg, *args, **kw)
     def fatal(self, el, msg, *args, **kw):
-        self.message(logging.FATAL, el, msg, *args, **kw)
+        return self.message(logging.FATAL, el, msg, *args, **kw)
+
+    def finish_request(self, req, resp):
+        if 'deliv_log' in req.GET:
+            resp.body += self.format_html_log()
+            resp.cache_expires()
+        return resp
+
+    log_template = HTMLTemplate('''\
+    <h1 style="border-top: 3px dotted #f00">Deliverance Information</h1>
+
+    {{if log.descriptions:}}
+      {{div}}
+        {{h2}}What happened?</h2>
+        {{div_inner}}
+        <ol>
+        {{for desc in log.descriptions}}
+          <li>{{desc}}</li>
+        {{endfor}}
+        </ol>
+      </div></div>
+    {{endif}}
+
+    {{if log.messages}}
+      {{div}}
+      {{h2}}Log</h2>
+      {{div_inner}}
+      <table>
+          <tr>
+            <th>Level</th><th>Message</th><th>Context</th>
+          </tr>
+        {{for level, level_name, el, message in log.resolved_messages():}}
+          {{py:color, bgcolor = log.color_for_level(level)}}
+          <tr style="color: {{color}}; background-color: {{bgcolor}}; vertical-align: top">
+            {{td}}{{level_name}}</td>
+            {{td}}{{message}}</td>
+            {{td}}{{log.obj_as_html(el) | html}}</td>
+          </tr>
+        {{endfor}}
+      </table>
+      </div></div>
+    {{else}}
+      {{h2}}No Log Messages</h2>
+    {{endif}}
+    ''', name='deliverance.log.SavingLogger.log_template')
+     
+    tags = dict(
+        h2=html('<h2 style="color: #000; background-color: #f90; margin-top: 0; border-bottom: 1px solid #630">'),
+        div=html('<div style="border: 2px solid #000; margin-bottom: 1em">'),
+        div_inner=html('<div style="padding: 0.25em">'),
+        td=html('<td style="margin-bottom: 0.25em; border-bottom: 1px solid #ddd; padding-right: 0.5em">'),
+        )
+
+    def format_html_log(self):
+        return self.log_template.substitute(
+            log=self, middleware=self.middleware, **self.tags)
+
+    def resolved_messages(self):
+        for level, el, msg in self.messages:
+            level_name = logging.getLevelName(level)
+            yield level, level_name, el, msg
+
+    def obj_as_html(self, el):
+        ## FIXME: another magic method?
+        if hasattr(el, 'log_description'):
+            return el.log_description(self)
+        elif isinstance(el, _Element):
+            return html_quote(tostring(el))
+        else:
+            return html_quote(unicode(el))
+
+    def color_for_level(self, level):
+        return {
+            logging.DEBUG: ('#666', '#fff'),
+            logging.INFO: ('#333', '#fff'),
+            NOTIFY: ('#000', '#fff'),
+            logging.WARNING: ('#600', '#fff'),
+            logging.ERROR: ('#fff', '#600'),
+            logging.CRITICAL: ('#000', '#f33')}[level]
+
+    def link_to(self, url, source=False, line=None, selector=None):
+        return self.middleware.link_to(self.request, url, source=source, line=line, selector=selector)
 
 class PrintingLogger(SavingLogger):
 
-    def __init__(self, request, description=True, print_level=logging.DEBUG):
-        super(PrintingLogger, self).__init__(request, description=description)
+    def __init__(self, request, middleware, description=True, print_level=logging.DEBUG):
+        super(PrintingLogger, self).__init__(request, middleware, description=description)
         self.print_level = print_level
 
     def add_description(self, msg):

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	Tue Jul  1 19:07:32 2008
@@ -1,6 +1,14 @@
-from webob import Request
+from webob import Request, Response
+from webob import exc
+from wsgiproxy.exactproxy import proxy_exact_request
 from deliverance.log import SavingLogger
-import urllib2
+import urllib
+import hmac
+import sha
+from pygments import highlight as pygments_highlight
+from pygments.lexers import XmlLexer, HtmlLexer, guess_lexer_for_filename
+from pygments.formatters import HtmlFormatter
+from tempita import HTMLTemplate, html_quote, html
 
 class DeliveranceMiddleware(object):
 
@@ -10,39 +18,170 @@
         self.log_factory = log_factory
         self.log_factory_kw = log_factory_kw
 
+    def log_description(self, log=None):
+        return 'Deliverance'
+
     def __call__(self, environ, start_response):
         ## FIXME: copy_get?:
-        orig_req = Request(environ.copy())
         req = Request(environ)
-        log = self.log_factory(req, **self.log_factory_kw)
+        req.environ['deliverance.base_url'] = req.application_url
+        orig_req = Request(environ.copy())
+        log = self.log_factory(req, self, **self.log_factory_kw)
+        def resource_fetcher(url):
+            return self.get_resource(url, orig_req, log)
+        if req.path_info_peek() == '.deliverance':
+            req.path_info_pop()
+            resp = self.internal_app(req, resource_fetcher)
+            return resp(environ, start_response)
         resp = req.get_response(self.app)
         ## FIXME: also XHTML?
         if resp.content_type != 'text/html':
             return resp(environ, start_response)
-        def get_resource(url):
-            assert url is not None
-            ## FIXME: should this return a webob.Response object?
-            if url.startswith(orig_req.application_url + '/'):
-                subreq = orig_req.copy_get()
-                subreq.environ['deliverance.subrequest_original_environ'] = orig_req.environ
-                new_path_info = url[len(orig_req.application_url):]
-                assert new_path_info.startswith('/')
-                subreq.path_info = new_path_info
-                subresp = subreq.get_response(self.app)
-                ## FIXME: error if not HTML?
-                ## FIXME: handle redirects?
-                ## FIXME: handle non-200?
-                return subresp.body
-            else:
-                ## FIXME: pluggable subrequest handler?
-                f = urllib2.urlopen(url)
-                body = f.read()
-                f.close()
-                return body
-        rule_set = self.rule_getter(get_resource, self.app, orig_req)
-        resp = rule_set.apply_rules(req, resp, get_resource, log)
+        rule_set = self.rule_getter(resource_fetcher, self.app, orig_req)
+        resp = rule_set.apply_rules(req, resp, resource_fetcher, log)
+        resp = log.finish_request(req, resp)
         return resp(environ, start_response)
 
+    def get_resource(self, url, orig_req, log):
+        assert url is not None
+        ## FIXME: should this return a webob.Response object?
+        if url.startswith(orig_req.application_url + '/'):
+            subreq = orig_req.copy_get()
+            subreq.environ['deliverance.subrequest_original_environ'] = orig_req.environ
+            new_path_info = url[len(orig_req.application_url):]
+            query_string = ''
+            if '?' in new_path_info:
+                new_path_info, query_string = new_path_info.split('?')
+            new_path_info = urllib.unquote(new_path_info)
+            assert new_path_info.startswith('/')
+            subreq.path_info = new_path_info
+            subreq.query_string = query_string
+            subresp = subreq.get_response(self.app)
+            ## FIXME: error if not HTML?
+            ## FIXME: handle redirects?
+            ## FIXME: handle non-200?
+            log.debug(self, 'Internal request for %s: %s content-type: %s',
+                            url, subresp.status, subresp.content_type)
+            return subresp
+        else:
+            ## FIXME: pluggable subrequest handler?
+            subreq = Request.blank(url)
+            resp = subreq.get_response(proxy_exact_request)
+            log.debug(self, 'External request for %s: %s content-type: %s',
+                      url, subresp.status, subresp.content_type)
+            return subresp
+
+
+    def link_to(self, req, url, source=False, line=None, selector=None):
+        base = req.environ['deliverance.base_url']
+        base += '/.deliverance/view'
+        source = int(bool(source))
+        args = {'url': url}
+        if source:
+            args['source'] = '1'
+        if line:
+            args['line'] = str(line)
+        if selector:
+            args['selector'] = selector
+        url = base + '?' + urllib.urlencode(args)
+        if selector:
+            url += '#deliverance-selection'
+        if line:
+            ## FIXME: figure out line anchor
+            pass
+        return url
+
+    def internal_app(self, req, resource_fetcher):
+        segment = req.path_info_peek()
+        method = 'action_%s' % segment
+        method = getattr(self, method, None)
+        if not method:
+            return exc.HTTPNotFound('There is no %r action' % segment)
+        try:
+            return method(req, resource_fetcher)
+        except exc.HTTPException, e:
+            return e
+
+    def action_view(self, req, resource_fetcher):
+        url = req.GET['url']
+        source = int(req.GET.get('source', '0'))
+        line = int(req.GET.get('line', '0')) or ''
+        selector = req.GET.get('selector', '')
+        subresp = resource_fetcher(url)
+        if source:
+            ct = subresp.content_type
+            if ct.content_type.startswith('application/xml'):
+                lexer = XmlLexer()
+            elif ct == 'text/html':
+                lexer = HtmlLexer()
+            else:
+                ## FIXME: what then?
+                lexer = HtmlLexer()
+            text = pygments_highlight(
+                subresp.body, lexer,
+                HtmlFormatter(linenos=linenos, lineanchors='code'))
+        else:
+            from deliverance.selector import Selector
+            from lxml.html import fromstring, document_fromstring, tostring
+            doc = document_fromstring(subresp.body)
+            selector = Selector.parse(selector)
+            type, elements, attributes = selector(doc)
+            if not elements:
+                template = self._not_found_template
+            else:
+                template = self._found_template
+            all_elements = []
+            for index, el in enumerate(elements):
+                anchor = 'deliverance-selection'
+                if index:
+                    anchor += '-%s' % index
+                ## FIXME: is a <a name> better?
+                el.set('id', anchor)
+                ## FIXME: add :target CSS rule
+                ## FIXME: or better, some Javascript
+                all_elements.append((anchor, el))
+                style = el.get('style', '')
+                if style:
+                    style += '; '
+                style += '/* deliverance */ border: 2px dotted #f00'
+                el.set('style', style)
+            def format_tag(tag):
+                return tostring(tag).split('>')[0]+'>'
+            text = template.substitute(
+                elements=all_elements, selector=selector, format_tag=format_tag)
+            message = fromstring(self._message_template.substitute(message=text))
+            if doc.body.text:
+                message.tail = doc.body.text
+                doc.body.text = ''
+            doc.body.insert(0, message)
+            text = tostring(doc)
+        resp = Response(text)
+        return resp
+
+    _not_found_template = HTMLTemplate('''\
+    There were no elements that matched the selector <code>{{selector}}</code>
+    ''', 'deliverance.middleware.DeliveranceMiddleware._not_found_template')
+
+    _found_template = HTMLTemplate('''\
+    {{if len(elements) == 1}}
+      One element matched the selector <code>{{selector}}</code>;
+      <a href="#{{elements[0][0]}}">jump to element</a>
+    {{else}}
+      {{len(elements)}} elements matched the selector <code>{{selector}}</code>:
+      <ol>
+      {{for anchor, el in elements}}
+        <li><a href="#{{anchor}}"><code>{{format_tag(el)}}</code></a></li>
+      {{endfor}}
+      </ol>
+    {{endif}}
+    ''', 'deliverance.middleware.DeliveranceMiddleware._found_template')
+
+    _message_template = HTMLTemplate('''\
+    <div style="color: #000; background-color: #f90; border-bottom: 2px dotted #f00">
+    {{message|html}}
+    </div>''', 'deliverance.middleware.DeliveranceMiddleware._message_template')
+
+
 class SubrequestRuleGetter(object):
 
     def __init__(self, url):
@@ -52,7 +191,16 @@
         from lxml.etree import XML, XMLSyntaxError
         import urlparse
         url = urlparse.urljoin(orig_req.url, self.url)
-        doc_text = get_resource(url)
+        doc_resp = get_resource(url)
+        if doc_resp.status_int != 200:
+            ## FIXME: better error
+            assert 0, "Bad response: %r" % doc_resp
+        ## FIXME: better content-type detection
+        if doc_resp.content_type != 'application/xml':
+            ## FIXME: better error
+            assert 0, "Bad response content-type: %s (from response %r)" % (
+                doc_resp.content_type, doc_resp)
+        doc_text = doc_resp.body
         try:
             doc = XML(doc_text, base_url=url)
         except XMLSyntaxError, e:

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	Tue Jul  1 19:07:32 2008
@@ -7,6 +7,7 @@
 from deliverance.util.converters import asbool, html_quote
 from deliverance.selector import Selector
 from lxml import etree
+from tempita import html
 
 CONTENT_ATTRIB = 'x-a-marker-attribute-for-deliverance'
 
@@ -197,32 +198,7 @@
     move_supported = True
 
     def describe_self(self):
-        """
-        A text description of this rule, for use in log messages and errors
-        """
-        parts = ['<%s' % self.name]
-        if getattr(self, 'content', None):
-            parts.append('content="%s"' % html_quote(self.content))
-        if getattr(self, 'content_href', None):
-            parts.append('href="%s"' % html_quote(self.content_href))
-        if self.move_supported and not getattr(self, 'move', False):
-            parts.append('move="1"')
-        v = getattr(self, 'nocontent', 'warn')
-        if v != 'warn':
-            parts.append(self.format_error('nocontent', v))
-        v = getattr(self, 'manycontent', ('warn', None))
-        if v != ('warn', None):
-            parts.append(self.format_error('manycontent', v))
-        if getattr(self, 'theme', None):
-            parts.append('theme="%s"' % html_quote(self.theme))
-        v = getattr(self, 'notheme', 'warn')
-        if v != 'warn':
-            parts.append(self.format_error('notheme', v))
-        v = getattr(self, 'manytheme', ('warn', None))
-        if v != ('warn', None):
-            parts.append(self.format_error('manytheme', v))
-        ## FIXME: add source_location
-        return ' '.join(parts) + ' />'
+        return self.log_description(log=None)
 
     def __str__(self):
         return self.describe_self()
@@ -295,6 +271,52 @@
                 elements.remove(el)
         return type, elements, attributes
 
+    def log_description(self, log=None):
+        """
+        A text description of this rule, for use in log messages and errors
+        """
+        def linked_item(url, body, source=None, line=None, selector=None):
+            body = html_quote(body)
+            if log is None or url is None:
+                return body
+            link = log.link_to(url, source=source, line=line, selector=selector)
+            return '<a href="%s">%s</a>' % (html_quote(link), body)
+        if log:
+            request_url = log.request.url
+        else:
+            request_url = None
+        parts = ['&lt;%s' % linked_item(self.source_location, self.name)]
+        if getattr(self, 'content', None):
+            body = 'content="%s"' % html_quote(self.content)
+            parts.append(linked_item(request_url, body, selector=self.content))
+        if getattr(self, 'content_href', None):
+            dest = self.content_href
+            if request_url:
+                dest = urlparse.urljoin(request_url, dest)
+            body = 'href="%s"' % html_quote(self.content_href)
+            parts.append(linked_item(dest, body, source=True))
+        if self.move_supported and not getattr(self, 'move', False):
+            parts.append('move="1"')
+        v = getattr(self, 'nocontent', 'warn')
+        if v != 'warn':
+            parts.append(self.format_error('nocontent', v))
+        v = getattr(self, 'manycontent', ('warn', None))
+        if v != ('warn', None):
+            parts.append(self.format_error('manycontent', v))
+        if getattr(self, 'theme', None):
+            body = 'theme="%s"' % html_quote(self.theme)
+            theme_url = getattr(log, 'theme_url', None)
+            parts.append(linked_item(theme_url, body, selector=self.theme))
+        v = getattr(self, 'notheme', 'warn')
+        if v != 'warn':
+            parts.append(self.format_error('notheme', v))
+        v = getattr(self, 'manytheme', ('warn', None))
+        if v != ('warn', None):
+            parts.append(self.format_error('manytheme', v))
+        ## FIXME: add source_location
+        return html(' '.join(parts) + ' /&gt;')
+        
+
 class TransformAction(AbstractAction):
     # Abstract class for the rules that move from the content to the theme (replace, append, prepend)
 
@@ -342,7 +364,8 @@
         """
         describe = log.describe
         if self.content_href:
-            content_doc = resource_fetcher(self.content_href)
+            ## FIXME: check response type
+            content_doc = resource_fetcher(self.content_href).body
         if not self.if_content_matches(content_doc, log):
             return
         content_type, content_els, content_attributes = self.select_elements(self.content, content_doc, theme=False)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	Tue Jul  1 19:07:32 2008
@@ -1,4 +1,4 @@
-from deliverance.exceptions import AbortTheme
+from deliverance.exceptions import AbortTheme, DeliveranceSyntaxError
 from deliverance.pagematch import run_matches, Match
 from deliverance.rules import Rule, remove_content_attribs
 from lxml.html import tostring, document_fromstring
@@ -55,10 +55,12 @@
         resp.body = tostring(theme_doc)
         return resp
 
-    def get_theme(self, url, resource_getter, log):
+    def get_theme(self, url, resource_fetcher, log):
         log.info(self, 'Fetching theme from %s' % url)
+        log.theme_url = url
         ## FIXME: should do caching
-        doc = self.parse_document(resource_getter(url), url)
+        ## FIXME: check response status
+        doc = self.parse_document(resource_fetcher(url).body, url)
         doc.make_links_absolute()
         return doc
 
@@ -82,8 +84,10 @@
                 ## FIXME: Add parse error
                 default_theme = el.get('href')
             else:
-                ## FIXME: better error
-                assert 0
+                ## FIXME: source location?
+                raise DeliveranceSyntaxError(
+                    "Invalid tag %s (unknown tag name %r)" % (tostring(el).split('>', 1)[0]+'>', el.tag),
+                    element=el)
         rules_by_class = {}
         for rule in rules:
             for class_name in rule.classes:
@@ -116,6 +120,7 @@
 standard_rule = Rule.parse_xml(XML('''\
 <rule>
   <!-- FIXME: append-or-replace for title? -->
+  <!-- FIXME: maybe something like notheme="append:/html/head" -->
   <replace content="children:/html/head/title" theme="children:/html/head/title" nocontent="ignore" />
   <append content="elements:/html/head/link" theme="children:/html/head" nocontent="ignore" />
   <append content="elements:/html/head/script" theme="children:/html/head" nocontent="ignore" />

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/index.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/index.html	Tue Jul  1 19:07:32 2008
@@ -0,0 +1,10 @@
+<html>
+<head>
+<title>My Site</title>
+</head>
+
+<body>
+
+This is my awesome site!
+
+</body> </html>

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml	Tue Jul  1 19:07:32 2008
@@ -0,0 +1,7 @@
+<ruleset>
+ <theme href="/theme.html" />
+ <rule>
+  <replace content="children:body" theme="children:#content" nocontent="abort" />
+  <replace content="children:title" theme="children:#header" nocontent="ignore" />
+ </rule>
+</ruleset>

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css	Tue Jul  1 19:07:32 2008
@@ -0,0 +1,9 @@
+body {
+  font-family: sans-serif;
+}
+
+#footer {
+  border-top: 1px solid #999;
+  font-size: 70%;
+  margin-top: 1em;
+}

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html	Tue Jul  1 19:07:32 2008
@@ -0,0 +1,14 @@
+<html>
+<head>
+<title>A theme</title>
+<link rel="stylesheet" href="style.css" type="text/css">
+</head>
+
+<body>
+<h1 id="header">A Page</h1>
+
+<div id="content">Some content</div>
+
+<div id="footer">Copyright (C) 2000 Some Corporation</div>
+
+</body> </html>

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example.py	Tue Jul  1 19:07:32 2008
@@ -0,0 +1,16 @@
+"""
+This sets up an example site
+"""
+
+import os
+from paste.urlparser import StaticURLParser
+from paste.httpserver import serve
+from weberror.evalexception import EvalException
+from deliverance.middleware import DeliveranceMiddleware, SubrequestRuleGetter
+
+base_path = os.path.join(os.path.dirname(__file__), 'example-files')
+app = StaticURLParser(base_path)
+deliv_app = DeliveranceMiddleware(app, SubrequestRuleGetter('/rules.xml'))
+
+if __name__ == '__main__':
+    serve(EvalException(deliv_app))

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py	Tue Jul  1 19:07:32 2008
@@ -30,6 +30,10 @@
       tests_require=['nose'],
       install_requires=[
         "lxml",
+        "WebOb",
+        "WSGIProxy",
+        "Tempita",
+        "Pygments",
       ],
       entry_points="""
       """,

From ianb at codespeak.net  Tue Jul  1 19:40:41 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  1 Jul 2008 19:40:41 +0200 (CEST)
Subject: [z3-checkins] r56210 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance: . tests
Message-ID: <20080701174041.9CDBE168509@codespeak.net>

Author: ianb
Date: Tue Jul  1 19:40:41 2008
New Revision: 56210

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_middleware.txt
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_selection.txt
Log:
Remove the description stuff (use only normal logging)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	Tue Jul  1 19:40:41 2008
@@ -18,24 +18,13 @@
     """
     Logger that saves all its messages locally.
     """
-    def __init__(self, request, middleware, description=None):
+    def __init__(self, request, middleware):
         self.messages = []
-        if description is None:
-            description = 'deliv_log' in request.GET
-        if description:
-            self.descriptions = []
-            self.describe = self.add_description
-        else:
-            self.descriptions = None
-            self.describe = None
         self.middleware = middleware
         self.request = request
         # This is writable:
         self.theme_url = None
 
-    def add_description(self, msg):
-        self.descriptions.append(msg)
-
     def message(self, level, el, msg, *args, **kw):
         if args:
             msg = msg % args
@@ -66,18 +55,6 @@
     log_template = HTMLTemplate('''\
     <h1 style="border-top: 3px dotted #f00">Deliverance Information</h1>
 
-    {{if log.descriptions:}}
-      {{div}}
-        {{h2}}What happened?</h2>
-        {{div_inner}}
-        <ol>
-        {{for desc in log.descriptions}}
-          <li>{{desc}}</li>
-        {{endfor}}
-        </ol>
-      </div></div>
-    {{endif}}
-
     {{if log.messages}}
       {{div}}
       {{h2}}Log</h2>
@@ -140,14 +117,10 @@
 
 class PrintingLogger(SavingLogger):
 
-    def __init__(self, request, middleware, description=True, print_level=logging.DEBUG):
-        super(PrintingLogger, self).__init__(request, middleware, description=description)
+    def __init__(self, request, middleware, print_level=logging.DEBUG):
+        super(PrintingLogger, self).__init__(request, middleware)
         self.print_level = print_level
 
-    def add_description(self, msg):
-        print 'description:', msg
-        super(PrintingLogger, self).add_description(msg)
-
     def message(self, level, el, msg, *args, **kw):
         msg = super(PrintingLogger, self).message(level, el, msg, *args, **kw)
         if level >= self.print_level:

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	Tue Jul  1 19:40:41 2008
@@ -186,9 +186,6 @@
         if ((not matched and not self.if_content.inverted)
             or (matched and self.if_content.inverted)):
             log.info(self, 'skipping rule because if-content="%s" does not match', self.if_content)
-            if log.describe:
-                log.describe('skipping rule %s because if-content="%s" does not match anything'
-                             % (self, self.if_content))
             return False
         return True
 
@@ -362,7 +359,6 @@
         """
         Applies this action to the theme_doc.
         """
-        describe = log.describe
         if self.content_href:
             ## FIXME: check response type
             content_doc = resource_fetcher(self.content_href).body
@@ -378,10 +374,6 @@
             else:
                 log_meth = log.warn
             log_meth(self, 'skipping rule because no content matches rule content="%s"', self.content)
-            if describe:
-                describe(
-                    'skipping rule %s because content="%s" does not match anything'
-                    % (self.describe_self(), html_quote(self.content)))
             return
         theme_type, theme_els, theme_attributes = self.select_elements(self.theme, theme_doc, theme=True)
         attributes = self.join_attributes(content_attributes, theme_attributes)
@@ -393,9 +385,6 @@
             else:
                 log_meth = log.warn
             log_meth(self, 'skipping rule because no theme element matches rule theme="%s"', self.theme)
-            if describe:
-                describe('skipping rule %s because theme="%s" does not match anything'
-                         % (self.describe_self(), html_quote(self.content)))
             return
         if len(theme_els) > 1:
             if self.manytheme[0] == 'warn':
@@ -448,7 +437,6 @@
     name = 'replace'
  
     def apply_transformation(self, content_type, content_els, attributes, theme_type, theme_el, log):
-        describe = log.describe
         if theme_type == 'children':
             existing_children = len(theme_el) or theme_el.text
             theme_el[:] = []
@@ -463,27 +451,10 @@
                     for el in content_els:
                         el.tail = None
                 theme_el.extend(content_els)
-                if describe:
-                    if existing_children:
-                        extra = ' and removed its children'
-                    else:
-                        extra = ''
-                    describe(
-                        "Rule %s moved elements %s into element %s%s"
-                        % (self.describe_self(), self.describe_content_elements(content_els), self.describe_theme_element(theme_el), extra))
             elif content_type == 'children':
                 text, els = self.prepare_content_children(content_els)
                 add_text(theme_el, text)
                 theme_el.extend(els)
-                if describe:
-                    if existing_children:
-                        extra = ' and removed its children'
-                    else:
-                        extra = ''
-                    describe(
-                        "Rule %s moved the children of elements %s into element %s%s"
-                        % (self.describe_self(), self.describe_content_elements(content_els, children=True),
-                           self.describe_theme_element(theme_el), extra))
                 if self.move:
                     # Since we moved just the children of the content elements, we still need to remove the parent
                     # elements.
@@ -577,7 +548,6 @@
         ]
 
     def apply_transformation(self, content_type, content_els, attributes, theme_type, theme_el, log):
-        describe = log.describe
         if theme_type == 'children':
             if content_type == 'elements':
                 if self.move:
@@ -702,7 +672,6 @@
         self.notheme = self.convert_error('notheme', notheme)
 
     def apply(self, content_doc, theme_doc, resource_fetcher, log):
-        describe = log.describe
         if not self.if_content_matches(content_doc, log):
             return
         for doc, selector, error, name in [(theme_doc, self.theme, self.notheme, 'theme'), (content_doc, self.content, self.nocontent, 'content')]:

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_middleware.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_middleware.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_middleware.txt	Tue Jul  1 19:40:41 2008
@@ -92,7 +92,7 @@
     >>> import logging
     >>> deliv = DeliveranceMiddleware(app, SubrequestRuleGetter('/rules.xml'),
     ...                               PrintingLogger,
-    ...                               log_factory_kw=dict(print_level=logging.WARNING, description=False))
+    ...                               log_factory_kw=dict(print_level=logging.WARNING))
 
 Now lets look at some plain content and its deliverated equivalent
 

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt	Tue Jul  1 19:40:41 2008
@@ -11,7 +11,7 @@
     >>> def match(matcher, request, response_headers, show_log=True):
     ...     if isinstance(matcher, basestring):
     ...         matcher = make(matcher)
-    ...     log = SavingLogger(None)
+    ...     log = SavingLogger(None, None)
     ...     if isinstance(response_headers, list):
     ...         response_headers = HeaderDict(response_headers)
     ...     result = matcher(request, response_headers, log)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_selection.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_selection.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_selection.txt	Tue Jul  1 19:40:41 2008
@@ -95,7 +95,7 @@
     ...     rule = parse_action(rule, None)
     ...     theme_copy = copy.deepcopy(theme)
     ...     theme_copy.make_links_absolute()
-    ...     logger = SavingLogger(request=None)
+    ...     logger = SavingLogger(request=None, middleware=None)
     ...     content_copy = copy.deepcopy(content)
     ...     rule.apply(content_copy, theme_copy, None, logger)
     ...     remove_content_attribs(theme_copy)

From ianb at codespeak.net  Tue Jul  1 22:21:16 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  1 Jul 2008 22:21:16 +0200 (CEST)
Subject: [z3-checkins] r56219 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance: .
	tests/example-files
Message-ID: <20080701202116.3250E169F10@codespeak.net>

Author: ianb
Date: Tue Jul  1 22:21:15 2008
New Revision: 56219

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml
Log:
Improved the log formatting, and the selection previewing

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	Tue Jul  1 22:21:15 2008
@@ -9,6 +9,8 @@
 from pygments.lexers import XmlLexer, HtmlLexer, guess_lexer_for_filename
 from pygments.formatters import HtmlFormatter
 from tempita import HTMLTemplate, html_quote, html
+from lxml.etree import _Element
+from lxml.html import fromstring, document_fromstring, tostring
 
 class DeliveranceMiddleware(object):
 
@@ -87,8 +89,7 @@
         if selector:
             url += '#deliverance-selection'
         if line:
-            ## FIXME: figure out line anchor
-            pass
+            url += '#code-%s' % line
         return url
 
     def internal_app(self, req, resource_fetcher):
@@ -110,7 +111,7 @@
         subresp = resource_fetcher(url)
         if source:
             ct = subresp.content_type
-            if ct.content_type.startswith('application/xml'):
+            if ct.startswith('application/xml'):
                 lexer = XmlLexer()
             elif ct == 'text/html':
                 lexer = HtmlLexer()
@@ -119,10 +120,9 @@
                 lexer = HtmlLexer()
             text = pygments_highlight(
                 subresp.body, lexer,
-                HtmlFormatter(linenos=linenos, lineanchors='code'))
+                HtmlFormatter(full=True, linenos=True, lineanchors='code'))
         else:
             from deliverance.selector import Selector
-            from lxml.html import fromstring, document_fromstring, tostring
             doc = document_fromstring(subresp.body)
             selector = Selector.parse(selector)
             type, elements, attributes = selector(doc)
@@ -131,25 +131,44 @@
             else:
                 template = self._found_template
             all_elements = []
+            els_in_head = False
             for index, el in enumerate(elements):
+                el_in_head = self._el_in_head(el)
+                if el_in_head:
+                    els_in_head = True
                 anchor = 'deliverance-selection'
                 if index:
                     anchor += '-%s' % index
+                if el.get('id'):
+                    anchor = el.get('id')
                 ## FIXME: is a <a name> better?
-                el.set('id', anchor)
+                if not el_in_head:
+                    el.set('id', anchor)
+                else:
+                    anchor = None
                 ## FIXME: add :target CSS rule
                 ## FIXME: or better, some Javascript
                 all_elements.append((anchor, el))
-                style = el.get('style', '')
-                if style:
-                    style += '; '
-                style += '/* deliverance */ border: 2px dotted #f00'
-                el.set('style', style)
+                if not el_in_head:
+                    style = el.get('style', '')
+                    if style:
+                        style += '; '
+                    style += '/* deliverance */ border: 2px dotted #f00'
+                    el.set('style', style)
+                else:
+                    el.set('DELIVERANCE-MATCH', '1')
+            def highlight(html_code):
+                if isinstance(html_code, _Element):
+                    html_code = tostring(html_code)
+                return html(pygments_highlight(html_code, HtmlLexer(),
+                                               HtmlFormatter(noclasses=True)))
             def format_tag(tag):
-                return tostring(tag).split('>')[0]+'>'
+                return highlight(tostring(tag).split('>')[0]+'>')
             text = template.substitute(
-                elements=all_elements, selector=selector, format_tag=format_tag)
-            message = fromstring(self._message_template.substitute(message=text))
+                els_in_head=els_in_head, doc=doc,
+                elements=all_elements, selector=selector, 
+                format_tag=format_tag, highlight=highlight)
+            message = fromstring(self._message_template.substitute(message=text, url=url))
             if doc.body.text:
                 message.tail = doc.body.text
                 doc.body.text = ''
@@ -158,6 +177,13 @@
         resp = Response(text)
         return resp
 
+    def _el_in_head(self, el):
+        while el is not None:
+            if el.tag == 'head':
+                return True
+            el = el.getparent()
+        return False
+
     _not_found_template = HTMLTemplate('''\
     There were no elements that matched the selector <code>{{selector}}</code>
     ''', 'deliverance.middleware.DeliveranceMiddleware._not_found_template')
@@ -165,19 +191,37 @@
     _found_template = HTMLTemplate('''\
     {{if len(elements) == 1}}
       One element matched the selector <code>{{selector}}</code>;
-      <a href="#{{elements[0][0]}}">jump to element</a>
+      {{if elements[0][0]}}
+        <a href="#{{elements[0][0]}}">jump to element</a>
+      {{else}}
+        element is in head: {{highlight(elements[0][1])}}
+      {{endif}}
     {{else}}
       {{len(elements)}} elements matched the selector <code>{{selector}}</code>:
       <ol>
       {{for anchor, el in elements}}
-        <li><a href="#{{anchor}}"><code>{{format_tag(el)}}</code></a></li>
+        {{if anchor}}
+          <li><a href="#{{anchor}}"><code>{{format_tag(el)}}</code></a></li>
+        {{else}}
+          <li>{{format_tag(el)}}</li>
+        {{endif}}
       {{endfor}}
       </ol>
     {{endif}}
+    {{if els_in_head}}
+      <div style="border-top: 2px solid #000">
+        <b>Elements matched in head.  Showing head:</b><br>
+        <div style="margin: 1em; padding: 0.25em; background-color: #fff">
+        {{highlight(doc.head)}}
+        </div>
+      </div>
+    {{endif}}
     ''', 'deliverance.middleware.DeliveranceMiddleware._found_template')
 
     _message_template = HTMLTemplate('''\
     <div style="color: #000; background-color: #f90; border-bottom: 2px dotted #f00">
+    <span style="float: right; font-size: 65%"><button onclick="window.close()">close</button></span>
+    Viewing <code><a style="text-decoration: none" href="{{url}}">{{url}}</a></code><br>
     {{message|html}}
     </div>''', 'deliverance.middleware.DeliveranceMiddleware._message_template')
 

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	Tue Jul  1 22:21:15 2008
@@ -7,7 +7,9 @@
 from deliverance.util.converters import asbool, html_quote
 from deliverance.selector import Selector
 from lxml import etree
+from lxml.html import document_fromstring
 from tempita import html
+import copy
 
 CONTENT_ATTRIB = 'x-a-marker-attribute-for-deliverance'
 
@@ -137,6 +139,8 @@
             assert isinstance(value, tuple), (
                 "Unexpected value: %r (for attribute %s)" % (
                     value, attr))
+            if value == ('warn', 'first'):
+                return None
             handler, pos = value
             if pos == 'last':
                 text = '%s:%s' % (handler, pos)
@@ -194,29 +198,8 @@
     # Set to true in subclasses if the move attribute means something:
     move_supported = True
 
-    def describe_self(self):
-        return self.log_description(log=None)
-
     def __str__(self):
-        return self.describe_self()
-
-    def describe_content_elements(self, els, children=False):
-        """
-        A text description of a list of content elements, for use in
-        log messages and errors.
-        """
-        text = ', '.join(el.tag for el in els)
-        if children:
-            return 'children of %s' % text
-        else:
-            return text
-
-    def describe_theme_element(self, el):
-        """
-        A text description of a theme element, for use in log messages
-        and errors.
-        """
-        return el.tag
+        return self.log_description(log=None)
 
     @classmethod
     def compile_selector(cls, el, attr, source_location):
@@ -259,6 +242,7 @@
         originating in the content are not selectable).
         """
         type, elements, attributes = selector(doc)
+        orig_elements = list(elements)
         if theme:
             bad_els = []
             for el in elements:
@@ -277,15 +261,22 @@
             if log is None or url is None:
                 return body
             link = log.link_to(url, source=source, line=line, selector=selector)
-            return '<a href="%s">%s</a>' % (html_quote(link), body)
+            return '<a href="%s" target="_blank">%s</a>' % (html_quote(link), body)
         if log:
             request_url = log.request.url
         else:
             request_url = None
-        parts = ['&lt;%s' % linked_item(self.source_location, self.name)]
+        parts = ['&lt;%s' % linked_item(self.source_location, self.name, source=True)]
         if getattr(self, 'content', None):
             body = 'content="%s"' % html_quote(self.content)
-            parts.append(linked_item(request_url, body, selector=self.content))
+            if self.content_href:
+                if request_url:
+                    content_url = urlparse.urljoin(request_url, self.content_href)
+                else:
+                    content_url = self.content_href
+            else:
+                content_url = request_url
+            parts.append(linked_item(content_url, body, selector=self.content))
         if getattr(self, 'content_href', None):
             dest = self.content_href
             if request_url:
@@ -293,12 +284,12 @@
             body = 'href="%s"' % html_quote(self.content_href)
             parts.append(linked_item(dest, body, source=True))
         if self.move_supported and not getattr(self, 'move', False):
-            parts.append('move="1"')
+            parts.append('move="0"')
         v = getattr(self, 'nocontent', 'warn')
         if v != 'warn':
             parts.append(self.format_error('nocontent', v))
         v = getattr(self, 'manycontent', ('warn', None))
-        if v != ('warn', None):
+        if v != ('warn', 'first'):
             parts.append(self.format_error('manycontent', v))
         if getattr(self, 'theme', None):
             body = 'theme="%s"' % html_quote(self.theme)
@@ -308,12 +299,44 @@
         if v != 'warn':
             parts.append(self.format_error('notheme', v))
         v = getattr(self, 'manytheme', ('warn', None))
-        if v != ('warn', None):
+        if v != ('warn', 'first'):
             parts.append(self.format_error('manytheme', v))
         ## FIXME: add source_location
         return html(' '.join(parts) + ' /&gt;')
         
+    def format_tags(self, elements, include_name=True):
+        if not elements:
+            if include_name:
+                return 'no elements'
+            else:
+                return '(none)'
+        text = ', '.join('<%s>' % el.tag for el in elements)
+        if include_name:
+            if len(elements) > 1:
+                return 'elements %s' % text
+            else:
+                return 'element %s' % text
+        else:
+            return text
 
+    def format_tag(self, tag, include_name=False):
+        return self.format_tags([tag], include_name=include_name)
+
+    def format_attribute_names(self, attributes, include_name=True):
+        if not attributes:
+            if include_name:
+                return 'no attributes'
+            else:
+                return '(none)'
+        text = ', '.join(attributes)
+        if include_name:
+            if len(attributes) > 1:
+                return 'attributes %s' % text
+            else:
+                return 'attribute %s' % text
+        else:
+            return text
+        
 class TransformAction(AbstractAction):
     # Abstract class for the rules that move from the content to the theme (replace, append, prepend)
 
@@ -361,13 +384,17 @@
         """
         if self.content_href:
             ## FIXME: check response type
-            content_doc = resource_fetcher(self.content_href).body
+            ## FIXME: Join content_href with request url?
+            content_resp = resource_fetcher(self.content_href)
+            log.debug(self, 'Fetching resource from href="%s": %s',
+                      self.content_href, content_resp.status)
+            content_doc = document_fromstring(content_resp.body, base_url=self.content_href)
         if not self.if_content_matches(content_doc, log):
             return
         content_type, content_els, content_attributes = self.select_elements(self.content, content_doc, theme=False)
         if not content_els:
             if self.nocontent == 'abort':
-                log.debug(self, 'aborting theme because no content matches rule content="%s"', self.content)
+                log.debug(self, 'aborting theming because no content matches rule content="%s"', self.content)
                 raise AbortTheme('No content matches content="%s"' % self.content)
             elif self.nocontent == 'ignore':
                 log_meth = log.debug
@@ -379,6 +406,7 @@
         attributes = self.join_attributes(content_attributes, theme_attributes)
         if not theme_els:
             if self.notheme == 'abort':
+                log.debug(self, 'aborting theming because no theme elements match rule theme="%s"', self.theme)
                 raise AbortTheme('No theme element matches theme="%s"' % self.theme)
             elif self.notheme == 'ignore':
                 log_meth = log.debug
@@ -387,19 +415,25 @@
             log_meth(self, 'skipping rule because no theme element matches rule theme="%s"', self.theme)
             return
         if len(theme_els) > 1:
-            if self.manytheme[0] == 'warn':
-                log.warn(self, '%s elements match theme="%s", using the %s match',
-                         len(theme_els), self.theme, self.manytheme[1])
-                pass
-            elif self.manytheme[0] == 'abort':
+            if self.manytheme[0] == 'abort':
+                log.debug(self, 'aborting theming because %i elements (%s) match theme="%s"',
+                          len(theme_els), self.format_tags(theme_els, include_name=False), self.theme)
                 raise AbortTheme('Many elements match theme="%s"' % self.theme)
+            elif self.manytheme[0] == 'warn':
+                log_meth = log.warn
+            else:
+                log_meth = log.debug
             if self.manytheme[1] == 'first':
-                theme_els = [theme_els[0]]
+                theme_el = theme_els[0]
             else:
-                theme_els = [theme_els[-1]]
-        theme_el = theme_els[0]
+                theme_el = theme_els[-1]
+            log_meth(self, '%s elements match theme="%s", using the %s match',
+                     len(theme_els), self.theme, self.manytheme[1])
+        else:
+            theme_el = theme_els[0]
         if not self.move and theme_type in ('children', 'elements'):
-            self.log.debug(self, 'content elements are being copied into theme (not moved)')
+            ## FIXME: is this message necessary?
+            log.debug(self, 'content elements are being copied into theme (not moved)')
             content_els = copy.deepcopy(content_els)
         mark_content_els(content_els)
         self.apply_transformation(content_type, content_els, attributes, theme_type, theme_el, log)
@@ -439,6 +473,13 @@
     def apply_transformation(self, content_type, content_els, attributes, theme_type, theme_el, log):
         if theme_type == 'children':
             existing_children = len(theme_el) or theme_el.text
+            theme_empty = not len(theme_el) and not theme_el.text
+            if len(theme_el):
+                log_text = 'and removed the chilren and text of the theme element'
+            elif theme_el.text:
+                log_text = 'and removed the text content of the theme element'
+            else:
+                log_text = '(the theme was already empty)'
             theme_el[:] = []
             theme_el.text = ''
             if content_type == 'elements':
@@ -446,11 +487,15 @@
                     # If we aren't working with copies then we have to move the tails up as we remove the elements:
                     for el in reversed(content_els):
                         move_tail_upward(el)
+                    verb = 'Moving'
                 else:
                     # If we are working with copies, then we can just throw away the tails:
                     for el in content_els:
                         el.tail = None
+                    verb = 'Copying'
                 theme_el.extend(content_els)
+                log.debug(self, '%s %s from content into theme element %s %s',
+                          verb, self.format_tags(content_els), self.format_tag(theme_el), log_text)
             elif content_type == 'children':
                 text, els = self.prepare_content_children(content_els)
                 add_text(theme_el, text)
@@ -460,6 +505,12 @@
                     # elements.
                     for el in content_els:
                         el.getparent().remove(el)
+                    log.debug(self, 'Moving children of content %s into theme element %s, '
+                              'and removing now-empty content elements %s',
+                              self.format_tags(content_els), self.format_tag(theme_el), log_text)
+                else:
+                    log.debug(self, 'Copying children of content %s into theme element %s %s',
+                              self.format_tags(content_els), self.format_tag(theme_el), log_text)
             else:
                 assert 0
             
@@ -471,10 +522,14 @@
                 if self.move:
                     for el in reversed(content_els):
                         move_tail_upwards(el)
+                    verb = 'moved'
                 else:
                     for el in content_els:
                         el.tail = None
+                    verb = 'copied'
                 parent[pos:pos+1] = content_els
+                log.debug(self, 'Replaced the theme element %s with the content %s (%s)',
+                          self.format_tag(theme_el), self.format_tags(content_els), verb)
             elif content_type == 'children':
                 text, els = self.prepare_content_children(content_els)
                 if pos == 0:
@@ -485,6 +540,12 @@
                 if self.move:
                     for el in content_els:
                         el.getparent().remove(el)
+                    log.debug(self, 'Replaced the theme element %s with the children of the content %s, '
+                              'and removed the now-empty content element(s)',
+                              self.format_tag(theme_el), self.format_tags(content_els))
+                else:
+                    log.debug(self, 'Replaced the theme element %s with copies of the children of the content %s',
+                              self.format_tag(theme_el), self.format_tags(content_els))
             else:
                 assert 0
 
@@ -493,20 +554,25 @@
             assert content_type == 'attributes'
             if len(content_els) > 1:
                 if self.manycontent[0] == 'abort':
-                    log.debug(self, 'aborting because %s elements in the content match content="%s"',
-                              len(content_els), self.content)
+                    log.debug(self, 'aborting because %i elements in the content (%s) match content="%s"',
+                              len(content_els), self.format_tags(content_els, include_name=False), self.content)
                     raise AbortTheme()
                 else:
                     if self.manycontent[0] == 'warn':
                         log_meth = log.warn
                     else:
                         log_meth = log.debug
-                    log_meth(self, '%s elements match content="%s" (but only one expected), using the %s match',
-                             len(content_els, self.content, self.manycontent[1]))
+                    log_meth(self, '%s elements match content="%s" (%s) when only one is expected, using the %s match',
+                             len(content_els), self.content, self.format_tags(content_els, include_name=False), self.manycontent[1])
                     if self.manycontent[1] == 'first':
                         content_els = [content_els[0]]
                     else:
                         content_els = [content_els[-1]]
+            if theme_el.attrib:
+                log_text = ' and cleared all existing theme attributes'
+            else:
+                log_text = ''
+            ## FIXME: should this only clear the named attribute? (when attributes are named)
             theme_el.attrib.clear()
             if attributes:
                 c_attrib = content_els[0].attrib
@@ -517,17 +583,31 @@
                     for name in attributes:
                         if name in c_attrib:
                             del c_attrib[name]
+                    log_text += ' and removed the attributes from the content'
+                ## FIXME: only list attributes that were actually found?
+                log.debug(self, 'Copied the %s from the content element %s to the '
+                          'theme element %s%s',
+                          self.format_attribute_names(attributes), self.format_tag(content_els[0]), 
+                          self.format_tag(theme_el), log_text)
             else:
                 theme_el.attrib.update(content_els[0].attrib)
                 if self.move:
                     content_els[0].attrib.clear()
+                    log_text += ' and removed all attributes from the content'
+                log.debug(self, 'Moved all the attributes from the content element %s to the '
+                          'theme element %s%s',
+                          self.format_tag(content_els[0]), self.format_tag(theme_el), log_text)
 
         if theme_type == 'tag':
+            ## FIXME: warn about manycontent
             assert content_type == 'tag'
+            old_tag = theme_el.tag
             theme_el.tag = content_els[0].tag
             theme_el.attrib.clear()
             theme_el.attrib.update(content_els[0].attrib)
             # "move" in this case doesn't mean anything
+            log.debug(self, 'Changed the tag name of the theme element <%s> to the name of the content element: %s',
+                      old_tag, self.format_tag(content_els[0]))
 
 _actions['replace'] = Replace
 
@@ -553,15 +633,21 @@
                 if self.move:
                     for el in reversed(content_els):
                         move_tail_upwards(el)
+                    verb = 'Moving'
                 else:
                     for el in content_els:
                         el.tail = None
+                    verb = 'Copying'
                 if self._append:
                     theme_el.extend(content_els)
+                    pos_text = 'end'
                 else:
                     add_tail(content_els[-1], theme_el.text)
                     theme_el.text = None
                     theme_el[:0] = content_els
+                    pos_text = 'beginning'
+                log.debug(self, '%s content %s to the %s of theme element %s',
+                          verb, self.format_tags(content_els), pos_text, self.format_tag(theme_el))
             elif content_type == 'children':
                 text, els = self.prepare_content_children(content_els)
                 if self._append:
@@ -570,10 +656,23 @@
                     else:
                         add_text(theme_el, text)
                     theme_el.extend(els)
+                    pos_text = 'end'
                 else:
                     add_tail(els[-1], theme_el.text)
                     theme_el.text = text
                     theme_el[:0] = els
+                    pos_text = 'beginning'
+                if self.move:
+                    for el in reversed(content_els):
+                        move_tail_upwards(el)
+                        el.getparent().remove(el)
+                    verb = 'Moving'
+                    log_text = ' and removing the now-empty content element(s)'
+                else:
+                    verb = 'Copying'
+                    log_text = ''
+                log.debug(self, '%s the children of content %s to the %s of the theme element %s%s',
+                          verb, self.format_tags(content_els), pos_text, self.format_tag(theme_el), log_text)
             else:
                 assert 0
 
@@ -584,61 +683,120 @@
                 if self.move:
                     for el in reversed(content_els):
                         move_tail_upwards(el)
+                    verb = 'Moving'
                 else:
                     for el in content_els:
                         el.tail = None
+                    verb = 'Copying'
                 if self._append:
                     parent[pos+1:pos+1] = content_els
+                    pos_text = 'after'
                 else:
                     parent[pos:pos] = content_els
+                    pos_text = 'before'
+                log.debug(self, '%s content %s %s the theme element %s',
+                          verb, self.format_tags(content_els), pos_text, self.format_tag(theme_el))
             elif content_type == 'children':
                 text, els = self.prepare_content_children(content_els)
                 if self._append:
                     add_tail(theme_el, text)
                     parent[pos+1:pos+1] = content_els
+                    pos_text = 'after'
                 else:
                     if pos == 0:
                         add_text(parent, text)
                     else:
                         add_tail(parent[pos-1], text)
                     parent[pos:pos] = content_els
+                    pos_text = 'before'
+                if self.move:
+                    for el in reversed(content_els):
+                        move_tail_upwards(el)
+                        el.getparent().remove(el)
+                    verb = 'Moving'
+                    log_text = ' and removing the now-empty content element(s)'
+                else:
+                    verb = 'Copying'
+                    log_text = ''
+                log.debug(self, '%s the children of content %s %s the theme element %s%s',
+                          verb, self.format_tags(content_els), pos_text, self.format_tag(theme_el), log_text)
 
         if theme_type == 'attributes':
             ## FIXME: handle named attributes
             assert content_type == 'attributes'
             if len(content_els) > 1:
                 if self.manycontent[0] == 'abort':
-                    log.debug(self, 'aborting because %s elements in the content match content="%s"',
-                              len(content_els), self.content)
+                    log.debug(self, 'aborting because %i elements in the content (%s) match content="%s"',
+                              len(content_els), self.format_tags(content_els), self.content)
                     raise AbortTheme()
                 else:
                     if self.manycontent[0] == 'warn':
                         log_meth = log.warn
                     else:
                         log_meth = log.debug
-                    log_meth(self, '%s elements match content="%s" (but only one expected), using the %s match',
-                             len(content_els, self.content, self.manycontent[1]))
+                    log_meth(self, '%s elements match content="%s" (%s) but only one is expected, using the %s match',
+                             len(content_els), self.content, self.format_tags(content_els), self.manycontent[1])
                     if self.manycontent[1] == 'first':
                         content_els = [content_els[0]]
                     else:
                         content_els = [content_els[-1]]
             content_attrib = content_els[0].attrib
             theme_attrib = theme_el.attrib
+            if self.move:
+                verb = 'Moved'
+            else:
+                verb = 'Copied'
             if self._append:
                 if attributes:
+                    avoided_attrs = []
+                    copied_attrs = []
                     for name in attributes:
                         if name in content_attrib:
-                            theme_attrib.setdefault(name, content_attrib[name])
+                            if name in theme_attrib:
+                                avoided_attrs.append(name)
+                            else:
+                                theme_attrib[name] = content_attrib[name]
+                                copied_attrs.append(name)
+                    if avoided_attrs:
+                        log.debug(self, '%s %s from the content element %s to the theme element %s, '
+                                  'and did not copy the %s because they were already '
+                                  'present in the theme element',
+                                  verb, self.format_attribute_names(copied_attrs), self.format_tag(content_els[0]), 
+                                  self.format_tag(theme_el), self.format_attribute_names(avoided_attrs))
+                    else:
+                        log.debug(self, '%s %s from the content element %s to the theme '
+                                  'element %s',
+                                  verb, self.format_attribute_names(copied_attrs), self.format_tag(content_els[0]),
+                                  self.format_tag(theme_el))
                 else:
+                    avoided_attrs = []
+                    copied_attrs = []
                     for key, value in content_attrib.items():
-                        theme_attrib.setdefault(key, value)
+                        if key in theme_attrib:
+                            avoided_attrs.append(key)
+                        else:
+                            copied_attrs.append(key)
+                            theme_attrib[key] = value
+                    if avoided_attrs:
+                        log.debug(self, '%s %s from the content element %s to the theme element %s, '
+                                  'and did not copy the %s because they were already present in the theme element',
+                                  verb, self.format_attribute_names(copied_attrs), self.format_tag(content_els[0]), 
+                                  self.format_tag(theme_el), self.format_attribute_names(avoided_attrs))
+                    else:
+                        log.debug(self, '%s %s from the content element %s to the theme element %s',
+                                  verb, self.format_attribute_names(copied_attrs), self.format_tag(content_els[0]), 
+                                  self.format_tab(theme_el))
             else:
                 if attributes:
                     for name in attributes:
                         if name in content_attrib:
                             theme_attrib.set(name, content_attrib[name])
+                    log.debug(self, '%s %s from the content element %s to the theme element %s',
+                              verb, self.format_attribute_names(attributes), self.format_tag(content_els[0]), self.format_tag(theme_el))
                 else:
                     theme_attrib.update(content_attrib)
+                    log.debug(self, '%s all the attributes from the content element %s to the theme element %s',
+                              verb, self.format_tag(content_els[0]), self.format_tag(theme_el))
             if self.move:
                 if attributes:
                     for name in attributes:
@@ -692,30 +850,43 @@
                 for el in els:
                     move_tail_upwards(el)
                     el.getparent().remove(el)
+                log.debug(self, 'Dropping %s %s', name, self.format_tags(els))
             elif sel_type == 'children':
-                el[:] = []
-                el.text = ''
+                for el in els:
+                    el[:] = []
+                    el.text = ''
+                log.debug(self, 'Dropping the children of %s %s', name, self.format_tags(els))
             elif sel_type == 'attributes':
-                attrib = el.attrib
+                for el in els:
+                    attrib = el.attrib
+                    if attributes:
+                        for name in attributes:
+                            if name in attrib:
+                                del attrib[name]
+                    else:
+                        attrib.clear()
                 if attributes:
-                    for name in attributes:
-                        if name in attrib:
-                            del attrib[name]
+                    log.debug(self, 'Dropping the %s from the %s %s',
+                              self.format_attribute_names(attributes), name, self.format_tags(els))
                 else:
-                    attrib.clear()
+                    log.debug(self, 'Dropping all the attributes of %s %s',
+                              name, self.format_tags(els))
             elif sel_type == 'tag':
-                children = list(el)
-                if children:
-                    add_tail(children[-1], el.tail)
-                else:
-                    add_text(el, el.tail)
-                parent = el.getparent()
-                pos = parent.index(el)
-                if pos == 0:
-                    add_text(parent, el.text)
-                else:
-                    add_tail(parent[pos-1], el.text)
-                parent[pos:pos+1] = children
+                for el in els:
+                    children = list(el)
+                    if children:
+                        add_tail(children[-1], el.tail)
+                    else:
+                        add_text(el, el.tail)
+                    parent = el.getparent()
+                    pos = parent.index(el)
+                    if pos == 0:
+                        add_text(parent, el.text)
+                    else:
+                        add_tail(parent[pos-1], el.text)
+                    parent[pos:pos+1] = children
+                log.debug(self, 'Dropping the tag (flattening the element) of %s %s',
+                          name, self.format_tags(els))
             else:
                 assert 0
 

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	Tue Jul  1 22:21:15 2008
@@ -9,10 +9,12 @@
 
 class RuleSet(object):
 
-    def __init__(self, matchers, rules_by_class, default_theme=None):
+    def __init__(self, matchers, rules_by_class, default_theme=None,
+                 source_location=None):
         self.matchers = matchers
         self.rules_by_class = rules_by_class
         self.default_theme = default_theme
+        self.source_location = source_location
 
     def apply_rules(self, req, resp, resource_fetcher, log):
         extra_headers = parse_meta_headers(resp.body)
@@ -67,6 +69,14 @@
     def parse_document(self, s, url):
         return document_fromstring(s, base_url=url)
 
+    def log_description(self, log=None):
+        if log is None:
+            name = 'ruleset'
+        else:
+            name = '<a href="%s" target="_blank">ruleset</a>' % log.link_to(self.source_location, source=True)
+        desc = '&lt;%s&gt;' % name
+        return desc
+
     @classmethod
     def parse_xml(cls, doc, source_location):
         assert doc.tag == 'ruleset'
@@ -94,7 +104,8 @@
                 rules_by_class.setdefault(class_name, []).append(rule)
         if default_theme:
             default_theme = urlparse.urljoin(doc.base, default_theme)
-        return cls(matchers, rules_by_class, default_theme=default_theme)
+        return cls(matchers, rules_by_class, default_theme=default_theme,
+                   source_location=source_location)
 
 _meta_tag_re = re.compile(r'<meta\s+(.*?)>', re.I | re.S)
 _http_equiv_re = re.compile(r'http-equiv=(?:"([^"]*)"|([^\s>]*))', re.I|re.S)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml	Tue Jul  1 22:21:15 2008
@@ -2,6 +2,6 @@
  <theme href="/theme.html" />
  <rule>
   <replace content="children:body" theme="children:#content" nocontent="abort" />
-  <replace content="children:title" theme="children:#header" nocontent="ignore" />
+  <replace content="children:title" theme="children:#header" nocontent="ignore" move="0" />
  </rule>
 </ruleset>

From ianb at codespeak.net  Tue Jul  1 23:21:08 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  1 Jul 2008 23:21:08 +0200 (CEST)
Subject: [z3-checkins] r56220 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance
Message-ID: <20080701212108.67A71169E21@codespeak.net>

Author: ianb
Date: Tue Jul  1 23:21:06 2008
New Revision: 56220

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
Log:
Add some links to the log; remove an unnecessary log message

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	Tue Jul  1 23:21:06 2008
@@ -55,6 +55,12 @@
     log_template = HTMLTemplate('''\
     <h1 style="border-top: 3px dotted #f00">Deliverance Information</h1>
 
+    <div>
+      <a href="{{theme_url}}">theme: {{log.theme_url}}</a>
+      | <a href="{{unthemed_url}}">unthemed content</a>
+      | <a href="{{content_source}}">content source</a>
+    </div>
+
     {{if log.messages}}
       {{div}}
       {{h2}}Log</h2>
@@ -86,8 +92,20 @@
         )
 
     def format_html_log(self):
+        content_source = self.link_to(self.request.url, source=True)
         return self.log_template.substitute(
-            log=self, middleware=self.middleware, **self.tags)
+            log=self, middleware=self.middleware, 
+            unthemed_url=self._add_notheme(self.request.url),
+            theme_url=self._add_notheme(self.theme_url),
+            content_source=content_source,
+            **self.tags)
+
+    def _add_notheme(self, url):
+        if '?' in url:
+            url += '&'
+        else:
+            url += '?'
+        return url + 'deliv_notheme'
 
     def resolved_messages(self):
         for level, el, msg in self.messages:

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	Tue Jul  1 23:21:06 2008
@@ -26,6 +26,8 @@
     def __call__(self, environ, start_response):
         ## FIXME: copy_get?:
         req = Request(environ)
+        if 'deliv_notheme' in req.GET:
+            return self.app(environ, start_response)
         req.environ['deliverance.base_url'] = req.application_url
         orig_req = Request(environ.copy())
         log = self.log_factory(req, self, **self.log_factory_kw)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	Tue Jul  1 23:21:06 2008
@@ -432,8 +432,6 @@
         else:
             theme_el = theme_els[0]
         if not self.move and theme_type in ('children', 'elements'):
-            ## FIXME: is this message necessary?
-            log.debug(self, 'content elements are being copied into theme (not moved)')
             content_els = copy.deepcopy(content_els)
         mark_content_els(content_els)
         self.apply_transformation(content_type, content_els, attributes, theme_type, theme_el, log)

From ianb at codespeak.net  Wed Jul  2 02:57:27 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Wed,  2 Jul 2008 02:57:27 +0200 (CEST)
Subject: [z3-checkins] r56223 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance: .
	tests/example-files
Message-ID: <20080702005727.071BD2A018D@codespeak.net>

Author: ianb
Date: Wed Jul  2 02:57:25 2008
New Revision: 56223

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/index.html
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html
Log:
Added some more hooks for the <theme> element.  Fixed the href attribute on rules.  Fancied up the example

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	Wed Jul  2 02:57:25 2008
@@ -56,9 +56,9 @@
     <h1 style="border-top: 3px dotted #f00">Deliverance Information</h1>
 
     <div>
-      <a href="{{theme_url}}">theme: {{log.theme_url}}</a>
-      | <a href="{{unthemed_url}}">unthemed content</a>
-      | <a href="{{content_source}}">content source</a>
+      <a href="{{theme_url}}" target="_blank">theme: {{log.theme_url}}</a>
+      | <a href="{{unthemed_url}}" target="_blank">unthemed content</a>
+      | <a href="{{content_source}}" target="_blank">content source</a>
     </div>
 
     {{if log.messages}}

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	Wed Jul  2 02:57:25 2008
@@ -70,7 +70,7 @@
         else:
             ## FIXME: pluggable subrequest handler?
             subreq = Request.blank(url)
-            resp = subreq.get_response(proxy_exact_request)
+            subresp = subreq.get_response(proxy_exact_request)
             log.debug(self, 'External request for %s: %s content-type: %s',
                       url, subresp.status, subresp.content_type)
             return subresp

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	Wed Jul  2 02:57:25 2008
@@ -33,7 +33,8 @@
         """
         Creates an instance of Match from the given parsed XML element.
         """
-        assert el.tag == 'match'
+        assert (el.tag == 'match'
+                or el.tag == 'rule')
         classes = el.get('class', '').split()
         abort = asbool(el.get('abort'))
         if not abort and not classes:

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	Wed Jul  2 02:57:25 2008
@@ -6,10 +6,13 @@
 from deliverance.exceptions import add_exception_info, DeliveranceSyntaxError
 from deliverance.util.converters import asbool, html_quote
 from deliverance.selector import Selector
+from deliverance.pagematch import Match
+from deliverance.themeref import Theme
 from lxml import etree
 from lxml.html import document_fromstring
 from tempita import html
 import copy
+import urlparse
 
 CONTENT_ATTRIB = 'x-a-marker-attribute-for-deliverance'
 
@@ -18,13 +21,17 @@
     This represents everything in a <rule></rule> section.
     """
 
-    def __init__(self, classes, actions, theme, suppress_standard, source_location):
+    def __init__(self, classes, actions, theme, match, suppress_standard, source_location):
         self.classes = classes
         self._actions = actions
         self.theme = theme
+        self.match = match
         self.suppress_standard = suppress_standard
         self.source_location = source_location
 
+    match_attrs = set([
+            'path', 'domain', 'request-header', 'response-header', 'environ'])
+
     @classmethod
     def parse_xml(cls, el, source_location):
         """
@@ -40,14 +47,27 @@
         for el in el.iterchildren():
             if el.tag == 'theme':
                 ## FIXME: error if more than one theme
-                ## FIXME: error if no href
-                theme = el.get('href')
+                theme = Theme.parse_xml(el, source_location)
                 continue
             if el.tag is etree.Comment:
                 continue
             action = parse_action(el, source_location)
             actions.append(action)
-        return cls(classes, actions, theme, suppress_standard, source_location)
+        match = None
+        for attr in cls.match_attrs:
+            if el.get(attr):
+                match = Match.parse_xml(el, source_location)
+                if match.abort:
+                    raise DeliveranceSyntaxError(
+                        "You cannot have an abort attribute on <rule> elements",
+                        element=el)
+                if match.last:
+                    ## FIXME: is last a good alternative to suppress-standard?
+                    raise DeliveranceSyntaxError(
+                        "You cannot have a last attribute on <rule> elements",
+                        element=el)
+                break
+        return cls(classes, actions, theme, match, suppress_standard, source_location)
 
     def apply(self, content_doc, theme_doc, resource_fetcher, log):
         """
@@ -383,11 +403,15 @@
         Applies this action to the theme_doc.
         """
         if self.content_href:
-            ## FIXME: check response type
-            ## FIXME: Join content_href with request url?
-            content_resp = resource_fetcher(self.content_href)
+            ## FIXME: Is this a weird way to resolve the href?
+            href = urlparse.urljoin(log.request.url, self.content_href)
+            content_resp = resource_fetcher(href)
             log.debug(self, 'Fetching resource from href="%s": %s',
-                      self.content_href, content_resp.status)
+                      href, content_resp.status)
+            if content_resp.status_int != 200:
+                log.warn(self, 'Resource %s returned the status %s; skipping rule',
+                         href, content_resp.status)
+                return
             content_doc = document_fromstring(content_resp.body, base_url=self.content_href)
         if not self.if_content_matches(content_doc, log):
             return

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	Wed Jul  2 02:57:25 2008
@@ -1,6 +1,7 @@
 from deliverance.exceptions import AbortTheme, DeliveranceSyntaxError
 from deliverance.pagematch import run_matches, Match
 from deliverance.rules import Rule, remove_content_attribs
+from deliverance.themeref import Theme
 from lxml.html import tostring, document_fromstring
 from lxml.etree import XML
 import re
@@ -26,6 +27,8 @@
             classes = run_matches(self.matchers, req, response_headers, log)
         except AbortTheme:
             return resp
+        if 'X-Deliverance-Page-Class' in resp.headers:
+            classes.extend(resp.headers['X-Deliverance-Page-Class'].strip().split())
         if not classes:
             classes = ['default']
         rules = []
@@ -42,10 +45,16 @@
             theme = self.default_theme
             ## FIXME: error if not theme still
         assert theme is not None
-        theme_doc = self.get_theme(theme, resource_fetcher, log)
+        theme_href = theme.resolve_href(req, resp)
+        theme_doc = self.get_theme(theme_href, resource_fetcher, log)
         content_doc = self.parse_document(resp.body, req.url)
         run_standard = True
         for rule in rules:
+            if rule.match is not None:
+                matches = rule.match(req, response_headers, log)
+                if not matches:
+                    log.debug(rule, "Skipping <rule>")
+                    continue
             rule.apply(content_doc, theme_doc, resource_fetcher, log)
             if rule.suppress_standard:
                 run_standard = False
@@ -92,7 +101,7 @@
                 rules.append(rule)
             elif el.tag == 'theme':
                 ## FIXME: Add parse error
-                default_theme = el.get('href')
+                default_theme = Theme.parse_xml(el, source_location)
             else:
                 ## FIXME: source location?
                 raise DeliveranceSyntaxError(
@@ -102,8 +111,8 @@
         for rule in rules:
             for class_name in rule.classes:
                 rules_by_class.setdefault(class_name, []).append(rule)
-        if default_theme:
-            default_theme = urlparse.urljoin(doc.base, default_theme)
+        if default_theme and default_theme.href:
+            default_theme.href = urlparse.urljoin(doc.base, default_theme.href)
         return cls(matchers, rules_by_class, default_theme=default_theme,
                    source_location=source_location)
 

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/index.html
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/index.html	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/index.html	Wed Jul  2 02:57:25 2008
@@ -5,6 +5,20 @@
 
 <body>
 
-This is my awesome site!
+<p>
+Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Proin ullamcorper sem at orci. Nulla facilisi. Aenean bibendum suscipit diam. Maecenas vel pede vitae lorem tempor sollicitudin. Quisque pretium sapien eget dolor. In metus sem, facilisis in, laoreet in, porta luctus, odio. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Nam viverra felis quis mauris. Phasellus vel quam sed arcu ullamcorper bibendum. Phasellus posuere tellus pellentesque ligula. Suspendisse volutpat pede rhoncus nisi. Vivamus interdum odio eu odio. Nulla felis sapien, pretium sed, facilisis quis, fringilla et, leo. Suspendisse potenti. Praesent quis tellus. Aliquam cursus, ipsum in sollicitudin commodo, dui lacus varius ipsum, mattis fermentum sapien nisl sit amet mauris. Nullam molestie. In commodo viverra nulla. Curabitur vitae velit id erat pellentesque consectetuer. Nulla quis quam.
+
+<p>
+Maecenas venenatis, lectus a luctus porttitor, ipsum pede pretium purus, et vehicula elit lacus a est. Proin ipsum magna, commodo sed, adipiscing et, imperdiet at, orci. Morbi molestie nisi in dui. Donec ultricies dui nec leo gravida lobortis. Donec lobortis dolor nec urna vulputate adipiscing. Vestibulum facilisis commodo est. Donec ornare, odio a rhoncus ornare, purus purus pellentesque dui, vitae iaculis dolor justo ultrices mauris. Donec eget libero. Ut tellus. Donec eget justo. In tempor augue sit amet ante. Donec vitae urna at eros vestibulum faucibus.
+
+<p>
+Praesent quam. Curabitur nec turpis. Donec dictum. Proin pellentesque, diam luctus convallis fermentum, mi magna consectetuer sapien, quis semper arcu nunc eget arcu. Nunc suscipit semper nunc. Sed felis metus, adipiscing a, vulputate nec, dapibus vitae, diam. Sed tristique nisi in tellus. Ut odio ipsum, ultricies placerat, vehicula nec, tempor non, mi. Vestibulum tortor. Mauris mollis augue ut nunc. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Mauris lacus. Donec ac odio ac mi euismod feugiat. Duis mauris velit, porttitor aliquam, faucibus ac, hendrerit sit amet, orci. Pellentesque ultricies pede sodales metus. Vestibulum mauris. Phasellus non leo. In eu magna eu sapien molestie porttitor. Nunc malesuada bibendum ligula. Etiam sagittis tristique neque.
+
+<p>
+Maecenas velit eros, accumsan porta, sodales in, ultrices auctor, felis. In metus nisi, mattis vel, bibendum id, placerat vel, elit. Vestibulum vestibulum. Sed auctor. Fusce lobortis. Vivamus porta cursus dui. Etiam neque tortor, egestas sed, ornare in, gravida eu, urna. Vestibulum a pede. In viverra sodales urna. Aliquam tempus molestie massa. Nullam congue viverra felis.
+
+<p>
+Nunc lacus odio, lacinia a, mattis quis, aliquam in, lectus. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Sed vel odio. Suspendisse ut est eu mauris auctor interdum. Sed et leo vel turpis condimentum elementum. Morbi neque mauris, sagittis eu, posuere id, molestie et, odio. Curabitur tortor. Curabitur eu pede vitae ipsum consequat porttitor. Nam at massa sed nulla lacinia sollicitudin. Donec dapibus. Duis felis justo, tincidunt sed, aliquet sit amet, rhoncus ut, neque. Duis id turpis at sem tincidunt adipiscing. Sed ante pede, facilisis a, sollicitudin sit amet, malesuada id, erat. Cras non purus. 
+
 
 </body> </html>

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml	Wed Jul  2 02:57:25 2008
@@ -3,5 +3,6 @@
  <rule>
   <replace content="children:body" theme="children:#content" nocontent="abort" />
   <replace content="children:title" theme="children:#header" nocontent="ignore" move="0" />
+  <replace content="children:#menu" href="/sidebar.html" theme="children:#sidebar" />
  </rule>
 </ruleset>

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css	Wed Jul  2 02:57:25 2008
@@ -1,5 +1,7 @@
 body {
   font-family: sans-serif;
+  padding: 0;
+  margin: 0;
 }
 
 #footer {
@@ -7,3 +9,28 @@
   font-size: 70%;
   margin-top: 1em;
 }
+
+table#table-layout {
+  border-spacing: 0;
+}
+
+table#table-layout td {
+  vertical-align: top;
+}
+
+td#sidebar {
+  background-color: #bfb;
+  border-right: 2px solid #0f0;
+  width: 8em;
+  padding-top: 1em;
+}
+
+td#content {
+  border-top: 2px solid #0f0;
+  padding: 0.5em;
+}
+
+h1#header {
+  background-color: #bfb;
+  margin-bottom: 0;
+}

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html	Wed Jul  2 02:57:25 2008
@@ -6,8 +6,17 @@
 
 <body>
 <h1 id="header">A Page</h1>
-
-<div id="content">Some content</div>
+<!-- table-based layout, so retro of me! -->
+<table width="100%" id="table-layout">
+  <tr>
+    <td id="sidebar">
+      Some sidebar links.
+    </td>
+    <td id="content">
+      Some content
+    </td>
+  </tr>
+</table>
 
 <div id="footer">Copyright (C) 2000 Some Corporation</div>
 

From ianb at codespeak.net  Wed Jul  2 03:10:52 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Wed,  2 Jul 2008 03:10:52 +0200 (CEST)
Subject: [z3-checkins] r56224 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance
Message-ID: <20080702011052.B224C2A018D@codespeak.net>

Author: ianb
Date: Wed Jul  2 03:10:52 2008
New Revision: 56224

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
Log:
But <base href> in page previews, and fix up the anchors so that the base href doesn't effect them.

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	Wed Jul  2 03:10:52 2008
@@ -10,7 +10,8 @@
 from pygments.formatters import HtmlFormatter
 from tempita import HTMLTemplate, html_quote, html
 from lxml.etree import _Element
-from lxml.html import fromstring, document_fromstring, tostring
+from lxml.html import fromstring, document_fromstring, tostring, Element
+import posixpath
 
 class DeliveranceMiddleware(object):
 
@@ -75,7 +76,6 @@
                       url, subresp.status, subresp.content_type)
             return subresp
 
-
     def link_to(self, req, url, source=False, line=None, selector=None):
         base = req.environ['deliverance.base_url']
         base += '/.deliverance/view'
@@ -126,6 +126,9 @@
         else:
             from deliverance.selector import Selector
             doc = document_fromstring(subresp.body)
+            el = Element('base')
+            el.set('href', posixpath.dirname(url) + '/')
+            doc.head.insert(0, el)
             selector = Selector.parse(selector)
             type, elements, attributes = selector(doc)
             if not elements:
@@ -167,6 +170,7 @@
             def format_tag(tag):
                 return highlight(tostring(tag).split('>')[0]+'>')
             text = template.substitute(
+                base_url=req.url,
                 els_in_head=els_in_head, doc=doc,
                 elements=all_elements, selector=selector, 
                 format_tag=format_tag, highlight=highlight)
@@ -194,7 +198,7 @@
     {{if len(elements) == 1}}
       One element matched the selector <code>{{selector}}</code>;
       {{if elements[0][0]}}
-        <a href="#{{elements[0][0]}}">jump to element</a>
+        <a href="{{base_url}}#{{elements[0][0]}}">jump to element</a>
       {{else}}
         element is in head: {{highlight(elements[0][1])}}
       {{endif}}
@@ -203,7 +207,7 @@
       <ol>
       {{for anchor, el in elements}}
         {{if anchor}}
-          <li><a href="#{{anchor}}"><code>{{format_tag(el)}}</code></a></li>
+          <li><a href="{{base_url}}#{{anchor}}"><code>{{format_tag(el)}}</code></a></li>
         {{else}}
           <li>{{format_tag(el)}}</li>
         {{endif}}

From ianb at codespeak.net  Wed Jul  2 19:30:03 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Wed,  2 Jul 2008 19:30:03 +0200 (CEST)
Subject: [z3-checkins] r56244 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance: .
	tests tests/example-files util
Message-ID: <20080702173003.106962A0192@codespeak.net>

Author: ianb
Date: Wed Jul  2 19:30:02 2008
New Revision: 56244

Added:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/sidebar.html   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/importstring.py   (contents, props changed)
Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/exceptions.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt
Log:
Factored out the python references.  Made an abstract superclass for matching (separate from the Match/<match> object).  Added python references to matching in addition to theming.

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/exceptions.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/exceptions.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/exceptions.py	Wed Jul  2 19:30:02 2008
@@ -8,10 +8,11 @@
     attached to it.  Elements are the objects (maybe XML, or maybe
     not) that is applicable.
     """
-    def __init__(self, msg=None, request=None, element=None):
+    def __init__(self, msg=None, request=None, element=None, source_location=None):
         Exception.__init__(self, msg)
         self.request = request
         self.element = element
+        self.source_location = source_location
 
 class DeliveranceSyntaxError(DeliveranceError):
     """

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	Wed Jul  2 19:30:02 2008
@@ -5,64 +5,65 @@
 from deliverance.exceptions import DeliveranceSyntaxError, AbortTheme
 from deliverance.stringmatch import compile_matcher, compile_header_matcher
 from deliverance.util.converters import asbool, html_quote
+from deliverance.pyref import PyReference, PyArgs
 
 __all__ = ['MatchSyntaxError', 'Match']
 
-class Match(object):
+class AbstractMatch(object):
     """
     Represents the <match> tags.
 
     You can call this object to apply the match
     """
 
-    def __init__(self, classes, path=None, domain=None,
+    def __init__(self, path=None, domain=None,
                  request_header=None, response_header=None, environ=None,
-                 abort=False, last=False, source_location=None):
-        self.classes = classes
+                 pyref=None, pyargs=None,
+                 source_location=None):
         self.path = path
         self.domain = domain
         self.request_header = request_header
         self.response_header = response_header
         self.environ = environ
-        self.abort = abort
-        self.last = last
+        self.pyref = pyref
+        self.pyargs = pyargs
         self.source_location = source_location
+
     
     @classmethod
-    def parse_xml(cls, el, source_location):
+    def parse_match_xml(cls, el, source_location):
         """
-        Creates an instance of Match from the given parsed XML element.
+        Parses out the match-related arguments
         """
-        assert (el.tag == 'match'
-                or el.tag == 'rule')
-        classes = el.get('class', '').split()
-        abort = asbool(el.get('abort'))
-        if not abort and not classes:
-            ## FIXME: source location
-            raise DeliveranceSyntaxError(
-                "You must provide some classes in the class attribute")
-        if abort and classes:
-            ## FIXME: source location
-            raise DeliveranceSyntaxError(
-                'You cannot provide both abort="1" and class="%s"'
-                % (' '.join(classes)))
         path = cls._parse_attr(el, 'path', default='path')
         domain = cls._parse_attr(el, 'domain', default='wildcard')
         request_header = cls._parse_attr(el, 'request-header', default='exact', header=True)
         response_header = cls._parse_attr(el, 'response-header', default='exact', header=True)
         environ = cls._parse_attr(el, 'environ', default='exact', header=True)
-        last = asbool(el.get('last'))
-        return cls(
-            classes,
+        pyref = el.get('pyref')
+        if pyref:
+            pyref = PyReference.parse(pyref, source_location=source_location,
+                                      default_function='match_request',
+                                      default_objs=dict(AbortTheme=AbortTheme))
+        pyargs = PyArgs.from_attrib(el.attrib)
+        if pyargs and not pyref:
+            raise DeliveranceSyntaxError(
+                'You cannot provide arguments (like %s) unless you provide a pyref attribute'
+                % pyargs,
+                element=el, source_location=source_location)
+        return dict(
             path=path,
             domain=domain,
             request_header=request_header,
             response_header=response_header,
             environ=environ,
-            abort=abort,
-            last=last,
+            pyref=pyref,
+            pyargs=pyargs,
             source_location=source_location)
 
+    match_attrs = [
+        'path', 'domain', 'request-header', 'response-header', 'environ', 'pyref']
+
     @staticmethod
     def _parse_attr(el, attr, default=None, header=False):
         """
@@ -77,28 +78,39 @@
             return compile_matcher(value, default)
 
     def __unicode__(self):
-        parts = [u'<match']
-        if self.classes:
-            parts.append(u'class="%s"' % html_quote(' '.join(self.classes)))
+        parts = [u'<%s' % self.element_name]
+        parts.extend(self._uni_early_args())
         for attr, value in [
             ('path', self.path),
             ('domain', self.domain),
             ('request-header', self.request_header),
             ('response-header', self.response_header),
-            ('environ', self.environ)]:
+            ('environ', self.environ),
+            ('pyref', self.pyref)]:
             if value:
                 parts.append(u'%s="%s"' % (attr, html_quote(unicode(self.path))))
-        if self.abort:
-            parts.append(u'abort="1"')
-        if self.last:
-            parts.append(u'last="1"')
+        if self.pyargs:
+            parts.append(unicode(self.pyargs))
+        parts.extend(self._uni_late_args())
         parts.append(u'/>')
         return ' '.join(parts)
 
+    def _uni_early_args(self):
+        return []
+
+    def _uni_late_args(self):
+        return []
+
     def __str__(self):
         return unicode(self).encode('utf8')
 
-    def __call__(self, request, response_headers, log):
+    def debug_description(self):
+        raise NotImplementedError
+
+    def log_context(self):
+        return self
+
+    def __call__(self, request, resp, response_headers, log):
         """
         Checks this match against the given request and response_headers object.
 
@@ -106,51 +118,105 @@
         :class:webob.Request object.
         """
         result = True
-        if self.abort:
-            class_name = 'abort'
-        elif len(self.classes) > 1:
-            class_name = '(%s)' % ' '.join(self.classes)
-        else:
-            class_name = self.classes[0]
+        debug_name = self.debug_description()
+        debug_context = self.log_context()
         if self.path:
             if not self.path(request.path):
-                log.debug(self, 'Skipping class %s because request URL (%s) does not match path="%s"',
-                          class_name, request.path, self.path)
+                log.debug(debug_context, 'Skipping %s because request URL (%s) does not match path="%s"',
+                          debug_name, request.path, self.path)
                 return False
         if self.domain:
             host = request.host.split(':', 1)[0]
             if not self.domain(host):
-                log.debug(self, 'Skipping class %s because request domain (%s) does not match domain="%s"',
-                          class_name, host, self.domain)
+                log.debug(debug_context, 'Skipping %s because request domain (%s) does not match domain="%s"',
+                          debug_name, host, self.domain)
                 return False
         if self.request_header:
             result, headers = self.request_header(request.headers)
             if not result:
-                log.debug(self, 'Skipping class %s because request headers %s do not match request-header="%s"',
-                          class_name, ', '.join(headers), self.request_header)
+                log.debug(debug_context, 'Skipping %s because request headers %s do not match request-header="%s"',
+                          debug_name, ', '.join(headers), self.request_header)
                 return False
         if self.response_header:
             result, headers = self.response_header(response_headers)
             if not result:
                 ## FIXME: maybe distinguish <meta> headers and real headers?
-                log.debug(self, 'Skipping class %s because the response headers %s do not match response-header="%s"',
-                          class_name, ', '.join(headers), self.response_header)
+                log.debug(debug_context, 'Skipping %s because the response headers %s do not match response-header="%s"',
+                          debug_name, ', '.join(headers), self.response_header)
                 return False
         if self.environ:
             result, keys = self.environ(request.environ)
             if not result:
-                log.debug(self, 'Skipping class %s because the request environ (keys %s) did not match environ="%s"',
-                          class_name, ', '.join(keys), self.environ)
+                log.debug(debug_context, 'Skipping %s because the request environ (keys %s) did not match environ="%s"',
+                          debug_name, ', '.join(keys), self.environ)
+                return False
+        if self.pyref:
+            result = self.pyref(request, resp, response_headers, log, **self.pyargs.dict)
+            if not result:
+                log.debug(debug_context, 'Skipping %s because the pyref="%s" returned false',
+                          debug_name, self.pyref)
                 return False
         return True
 
-def run_matches(matchers, request, response_headers, log):
+class Match(AbstractMatch):
+
+    element_name = 'match'
+
+    def __init__(self, classes=None, abort=False, last=False, **kw):
+        super(Match, self).__init__(**kw)
+        self.classes = classes
+        self.abort = abort
+        self.last = last
+
+    @classmethod
+    def parse_xml(cls, el, source_location):
+        """
+        Parses the <match> element into a match object
+        """
+        matchargs = cls.parse_match_xml(el, source_location)
+        assert el.tag == cls.element_name
+        classes = el.get('class', '').split()
+        abort = asbool(el.get('abort'))
+        if not abort and not classes:
+            ## FIXME: source location
+            raise DeliveranceSyntaxError(
+                "You must provide some classes in the class attribute")
+        if abort and classes:
+            ## FIXME: source location
+            raise DeliveranceSyntaxError(
+                'You cannot provide both abort="1" and class="%s"'
+                % (' '.join(classes)))
+        last = asbool(el.get('last'))
+        return cls(
+            classes=classes, abort=abort, last=last, **matchargs)
+
+    def _uni_early_args(self):
+        if self.classes:
+            return [u'class="%s"' % html_quote(' '.join(self.classes))]
+        else:
+            return []
+
+    def _uni_late_args(self):
+        parts = []
+        if self.abort:
+            parts.append(u'abort="1"')
+        if self.last:
+            parts.append(u'last="1"')
+        return parts
+
+    def debug_description(self):
+        if self.abort:
+            return 'abort'
+        else:
+            return 'class="%s"' % ' '.join(self.classes)
+
+def run_matches(matchers, request, resp, response_headers, log):
     """
     Runs all the match objects in matchers, returning the list of matched classes.
     """
     results = []
     for matcher in matchers:
-        if matcher(request, response_headers, log):
+        if matcher(request, resp, response_headers, log):
             if matcher.abort:
                 log.debug(matcher, '<match> matched request, aborting')
                 raise AbortTheme('<match> matched request, aborting')

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py	Wed Jul  2 19:30:02 2008
@@ -0,0 +1,181 @@
+"""
+Handles loading modules or files for use with the ``pyfunc`` attribute
+in <match>, <theme> or other places with python hooks
+"""
+from string import Template
+import os
+import new
+from UserDict import DictMixin
+from deliverance.exceptions import DeliveranceSyntaxError
+from deliverance.util.importstring import simple_import
+
+__all__ = ['PyReference', 'PyArgs']
+
+class DefaultDict(DictMixin):
+    """
+    A dictionary where all values have a default
+    """
+    def __init__(self, wrapping, default=''):
+        self.wrapping = wrapping
+        self.default = default
+    def __getitem__(self, key):
+        return self.wrapping.get(key, self.default)
+    def __setitem__(self, key, value):
+        self.wrapping[key] = value
+    def __delitem__(self, key):
+        if key in self.wrapping:
+            del self.wrapping[key]
+    def keys(self):
+        return self.wrapping.keys()
+    def __contains__(self, key):
+        return True
+
+class PyReference(object):
+    """
+    Represents a reference to a Python function that can be called
+    """
+
+    def __init__(self, module_name=None, filename=None, function_name=None, default_objs={}, source_location=None):
+        self.module_name = module_name
+        self.filename = filename
+        self.function_name = function_name
+        self.default_objs = default_objs
+        self.source_location = source_location
+        self._modules = {}
+
+    @classmethod
+    def parse(cls, s, source_location, default_function=None, default_objs={}):
+        s = s.strip()
+        module = filename = None
+        if s.startswith('file:'):
+            filename = s[len('file:'):]
+            if ':' in filename:
+                filename, func = filename.split(':', 1)
+            else:
+                func = default_function
+        else:
+            # A module name
+            if ':' in s:
+                module, func = s.split(':', 1)
+            else:
+                module = s
+        if func is None:
+            raise DeliveranceSyntaxError(
+                "You must provide a function name",
+                element=s, source_location=source_location)
+        if filename:
+            full_file = cls.expand_filename(filename, source_location)
+            if not os.path.exists(full_file):
+                if full_file != filename:
+                    raise DeliveranceSyntaxError(
+                        "The filename %r (expanded from %r) does not exist"
+                        % (full_file, filename),
+                        element=s, source_location=source_location)
+                else:
+                    raise DeliveranceSyntaxError(
+                        "The filename %r does not exist" % full_file,
+                        element=s, source_location=source_location)
+        return cls(module_name=module, file=filename, function=function, source_location=source_location,
+                   default_objs=default_objs)
+
+    def __repr__(self):
+        args = [repr(str(self))]
+        if self.default_objs:
+            args.append('default_objs=%r' % self.default_objs)
+        if self.source_location:
+            args.append('source_location=%r' % self.source_location)
+        return '%s.parse(%s)' % (
+            self.__class__.__name__, ', '.join(args))
+
+    def __unicode__(self):
+        if self.file:
+            return 'file:%s:%s' % (self.file, self.function)
+        else:
+            return '%s:%s' % (self.module, self.function)
+
+    def __str__(self):
+        return unicode(self).encode('utf8')
+
+    @property
+    def module(self):
+        """
+        Returns the instantiated module, or a module created from the filename
+        """
+        if module_name:
+            if module_name not in self._modules:
+                new_mod = simple_import(self.module_name)
+                for name, value in self.default_objs.items():
+                    if not hasattr(new_mod, name):
+                        setattr(new_mod, name, value)
+                self._modules[module_name] = new_mod
+            return self._modules[module_name]
+        else:
+            filename = self.expand_filename(self.filename, self.source_location)
+            if filename not in self._modules:
+                name = self.pyfile.strip('/').strip('\\')
+                name = os.path.splitext(name)[0]
+                name = name.replace('\\', '_').replace('/', '_')
+                new_mod = new.module(name)
+                new_mod.__file__ = filename
+                for name, value in self.default_objs.items():
+                    if not hasattr(new_mod, name):
+                        setattr(new_mod, name, value)
+                self._modules[filename] = new_mod
+            return self._modules[filename]
+
+    @property
+    def function(self):
+        """
+        Returns the function object
+        """
+        obj = self.module
+        for p in self.function_name.split('.'):
+            ## FIXME: better error handling:
+            obj = getattr(obj, p)
+        return obj
+    
+    def __call__(self, *args, **kw):
+        return self.function(*args, **kw)
+    
+    @staticmethod
+    def expand_filename(filename, source_location=None):
+        """
+        Expand environmental variables in a filename
+        """
+        vars = DefautDict(os.environ)
+        tmpl = Template(filename)
+        try:
+            return tmpl.substitute(os.environ)
+        except ValueError, e:
+            raise DeliveranceSyntaxError(
+                "The filename %r contains bad $ substitutions: %s"
+                % (filename, e),
+                filename, source_location=source_location)
+
+class PyArgs(object):
+    """
+    Represents pyarg-* arguments
+    """
+    def __init__(self, dict):
+        self.dict = dict
+        
+    def __nonzero__(self):
+        return bool(self.dict)
+
+    def __unicode__(self):
+        return ' '.join('pyargs-%s="%s"' % (name, value) 
+                        for name, value in sorted(self.dict.items()))
+
+    def __str__(self):
+        return unicode(self).encode('utf8')
+    
+    def __repr__(self):
+        return '%s(%r)' % (self.__class__.__name__, self.dict)
+    
+    @classmethod
+    def from_attrib(cls, attrib):
+        kw = {}
+        for name in attrib:
+            if name.startswith('pyarg-'):
+                kw[name[len('pyarg-'):]] = value
+        return cls(kw)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	Wed Jul  2 19:30:02 2008
@@ -6,7 +6,7 @@
 from deliverance.exceptions import add_exception_info, DeliveranceSyntaxError
 from deliverance.util.converters import asbool, html_quote
 from deliverance.selector import Selector
-from deliverance.pagematch import Match
+from deliverance.pagematch import AbstractMatch
 from deliverance.themeref import Theme
 from lxml import etree
 from lxml.html import document_fromstring
@@ -29,9 +29,6 @@
         self.suppress_standard = suppress_standard
         self.source_location = source_location
 
-    match_attrs = set([
-            'path', 'domain', 'request-header', 'response-header', 'environ'])
-
     @classmethod
     def parse_xml(cls, el, source_location):
         """
@@ -54,18 +51,10 @@
             action = parse_action(el, source_location)
             actions.append(action)
         match = None
-        for attr in cls.match_attrs:
+        for attr in RuleMatch.match_attrs:
             if el.get(attr):
-                match = Match.parse_xml(el, source_location)
-                if match.abort:
-                    raise DeliveranceSyntaxError(
-                        "You cannot have an abort attribute on <rule> elements",
-                        element=el)
-                if match.last:
-                    ## FIXME: is last a good alternative to suppress-standard?
-                    raise DeliveranceSyntaxError(
-                        "You cannot have a last attribute on <rule> elements",
-                        element=el)
+                match = RuleMatch.parse_xml(self, el, source_location)
+                ## FIXME: would last="1" be a good alternative to suppress-standard?
                 break
         return cls(classes, actions, theme, match, suppress_standard, source_location)
 
@@ -81,6 +70,25 @@
             action.apply(content_doc, theme_doc, resource_fetcher, log)
         return theme_doc
 
+class RuleMatch(AbstractMatch):
+    """
+    Represents match rules in the <rule> element
+    """
+
+    element_name = 'rule'
+
+    @classmethod
+    def parse_xml(cls, rule, el, source_location):
+        inst = cls(**cls.parse_match_xml(el, source_location))
+        inst.rule = rule
+        return rule
+
+    def debug_description(self):
+        return '<rule>'
+
+    def log_context(self):
+        return self.rule
+
 ## A dictionary mapping element names to their rule classes:
 _actions = {}
 

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	Wed Jul  2 19:30:02 2008
@@ -24,7 +24,7 @@
         else:
             response_headers = resp.headers
         try:
-            classes = run_matches(self.matchers, req, response_headers, log)
+            classes = run_matches(self.matchers, req, resp, response_headers, log)
         except AbortTheme:
             return resp
         if 'X-Deliverance-Page-Class' in resp.headers:
@@ -45,13 +45,13 @@
             theme = self.default_theme
             ## FIXME: error if not theme still
         assert theme is not None
-        theme_href = theme.resolve_href(req, resp)
+        theme_href = theme.resolve_href(req, resp, log)
         theme_doc = self.get_theme(theme_href, resource_fetcher, log)
         content_doc = self.parse_document(resp.body, req.url)
         run_standard = True
         for rule in rules:
             if rule.match is not None:
-                matches = rule.match(req, response_headers, log)
+                matches = rule.match(req, resp, response_headers, log)
                 if not matches:
                     log.debug(rule, "Skipping <rule>")
                     continue

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/sidebar.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/sidebar.html	Wed Jul  2 19:30:02 2008
@@ -0,0 +1,26 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-15">
+<title>The sidebar</title>
+</head>
+
+<body>
+
+This contains links that go into the sidebar.  Everything in the
+following div goes into the sidebar:
+
+<div id="menu" style="border: 2px solid #000">
+About this site:<br>
+  This is my homepage on the world wide web!
+  
+<ul>
+  <li><a href="/">Home</a></li>
+  <li><a href="/blog/">My blog</a></li>
+  <li><a href="/resume.html">Resume</a></li>
+</ul>
+
+</div>
+
+
+</body> </html>

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt	Wed Jul  2 19:30:02 2008
@@ -2,19 +2,19 @@
 
     >>> from deliverance.pagematch import Match
     >>> from lxml.etree import XML
-    >>> from webob import Request
+    >>> from webob import Request, Response
     >>> from webob.headerdict import HeaderDict
     >>> from deliverance.log import SavingLogger
     >>> def make(xml):
     ...     el = XML(xml)
     ...     return Match.parse_xml(el, source_location=None)
-    >>> def match(matcher, request, response_headers, show_log=True):
+    >>> def match(matcher, request, resp, response_headers, show_log=True):
     ...     if isinstance(matcher, basestring):
     ...         matcher = make(matcher)
     ...     log = SavingLogger(None, None)
     ...     if isinstance(response_headers, list):
     ...         response_headers = HeaderDict(response_headers)
-    ...     result = matcher(request, response_headers, log)
+    ...     result = matcher(request, resp, response_headers, log)
     ...     if show_log:
     ...         for level, rule, message in log.messages:
     ...             print 'log:', message
@@ -35,16 +35,16 @@
 Now, some matches:
 
     >>> m = make('<match path="/foo" class="a" />')
-    >>> match(m, Request.blank('/foo'), [])
+    >>> match(m, Request.blank('/foo'), Response(), [])
     True
-    >>> match(m, Request.blank('/foobar'), [])
-    log: Skipping class a because request URL (/foobar) does not match path="path:/foo/"
+    >>> match(m, Request.blank('/foobar'), Response(), [])
+    log: Skipping class="a" because request URL (/foobar) does not match path="path:/foo/"
     False
-    >>> match(m, Request.blank('/foo/bar'), [])
+    >>> match(m, Request.blank('/foo/bar'), Response(), [])
     True
     >>> m = make('<match response-header="Content-Type: contains: html" class="x" />')
-    >>> match(m, Request.blank('/'), [('content-type', 'text/plain')])
-    log: Skipping class x because the response headers Content-Type do not match response-header="Content-Type: contains:html"
+    >>> match(m, Request.blank('/'), Response(content_type='text/plain'), [('content-type', 'text/plain')])
+    log: Skipping class="x" because the response headers Content-Type do not match response-header="Content-Type: contains:html"
     False
-    >>> match(m, Request.blank('/'), [('content-type', 'text/html')])
+    >>> match(m, Request.blank('/'), Response(content_type='text/html'), [('content-type', 'text/html')])
     True

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py	Wed Jul  2 19:30:02 2008
@@ -0,0 +1,50 @@
+"""
+Represents <theme> elements
+"""
+from deliverance.exceptions import DeliveranceSyntaxError, AbortTheme
+from deliverance.pyref import PyReference, PyArgs
+import urlparse
+
+class Theme(object):
+    """
+    Represents the <theme> element
+    """
+
+    def __init__(self, href=None, pyref=None, pyargs=None, source_location=None):
+        self.href = href
+        self.pyref = pyref
+        self.pyargs = pyargs
+        self.source_location = source_location
+
+    @classmethod
+    def parse_xml(cls, el, source_location):
+        assert el.tag == 'theme'
+        href = el.get('href')
+        pyref = el.get('pyref')
+        pyargs = PyArgs.from_attrib(el.attrib)
+        if not pyref and pyargs:
+            raise DeliveranceSyntaxError(
+                'You cannot provide arguments (like %s) unless you provide a pyref attribute'
+                % pyargs,
+                element=el)
+        if pyref:
+            pyref = PythonReference.parse(pyref, source_location, 
+                                          default_function='get_theme',
+                                          default_objs=dict(AbortTheme=AbortTheme))
+        if not pyref and not href:
+            ## FIXME: also warn when pyref and href?
+            raise DeliveranceSyntaxError(
+                'You must provide at least one of href, pymodule, or the pyfile attribute',
+                element=el)
+        return cls(href=href, pyref=pyref,
+                   pyargs=pyargs, source_location=source_location)
+
+    def resolve_href(self, req, resp, log):
+        if self.pyref:
+            href = self.pyref(req, resp, log, **self.pyargs.dict)
+        else:
+            href = self.href
+        ## FIXME: is this join a good idea?
+        if href:
+            href = urlparse.urljoin(req.url, href)
+        return href

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/importstring.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/importstring.py	Wed Jul  2 19:30:02 2008
@@ -0,0 +1,95 @@
+# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
+# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
+
+"""
+'imports' a string -- converts a string to a Python object, importing
+any necessary modules and evaluating the expression.  Everything
+before the : in an import expression is the module path; everything
+after is an expression to be evaluated in the namespace of that
+module.
+
+Alternately, if no : is present, then import the modules and get the
+attributes as necessary.  Arbitrary expressions are not allowed in
+that case.
+"""
+
+def eval_import(s):
+    """
+    Import a module, or import an object from a module.
+
+    A module name like ``foo.bar:baz()`` can be used, where
+    ``foo.bar`` is the module, and ``baz()`` is an expression
+    evaluated in the context of that module.  Note this is not safe on
+    arbitrary strings because of the eval.
+    """
+    if ':' not in s:
+        return simple_import(s)
+    module_name, expr = s.split(':', 1)
+    module = import_module(module_name)
+    obj = eval(expr, module.__dict__)
+    return obj
+
+def simple_import(s):
+    """
+    Import a module, or import an object from a module.
+
+    A name like ``foo.bar.baz`` can be a module ``foo.bar.baz`` or a
+    module ``foo.bar`` with an object ``baz`` in it, or a module
+    ``foo`` with an object ``bar`` with an attribute ``baz``.
+    """
+    parts = s.split('.')
+    module = import_module(parts[0])
+    name = parts[0]
+    parts = parts[1:]
+    last_import_error = None
+    while parts:
+        name += '.' + parts[0]
+        try:
+            module = import_module(name)
+            parts = parts[1:]
+        except ImportError, e:
+            last_import_error = e
+            break
+    obj = module
+    while parts:
+        try:
+            obj = getattr(module, parts[0])
+        except AttributeError:
+            raise ImportError(
+                "Cannot find %s in module %r (stopped importing modules with error %s)" % (parts[0], module, last_import_error))
+        parts = parts[1:]
+    return obj
+
+def import_module(s):
+    """
+    Import a module.
+    """
+    mod = __import__(s)
+    parts = s.split('.')
+    for part in parts[1:]:
+        mod = getattr(mod, part)
+    return mod
+
+def try_import_module(module_name):
+    """
+    Imports a module, but catches import errors.  Only catches errors
+    when that module doesn't exist; if that module itself has an
+    import error it will still get raised.  Returns None if the module
+    doesn't exist.
+    """
+    try:
+        return import_module(module_name)
+    except ImportError, e:
+        if not getattr(e, 'args', None):
+            raise
+        desc = e.args[0]
+        if not desc.startswith('No module named '):
+            raise
+        desc = desc[len('No module named '):]
+        # If you import foo.bar.baz, the bad import could be any
+        # of foo.bar.baz, bar.baz, or baz; we'll test them all:
+        parts = module_name.split('.')
+        for i in range(len(parts)):
+            if desc == '.'.join(parts[i:]):
+                return None
+        raise

From z3-checkins at codespeak.net  Fri Jul  4 18:07:22 2008
From: z3-checkins at codespeak.net (VIAGRA ® Official Site)
Date: Fri,  4 Jul 2008 18:07:22 +0200 (CEST)
Subject: [z3-checkins] Dear z3-checkins@codespeak.net SALE 84% 0FF on Pfizer
Message-ID: <20080704070515.2968.qmail@cust-214-137.dsl.versateladsl.be>

An HTML attachment was scrubbed...
URL: http://codespeak.net/pipermail/z3-checkins/attachments/20080704/135b7359/attachment.htm 

From ianb at codespeak.net  Tue Jul  8 21:40:16 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  8 Jul 2008 21:40:16 +0200 (CEST)
Subject: [z3-checkins] r56372 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk: . deliverance
	deliverance/tests deliverance/util docs
Message-ID: <20080708194016.0F4792A805E@codespeak.net>

Author: ianb
Date: Tue Jul  8 21:40:14 2008
New Revision: 56372

Added:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxycommand.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/security.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/nesteddict.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/uritemplate.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/proxy.txt   (contents, props changed)
Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/exceptions.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/stringmatch.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py
Log:
Added a proxy command

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/exceptions.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/exceptions.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/exceptions.py	Tue Jul  8 21:40:14 2008
@@ -25,6 +25,11 @@
     aborted.
     """
 
+class AbortProxy(Exception):
+    """
+    Raised (and caught) when a proxy rule should be ignored
+    """
+
 def add_exception_info(info, exc_info=None):
     """
     Add the given information to the exception (typically context information)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	Tue Jul  8 21:40:14 2008
@@ -9,6 +9,7 @@
 import logging
 from lxml.etree import tostring, _Element
 from tempita import HTMLTemplate, html_quote, html
+from deliverance.security import display_logging
 
 NOTIFY = (logging.INFO + logging.WARN) / 2
 
@@ -47,7 +48,7 @@
         return self.message(logging.FATAL, el, msg, *args, **kw)
 
     def finish_request(self, req, resp):
-        if 'deliv_log' in req.GET:
+        if 'deliv_log' in req.GET and display_logging(req):
             resp.body += self.format_html_log()
             resp.cache_expires()
         return resp
@@ -56,7 +57,11 @@
     <h1 style="border-top: 3px dotted #f00">Deliverance Information</h1>
 
     <div>
-      <a href="{{theme_url}}" target="_blank">theme: {{log.theme_url}}</a>
+      {{if log.theme_url}}
+        <a href="{{theme_url}}" target="_blank">theme: {{log.theme_url}}</a>
+      {{else}}
+        theme: no theme set
+      {{endif}}
       | <a href="{{unthemed_url}}" target="_blank">unthemed content</a>
       | <a href="{{content_source}}" target="_blank">content source</a>
     </div>
@@ -101,6 +106,8 @@
             **self.tags)
 
     def _add_notheme(self, url):
+        if url is None:
+            return None
         if '?' in url:
             url += '&'
         else:

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	Tue Jul  8 21:40:14 2008
@@ -2,6 +2,7 @@
 from webob import exc
 from wsgiproxy.exactproxy import proxy_exact_request
 from deliverance.log import SavingLogger
+from deliverance.security import display_logging, display_local_files
 import urllib
 import hmac
 import sha
@@ -12,9 +13,12 @@
 from lxml.etree import _Element
 from lxml.html import fromstring, document_fromstring, tostring, Element
 import posixpath
+import mimetypes
+import os
 
 class DeliveranceMiddleware(object):
 
+    ## FIXME: is log_factory etc very useful?
     def __init__(self, app, rule_getter, log_factory=SavingLogger, log_factory_kw={}):
         self.app = app
         self.rule_getter = rule_getter
@@ -25,13 +29,18 @@
         return 'Deliverance'
 
     def __call__(self, environ, start_response):
-        ## FIXME: copy_get?:
         req = Request(environ)
         if 'deliv_notheme' in req.GET:
             return self.app(environ, start_response)
         req.environ['deliverance.base_url'] = req.application_url
+        ## FIXME: copy_get?:
         orig_req = Request(environ.copy())
-        log = self.log_factory(req, self, **self.log_factory_kw)
+        if 'deliverance.log' in req.environ:
+            log = req.environ['deliverance.log']
+        else:
+            log = self.log_factory(req, self, **self.log_factory_kw)
+            ## FIXME: should this be put in both the orig_req and this req?
+            req.environ['deliverance.log'] = log
         def resource_fetcher(url):
             return self.get_resource(url, orig_req, log)
         if req.path_info_peek() == '.deliverance':
@@ -49,8 +58,31 @@
 
     def get_resource(self, url, orig_req, log):
         assert url is not None
-        ## FIXME: should this return a webob.Response object?
-        if url.startswith(orig_req.application_url + '/'):
+        if url.lower().startswith('file:'):
+            if not display_local_files(orig_req):
+                ## FIXME: not sure if this applies generally; some calls to get_resource might
+                ## be because of a more valid subrequest than displaying a file
+                return exc.HTTPForbidden(
+                    "You cannot access file: URLs (like %r)" % url)
+            filename = '/' + url[len('file:'):].lstrip('/')
+            filename = urllib.unquote(filename)
+            if not os.path.exists(filename):
+                return exc.HTTPNotFound(
+                    "The file %r was not found" % filename)
+            if os.path.isdir(filename):
+                return exc.HTTPForbidden(
+                    "You cannot display a directory (%r)" % filename)
+            subresp = Response()
+            type, encoding = mimetypes.guess_type(filename)
+            if not type:
+                type = 'application/octet-stream'
+            subresp.content_type = type
+            ## FIXME: reading the whole thing obviously ain't great:
+            f = open(filename, 'rb')
+            subresp.body = f.read()
+            f.close()
+            return subresp
+        elif url.startswith(orig_req.application_url + '/'):
             subreq = orig_req.copy_get()
             subreq.environ['deliverance.subrequest_original_environ'] = orig_req.environ
             new_path_info = url[len(orig_req.application_url):]
@@ -95,6 +127,9 @@
         return url
 
     def internal_app(self, req, resource_fetcher):
+        if not display_logging(req):
+            return exc.HTTPForbidden(
+                "Logging is not enabled for you")
         segment = req.path_info_peek()
         method = 'action_%s' % segment
         method = getattr(self, method, None)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	Tue Jul  8 21:40:14 2008
@@ -5,7 +5,8 @@
 from deliverance.exceptions import DeliveranceSyntaxError, AbortTheme
 from deliverance.stringmatch import compile_matcher, compile_header_matcher
 from deliverance.util.converters import asbool, html_quote
-from deliverance.pyref import PyReference, PyArgs
+from deliverance.pyref import PyReference
+from deliverance.security import execute_pyref
 
 __all__ = ['MatchSyntaxError', 'Match']
 
@@ -18,15 +19,13 @@
 
     def __init__(self, path=None, domain=None,
                  request_header=None, response_header=None, environ=None,
-                 pyref=None, pyargs=None,
-                 source_location=None):
+                 pyref=None, source_location=None):
         self.path = path
         self.domain = domain
         self.request_header = request_header
         self.response_header = response_header
         self.environ = environ
         self.pyref = pyref
-        self.pyargs = pyargs
         self.source_location = source_location
 
     
@@ -40,17 +39,10 @@
         request_header = cls._parse_attr(el, 'request-header', default='exact', header=True)
         response_header = cls._parse_attr(el, 'response-header', default='exact', header=True)
         environ = cls._parse_attr(el, 'environ', default='exact', header=True)
-        pyref = el.get('pyref')
-        if pyref:
-            pyref = PyReference.parse(pyref, source_location=source_location,
-                                      default_function='match_request',
-                                      default_objs=dict(AbortTheme=AbortTheme))
-        pyargs = PyArgs.from_attrib(el.attrib)
-        if pyargs and not pyref:
-            raise DeliveranceSyntaxError(
-                'You cannot provide arguments (like %s) unless you provide a pyref attribute'
-                % pyargs,
-                element=el, source_location=source_location)
+        pyref = PyReference.parse_xml(
+            el, source_location=source_location,
+            default_function='match_request',
+            default_objs=dict(AbortTheme=AbortTheme))
         return dict(
             path=path,
             domain=domain,
@@ -58,7 +50,6 @@
             response_header=response_header,
             environ=environ,
             pyref=pyref,
-            pyargs=pyargs,
             source_location=source_location)
 
     match_attrs = [
@@ -85,12 +76,11 @@
             ('domain', self.domain),
             ('request-header', self.request_header),
             ('response-header', self.response_header),
-            ('environ', self.environ),
-            ('pyref', self.pyref)]:
+            ('environ', self.environ)]:
             if value:
                 parts.append(u'%s="%s"' % (attr, html_quote(unicode(self.path))))
-        if self.pyargs:
-            parts.append(unicode(self.pyargs))
+        if self.pyref:
+            parts.append(unicode(self.pyref))
         parts.extend(self._uni_late_args())
         parts.append(u'/>')
         return ' '.join(parts)
@@ -151,12 +141,20 @@
                           debug_name, ', '.join(keys), self.environ)
                 return False
         if self.pyref:
-            result = self.pyref(request, resp, response_headers, log, **self.pyargs.dict)
-            if not result:
-                log.debug(debug_context, 'Skipping %s because the pyref="%s" returned false',
-                          debug_name, self.pyref)
-                return False
-        return True
+            if not execute_pyref(request):
+                log.error(
+                    self, "Security disallows executing pyref %s")
+            else:
+                result = self.pyref(request, resp, response_headers, log)
+                if not result:
+                    log.debug(debug_context, 'Skipping %s because the reference <%s> returned false',
+                              debug_name, self.pyref)
+                    return False
+                if isinstance(result, basestring):
+                    result = result.split()
+                if isinstance(result, (list, tuple)):
+                    return self.classes + list(result)
+        return getattr(self, 'classes', None) or True
 
 class Match(AbstractMatch):
 
@@ -216,13 +214,14 @@
     """
     results = []
     for matcher in matchers:
-        if matcher(request, resp, response_headers, log):
+        classes = matcher(request, resp, response_headers, log)
+        if classes:
             if matcher.abort:
                 log.debug(matcher, '<match> matched request, aborting')
                 raise AbortTheme('<match> matched request, aborting')
             log.debug(matcher, '<match> matched request, adding classes %s',
-                      ', '.join(matcher.classes))
-            for item in matcher.classes:
+                      ', '.join(classes))
+            for item in classes:
                 if item not in results:
                     results.append(item)
             if matcher.last:

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	Tue Jul  8 21:40:14 2008
@@ -0,0 +1,567 @@
+from deliverance.exceptions import DeliveranceSyntaxError, AbortProxy
+from deliverance.pagematch import AbstractMatch
+from deliverance.util.converters import asbool
+from deliverance.middleware import DeliveranceMiddleware
+from deliverance.ruleset import RuleSet
+from deliverance.log import SavingLogger
+from deliverance.util.uritemplate import uri_template_substitute
+from deliverance.util.nesteddict import NestedDict
+from deliverance.security import execute_pyref
+from lxml.etree import tostring as xml_tostring
+from lxml.html import document_fromstring, tostring
+from pyref import PyReference
+from webob import Request
+from webob import exc
+import urlparse
+from wsgiproxy.exactproxy import proxy_exact_request
+import re
+import socket
+from lxml.etree import Comment
+from tempita import html_quote
+import os
+import string
+from paste.fileapp import FileApp
+import urllib
+import posixpath
+
+class ProxySet(object):
+
+    def __init__(self, proxies, ruleset, source_location=None):
+        self.proxies = proxies
+        self.ruleset = ruleset
+        self.source_location = source_location
+        self.deliverator = DeliveranceMiddleware(self.proxy_app, self.rule_getter)
+
+    @classmethod
+    def parse_xml(cls, el, source_location):
+        proxies = []
+        for child in el:
+            if child.tag == 'proxy':
+                proxies.append(Proxy.parse_xml(child, source_location))
+        ruleset = RuleSet.parse_xml(el, source_location)
+        return cls(proxies, ruleset, source_location)
+
+    def proxy_app(self, environ, start_response):
+        request = Request(environ)
+        log = environ['deliverance.log']
+        for proxy in self.proxies:
+            ## FIXME: obviously this is wonky:
+            if proxy.match(request, None, None, log):
+                try:
+                    return proxy.forward_request(environ, start_response)
+                except AbortProxy, e:
+                    log.debug(
+                        self, '<proxy> aborted (%s), trying next proxy' % e)
+                    continue
+                ## FIXME: should also allow for AbortTheme?
+        log.error(self, 'No proxy matched the request; aborting with a 404 Not Found error')
+        ## FIXME: better error handling would be nice:
+        resp = exc.HTTPNotFound()
+        return resp(environ, start_response)
+
+    def rule_getter(self, get_resource, app, orig_req):
+        return self.ruleset
+
+    def application(self, environ, start_response):
+        req = Request(environ)
+        log = SavingLogger(req, self.deliverator)
+        req.environ['deliverance.log'] = log
+        return self.deliverator(environ, start_response)
+
+class Proxy(object):
+
+    def __init__(self, match, dest,
+                 request_modifications, response_modifications,
+                 strip_script_name=False, keep_host=False,
+                 source_location=None):
+        self.match = match
+        self.match.proxy = self
+        self.dest = dest
+        self.strip_script_name = strip_script_name
+        self.keep_host = keep_host
+        self.request_modifications = request_modifications
+        self.response_modifications = response_modifications
+        self.source_location = source_location
+
+    def log_description(self, log=None):
+        parts = []
+        if log is None:
+            parts.append('&lt;proxy')
+        else:
+            parts.append('&lt;<a href="%s" target="_blank">proxy</a>' % log.link_to(self.source_location, source=True))
+        if self.strip_script_name:
+            parts.append('strip-script-name="1"')
+        if self.keep_host:
+            parts.append('keep-host="1"')
+        parts.append('/&gt;<br>\n')
+        parts.append('&nbsp;' + self.dest.log_description(log))
+        parts.append('<br>\n')
+        if self.request_modifications:
+            if len(self.request_modifications) > 1:
+                parts.append('&nbsp;%i request modifications<br>\n' % len(self.request_modifications))
+            else:
+                parts.append('&nbsp;1 request modification<br>\n')
+        if self.response_modifications:
+            if len(self.response_modifications) > 1:
+                parts.append('&nbsp;%i response modifications<br>\n' % len(self.response_modifications))
+            else:
+                parts.append('&nbsp;1 response modification<br>\n')
+        parts.append('&lt;/proxy&gt;')
+        return ' '.join(parts)
+
+    @classmethod
+    def parse_xml(cls, el, source_location):
+        assert el.tag == 'proxy'
+        match = ProxyMatch.parse_xml(el, source_location)
+        dest = None
+        request_modifications = []
+        response_modifications = []
+        strip_script_name = False
+        keep_host = False
+        for child in el:
+            if child.tag == 'dest':
+                if dest is not None:
+                    raise DeliveranceSyntaxError(
+                        "You cannot have more than one <dest> tag (second tag: %s)"
+                        % xml_tostring(child),
+                        element=child, source_location=source_location)
+                dest = ProxyDest.parse_xml(child, source_location)
+            elif child.tag == 'transform':
+                if child.get('strip-script-name'):
+                    strip_script_name = asbool(child.get('strip-script-name'))
+                if child.get('keep-host'):
+                    keep_host = asbool(child.get('keep-host'))
+                ## FIXME: error on other attrs
+            elif child.tag == 'request':
+                request_modifications.append(
+                    ProxyRequestModification.parse_xml(child, source_location))
+            elif child.tag == 'response':
+                response_modifications.append(
+                    ProxyResponseModification.parse_xml(child, source_location))
+            elif child.tag is Comment:
+                continue
+            else:
+                raise DeliveranceSyntaxError(
+                    "Unknown tag in <proxy>: %s" % xml_tostring(child),
+                    element=child, source_location=source_location)
+        return cls(match, dest, request_modifications, response_modifications,
+                   strip_script_name=strip_script_name, keep_host=keep_host,
+                   source_location=source_location)
+
+    def forward_request(self, environ, start_response):
+        request = Request(environ)
+        prefix = self.match.strip_prefix()
+        if prefix:
+            if prefix.endswith('/'):
+                prefix = prefix[:-1]
+            path_info = request.path_info
+            if not path_info.startswith(prefix + '/'):
+                log.warn(self, "The match would strip the prefix %r from the request path (%r), but they do not match"
+                         % (prefix + '/', path_info))
+            else:
+                request.script_name = request.script_name + prefix
+                request.path_info = path_info[len(prefix):]
+        log = request.environ['deliverance.log']
+        for modifier in self.request_modifications:
+            request = modifier.modify_request(request, log)
+        if self.dest.next:
+            raise AbortProxy
+        dest = self.dest(request, log)
+        log.debug(self, '<proxy> matched; forwarding request to %s' % dest)
+        response, orig_base, proxied_base, proxied_url = self.proxy_to_dest(request, dest)
+        for modifier in self.response_modifications:
+            response = modifier.modify_response(request, response, orig_base, proxied_base, proxied_url, log)
+        return response(environ, start_response)
+
+    def proxy_to_dest(self, request, dest):
+        # Not using request.copy because I don't want to copy wsgi.input:
+        # FIXME: handle file:
+        orig_base = request.application_url
+        proxy_req = Request(request.environ.copy())
+        scheme, netloc, path, query, fragment = urlparse.urlsplit(dest)
+        assert not fragment, (
+            "Unexpected fragment: %r" % fragment)
+        if scheme == 'file':
+            return self.proxy_to_file(request, dest)
+        proxy_req.path_info = path + request.path_info
+        proxy_req.server_name = netloc.split(':', 1)[0]
+        if ':' in netloc:
+            proxy_req.server_port = netloc.split(':', 1)[1]
+        elif scheme == 'http':
+            proxy_req.server_port = '80'
+        elif scheme == 'https':
+            proxy_req.server_port = '443'
+        else:
+            assert 0, "bad scheme: %r (from %r)" % (scheme, dest)
+        if not self.keep_host:
+            proxy_req.host = netloc
+        proxied_url = '%s://%s%s' % (scheme, netloc, proxy_req.path_qs)
+        if query:
+            if proxy_req.query_string:
+                proxy_req.query_string += '&'
+            ## FIXME: add query before or after existing query?
+            proxy_req.query_string += query
+        proxy_req.headers['X-Forwarded-For'] = request.remote_addr
+        proxy_req.headers['X-Forwarded-Scheme'] = request.scheme
+        proxy_req.headers['X-Forwarded-Server'] = request.host
+        ## FIXME: something with path? proxy_req.headers['X-Forwarded-Path']
+        if self.strip_script_name:
+            proxy_req.script_name = ''
+        try:
+            resp = proxy_req.get_response(proxy_exact_request)
+        except socket.error, e:
+            ## FIXME: really wsgiproxy should handle this
+            ## FIXME: which error?
+            ## 502 HTTPBadGateway, 503 HTTPServiceUnavailable, 504 HTTPGatewayTimeout?
+            if isinstance(e.args, tuple) and len(e.args) > 1:
+                error = e.args[1]
+            else:
+                error = str(e)
+            resp = exc.HTTPServiceUnavailable(
+                'Could not proxy the request to %s:%s : %s' % (proxy_req.server_name, proxy_req.server_port, error))
+        return resp, orig_base, dest, proxied_url
+
+    def proxy_to_file(self, request, dest):
+        orig_base = request.application_url
+        ## FIXME: security restrictions here?
+        assert dest.startswith('file:')
+        filename = urllib.unquote('/' + dest[len('file:'):].lstrip('/'))
+        rest = posixpath.normpath(request.path_info)
+        proxied_url = dest.lstrip('/') + '/' + urllib.quote(rest.lstrip('/'))
+        ## FIXME: handle /->/index.html
+        filename = filename.rstrip('/') + '/' + rest.lstrip('/')
+        app = FileApp(filename)
+        # I don't really need a copied request here, because FileApp is so simple:
+        resp = request.get_response(app)
+        return resp, orig_base, dest, proxied_url
+        
+class ProxyMatch(AbstractMatch):
+    
+    element_name = 'proxy'
+    
+    @classmethod
+    def parse_xml(cls, el, source_location):
+        ## FIXME: this should have a way of indicating what portion of the path to strip
+        return cls(**cls.parse_match_xml(el, source_location))
+    
+    def debug_description(self):
+        return '<proxy>'
+
+    def log_context(self):
+        return self.proxy
+
+    def strip_prefix(self):
+        if self.path:
+            return self.path.strip_prefix()
+        return None
+
+class ProxyDest(object):
+
+    def __init__(self, href=None, pyref=None, next=False, source_location=None):
+        self.href = href
+        self.pyref = pyref
+        self.next = next
+        self.source_location = source_location
+
+    @classmethod
+    def parse_xml(cls, el, source_location):
+        href = el.get('href')
+        pyref = PyReference.parse_xml(el, source_location, 
+                                      default_function='get_proxy_dest', default_objs=dict(AbortProxy=AbortProxy))
+        next = asbool(el.get('next'))
+        if next and (href or pyref):
+            raise DeliveranceSyntaxError(
+                'If you have a next="1" attribute you cannot also have an href or pyref attribute',
+                element=el, source_location=source_location)
+        return cls(href, pyref, next=next, source_location=source_location)
+
+    def __call__(self, request, log):
+        assert not self.next
+        if self.pyref:
+            if not execute_pyref(request):
+                log.error(
+                    self, "Security disallows executing pyref %s" % self.pyref)
+            else:
+                return self.pyref(request, log)
+        ## FIXME: is this nesting really needed?
+        ## we could just use HTTP_header keys...
+        vars = NestedDict(request.environ, request.headers, dict(here=posixpath.dirname(self.source_location)))
+        return uri_template_substitute(self.href, vars)
+
+    def log_description(self, log=None):
+        parts = ['&lt;dest']
+        if self.href:
+            if log is not None:
+                parts.append('href="%s"' % html_quote(html_quote(self.href)))
+            else:
+                ## FIXME: definite security issue with the link through here:
+                ## FIXME: Should this be source=True?
+                parts.append('href="<a href="%s" target="_blank">%s</a>"' % 
+                             (html_quote(log.link_to(self.href)), html_quote(html_quote(self.href))))
+        if self.pyref:
+            parts.append('pref="%s"' % html_quote(self.pyref))
+        if self.next:
+            parts.append('next="1"')
+        parts.append('/&gt;')
+        return ' '.join(parts)
+
+class ProxyRequestModification(object):
+    def __init__(self, pyref=None, header=None, content=None,
+                 source_location=None):
+        self.pyref = pyref
+        self.header = header
+        self.content = content
+        self.source_location = source_location
+
+    @classmethod
+    def parse_xml(cls, el, source_location):
+        assert el.tag == 'request'
+        pyref = PyReference.parse_xml(
+            el, source_location,
+            default_function='modify_proxy_request', 
+            default_objs=dict(AbortProxy=AbortProxy))
+        header = el.get('header')
+        content = el.get('content')
+        if (not header and content) or (not content and header):
+            raise DeliveranceSyntaxError(
+                "If you provide a header attribute you must provide a content attribute, and vice versa",
+                element=el, source_location=source_location)
+        return cls(pyref, header, content, source_location)
+        
+    def modify_request(self, request, log):
+        if self.pyref:
+            if not execute_pyref(request):
+                log.error(
+                    self, "Security disallows executing pyref %s" % self.pyref)
+            else:
+                result = self.pyref(request, log)
+                if isinstance(result, dict):
+                    request = Request(result)
+                elif isinstance(result, Request):
+                    request = result
+        if self.header:
+            request.headers[self.header] = self.content
+        return request
+
+class ProxyResponseModification(object):
+    def __init__(self, pyref=None, header=None, content=None, rewrite_links=False,
+                 source_location=None):
+        self.pyref = pyref
+        self.header = header
+        self.content = content
+        self.rewrite_links = rewrite_links
+
+    @classmethod
+    def parse_xml(cls, el, source_location):
+        assert el.tag == 'response'
+        pyref = PyReference.parse_xml(
+            el, source_location,
+            default_function='modify_proxy_response', 
+            default_objs=dict(AbortProxy=AbortProxy))
+        header = el.get('header')
+        content = el.get('content')
+        if (not header and content) or (not content and header):
+            raise DeliveranceSyntaxError(
+                "If you provide a header attribute you must provide a content attribute, and vice versa",
+                element=el, source_location=source_location)
+        rewrite_links = asbool(el.get('rewrite-links'))
+        return cls(pyref=pyref, header=header, content=content, rewrite_links=rewrite_links, 
+                   source_location=source_location)
+
+    _cookie_domain_re = re.compile(r'(domain="?)([a-z0-9._-]*)("?)', re.I)
+
+    ## FIXME: instead of proxied_base/proxied_path, should I keep the modified request object?
+    def modify_response(self, request, response, orig_base, proxied_base, proxied_url, log):
+        if not proxied_base.endswith('/'):
+            proxied_base += '/'
+        if not orig_base.endswith('/'):
+            orig_base += '/'
+        assert proxied_url.startswith(proxied_base), (
+            "Unexpected proxied_url %r, doesn't start with proxied_base %r"
+            % (proxied_url, proxied_base))
+        assert request.url.startswith(orig_base), (
+            "Unexpected request.url %r, doesn't start with orig_base %r"
+            % (request.url, orig_base))
+        if self.pyref:
+            if not execute_pyref(request):
+                log.error(
+                    self, "Security disallows executing pyref %s" % self.pyref)
+            else:
+                result = self.pyref(request, response, orig_base, proxied_base, proxied_url, log)
+                if isinstance(result, Response):
+                    response = result
+        if self.header:
+            response.headers[self.header] = self.content
+        if self.rewrite_links:
+            if response.content_type != 'text/html':
+                log.debug(self, 'Not rewriting links in response from %s, because Content-Type is %s' % (proxied_url, response.content_type))
+                return response
+            body_doc = document_fromstring(response.body, base_url=proxied_url)
+            body_doc.make_links_absolute()
+            def link_repl_func(link):
+                if not link.startswith(proxied_base):
+                    # External link, so we don't rewrite it
+                    return link
+                new = orig_base + link[len(proxied_base):]
+                return new
+            body_doc.rewrite_links(link_repl_func)
+            response.body = tostring(body_doc)
+            if response.location:
+                ## FIXME: if you give a proxy like http://openplans.org, and it redirects to
+                ## http://www.openplans.org, it won't be rewritten and that can be confusing
+                ## -- it *shouldn't* be rewritten, but some better log message is required
+                loc = urlparse.urljoin(proxied_url, response.location)
+                loc = link_repl_func(loc)
+                response.location = loc
+            if response.headers.get('set-cookie'):
+                cook = response.headers['set-cookie']
+                old_domain = urlparse.urlsplit(proxied_url)[1].lower()
+                new_domain = req.host.split(':', 1)[0].lower()
+                def rewrite_domain(match):
+                    domain = match.group(2)
+                    if domain == old_domain:
+                        ## FIXME: doesn't catch wildcards and the sort
+                        return match.group(1) + new_domain + match.group(3)
+                    else:
+                        return match.group(0)
+                cook = self._cookie_domain_re.sub(rewrite_domain, cook)
+                response.headers['set-cookie'] = cook
+        return response
+
+class ProxySettings(object):
+    """
+    Represents the settings for the proxy
+    """
+    def __init__(self, server_host, execute_pyref=True, display_local_files=True,
+                 dev_allow_ips=None, dev_deny_ips=None, dev_htpasswd=None, dev_users=None,
+                 dev_expiration=60,
+                 source_location=None):
+        self.server_host = server_host
+        self.execute_pyref = execute_pyref
+        self.display_local_files = display_local_files
+        self.dev_allow_ips = dev_allow_ips
+        self.dev_deny_ips = dev_deny_ips
+        self.dev_htpasswd = dev_htpasswd
+        self.dev_expiration = dev_expiration
+        self.dev_users = dev_users
+        self.source_location = source_location
+
+    @classmethod
+    def parse_xml(cls, el, source_location, environ=None, traverse=False):
+        if traverse and el.tag != 'server-settings':
+            try:
+                el = el.xpath('//server-settings')[0]
+            except IndexError:
+                raise DeliveranceSyntaxError(
+                    "There is no <server-settings> element",
+                    element=el)
+        if environ is None:
+            environ = os.environ
+        assert el.tag == 'server-settings'
+        server_host = 'localhost:8080'
+        ## FIXME: should these defaults be passed in:
+        execute_pyref = True
+        display_local_files = True
+        dev_allow_ips = []
+        dev_deny_ips = []
+        dev_htpasswd = None
+        dev_expiration = 60
+        dev_users = {}
+        for child in el:
+            if child.tag is Comment:
+                continue
+            ## FIXME: should some of these be attributes?
+            elif child.tag == 'server':
+                server_host = cls.substitute(child.text, environ)
+            elif child.tag == 'execute-pyref':
+                pyref = asbool(cls.substitute(child.text, environ))
+            elif child.tag == 'dev-allow':
+                dev_allow_ips.extend(cls.substitute(child.text, environ).split())
+            elif child.tag == 'dev-deny':
+                dev_deny_ips.extend(cls.substitute(child.text, environ).split())
+            elif child.tag == 'dev-htpasswd':
+                dev_htpasswd = cls.substitute(child.text, environ)
+            elif child.tag == 'dev-expiration':
+                dev_expiration = cls.substitute(child.text, environ)
+                if dev_expiration:
+                    dev_expiration = int(dev_expiration)
+            elif child.tag == 'display-local-files':
+                display_local_files = asbool(cls.substitute(child.text, environ))
+            elif child.tag == 'dev-user':
+                username = cls.substitute(child.get('username', ''), environ)
+                ## FIXME: allow hashed password?
+                password = cls.substitute(child.get('password', ''), environ)
+                if not username or not password:
+                    raise DeliveranceSyntaxError(
+                        "<dev-user> must have both a username and password attribute",
+                        element=child)
+                if username in dev_users:
+                    raise DeliveranceSyntaxError(
+                        '<dev-user username="%s"> appears more than once' % username,
+                        element=el)
+                dev_users[username] = password
+            else:
+                raise DeliveranceSyntaxError(
+                    'Unknown element in <server-settings>: <%s>' % child.tag,
+                    element=child)
+        if dev_users and dev_htpasswd:
+            raise DeliveranceSyntaxError(
+                "You can use <dev-htpasswd> or <dev-user>, but not both",
+                element=el)
+        ## FIXME: add a default allow_ips of 127.0.0.1?
+        return cls(server_host, execute_pyref=execute_pyref, display_local_files=display_local_files,
+                   dev_allow_ips=dev_allow_ips, dev_deny_ips=dev_deny_ips, 
+                   dev_users=dev_users, dev_expiration=dev_expiration,
+                   source_location=source_location)
+
+    @property
+    def host(self):
+        return self.server_host.split(':', 1)[0]
+
+    @property
+    def port(self):
+        if ':' in self.server_host:
+            return int(self.server_host.split(':', 1)[1])
+        else:
+            return 80
+
+    @property
+    def base_url(self):
+        host = self.host
+        if host == '0.0.0.0' or not host:
+            host = '127.0.0.1'
+        if self.port != 80:
+            host += ':%s' % self.port
+        return 'http://' + host
+
+    @staticmethod
+    def substitute(template, environ):
+        if environ is None:
+            return template
+        return string.Template(template).substitute(environ)
+
+    def middleware(self, app):
+        """
+        Wrap the given application in an appropriate DevAuth and Security instance
+        """
+        from devauth import DevAuth, convert_ip_mask
+        from deliverance.security import SecurityContext
+        if self.dev_users:
+            password_checker = self.check_password
+        else:
+            password_checker = None
+        app = SecurityContext.middleware(app, execute_pyref=self.execute_pyref,
+                                         display_local_files=self.display_local_files)
+        app = DevAuth(
+            app,
+            allow=convert_ip_mask(self.dev_allow_ips),
+            deny=convert_ip_mask(self.dev_deny_ips),
+            password_file=self.dev_htpasswd,
+            password_checker=password_checker,
+            expiration=self.dev_expiration,
+            login_mountpoint='/.deliverance')
+        return app
+
+    def check_password(self, username, password):
+        assert self.dev_users
+        return self.dev_users.get(username) == password

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxycommand.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxycommand.py	Tue Jul  8 21:40:14 2008
@@ -0,0 +1,65 @@
+import sys
+import os
+import optparse
+from deliverance.proxy import ProxySet
+from deliverance.proxy import ProxySettings
+
+from lxml.etree import parse
+from paste.httpserver import serve
+import urllib
+
+description = """\
+Starts up a proxy server using the given rule file.
+"""
+
+parser = optparse.OptionParser(
+    usage='%prog [OPTIONS] RULE.xml',
+    ## FIXME: get from pkg_resources:
+    version='0.1',
+    description=description,
+    )
+## FIXME: these should be handled by the settings (or just picked up from devauth):
+parser.add_option(
+    '--debug',
+    action='store_true',
+    dest='debug',
+    help='Show debugging information about unexpected exceptions in the browser')
+parser.add_option(
+    '--interactive-debugger',
+    action='store_true',
+    dest='interactive_debugger',
+    help='Use an interactive debugger (note: security hole when done publically; '
+    'if interface is not explicitly given it will be set to 127.0.0.1)')
+
+def run_command(rule_filename, debug=False, interactive_debugger=False):
+    rule_url = 'file://' + urllib.quote(os.path.abspath(rule_filename).replace(os.path.sep, '/'))
+    el = parse(rule_filename, base_url=rule_url).getroot()
+    ## FIXME: rule_filename isn't browsable in the logs
+    ps = ProxySet.parse_xml(el, rule_url)
+    settings = ProxySettings.parse_xml(el, rule_url, traverse=True)
+    app = ps.application
+    app = settings.middleware(app)
+    if interactive_debugger:
+        from weberror.evalexception import EvalException
+        app = EvalException(app, debug=True)
+    else:
+        from weberror.errormiddleware import ErrorMiddleware
+        app = ErrorMiddleware(app, debug=debug)
+    print 'To see logging, visit %s/.deliverance/login' % settings.base_url
+    serve(app, host=settings.host, port=settings.port)
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+    options, args = parser.parse_args()
+    if not args:
+        parser.error('You must provide a rule file')
+    if len(args) > 1:
+        parser.error('Only one argument (the rule file) allowed')
+    rule_filename = args[0]
+    run_command(rule_filename,
+                interactive_debugger=options.interactive_debugger,
+                debug=options.debug)
+
+if __name__ == '__main__':
+    main()

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py	Tue Jul  8 21:40:14 2008
@@ -3,13 +3,14 @@
 in <match>, <theme> or other places with python hooks
 """
 from string import Template
+from tempita import html_quote
 import os
 import new
 from UserDict import DictMixin
 from deliverance.exceptions import DeliveranceSyntaxError
 from deliverance.util.importstring import simple_import
 
-__all__ = ['PyReference', 'PyArgs']
+__all__ = ['PyReference']
 
 class DefaultDict(DictMixin):
     """
@@ -35,16 +36,31 @@
     Represents a reference to a Python function that can be called
     """
 
-    def __init__(self, module_name=None, filename=None, function_name=None, default_objs={}, source_location=None):
+    def __init__(self, module_name=None, filename=None, function_name=None, 
+                 args={}, default_objs={}, attr_name=None, source_location=None):
         self.module_name = module_name
         self.filename = filename
         self.function_name = function_name
+        self.args = args
         self.default_objs = default_objs
+        self.attr_name = attr_name
         self.source_location = source_location
         self._modules = {}
 
     @classmethod
-    def parse(cls, s, source_location, default_function=None, default_objs={}):
+    def parse_xml(cls, el, source_location, attr_name='pyref', default_function=None, default_objs={}):
+        s = el.get(attr_name)
+        args = {}
+        for name, value in el.attrib.items():
+            if name.startswith('pyarg-'):
+                args[name[len('pyarg-'):]] = value
+        if not s:
+            if args:
+                raise DeliveranceSyntaxError(
+                    "You provided pyargs-* attributes (%s) but no %s attribute"
+                    % (cls._format_args(args), attr_name),
+                    element=el, source_location=source_location)
+            return None
         s = s.strip()
         module = filename = None
         if s.startswith('file:'):
@@ -75,33 +91,16 @@
                     raise DeliveranceSyntaxError(
                         "The filename %r does not exist" % full_file,
                         element=s, source_location=source_location)
-        return cls(module_name=module, file=filename, function=function, source_location=source_location,
-                   default_objs=default_objs)
-
-    def __repr__(self):
-        args = [repr(str(self))]
-        if self.default_objs:
-            args.append('default_objs=%r' % self.default_objs)
-        if self.source_location:
-            args.append('source_location=%r' % self.source_location)
-        return '%s.parse(%s)' % (
-            self.__class__.__name__, ', '.join(args))
-
-    def __unicode__(self):
-        if self.file:
-            return 'file:%s:%s' % (self.file, self.function)
-        else:
-            return '%s:%s' % (self.module, self.function)
-
-    def __str__(self):
-        return unicode(self).encode('utf8')
+        return cls(module_name=module, file=filename, function_name=func, args=args, 
+                   attr_name=attr_name, default_objs=default_objs,
+                   source_location=source_location)
 
     @property
     def module(self):
         """
         Returns the instantiated module, or a module created from the filename
         """
-        if module_name:
+        if self.module_name:
             if module_name not in self._modules:
                 new_mod = simple_import(self.module_name)
                 for name, value in self.default_objs.items():
@@ -135,6 +134,8 @@
         return obj
     
     def __call__(self, *args, **kw):
+        for name, value in self.args.iteritems():
+            kw.setdefault(name, value)
         return self.function(*args, **kw)
     
     @staticmethod
@@ -152,30 +153,16 @@
                 % (filename, e),
                 filename, source_location=source_location)
 
-class PyArgs(object):
-    """
-    Represents pyarg-* arguments
-    """
-    def __init__(self, dict):
-        self.dict = dict
-        
-    def __nonzero__(self):
-        return bool(self.dict)
-
     def __unicode__(self):
-        return ' '.join('pyargs-%s="%s"' % (name, value) 
-                        for name, value in sorted(self.dict.items()))
-
+        if self.filename:
+            base = 'file:%s:%s' % (self.filename, self.function_name)
+        else:
+            base = '%s:%s' % (self.module_name, self.function_name)
+        parts = ['%s="%s"' % (self.attr_name, html_quote(base))]
+        for name, value in sorted(self.args.items()):
+            parts.append('pyarg-%s="%s"' % (name, html_quote(value)))
+        return ' '.join(parts)
+    
     def __str__(self):
         return unicode(self).encode('utf8')
-    
-    def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, self.dict)
-    
-    @classmethod
-    def from_attrib(cls, attrib):
-        kw = {}
-        for name in attrib:
-            if name.startswith('pyarg-'):
-                kw[name[len('pyarg-'):]] = value
-        return cls(kw)
+

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	Tue Jul  8 21:40:14 2008
@@ -3,7 +3,7 @@
 from deliverance.rules import Rule, remove_content_attribs
 from deliverance.themeref import Theme
 from lxml.html import tostring, document_fromstring
-from lxml.etree import XML
+from lxml.etree import XML, Comment
 import re
 import urlparse
 from webob.headerdict import HeaderDict
@@ -44,10 +44,15 @@
         if theme is None:
             theme = self.default_theme
             ## FIXME: error if not theme still
-        assert theme is not None
-        theme_href = theme.resolve_href(req, resp, log)
-        theme_doc = self.get_theme(theme_href, resource_fetcher, log)
-        content_doc = self.parse_document(resp.body, req.url)
+        if theme is None:
+            log.error(self, "No theme has been defined for the request")
+            return resp
+        try:
+            theme_href = theme.resolve_href(req, resp, log)
+            theme_doc = self.get_theme(theme_href, resource_fetcher, log)
+            content_doc = self.parse_document(resp.body, req.url)
+        except AbortTheme:
+            return resp
         run_standard = True
         for rule in rules:
             if rule.match is not None:
@@ -71,7 +76,12 @@
         log.theme_url = url
         ## FIXME: should do caching
         ## FIXME: check response status
-        doc = self.parse_document(resource_fetcher(url).body, url)
+        resp = resource_fetcher(url)
+        if resp.status_int != 200:
+            log.fatal(self, "The resource %s was not 200 OK: %s" % (url, resp.status))
+            raise AbortTheme(
+                "The resource %s returned an error: %s" % (url, resp.status))
+        doc = self.parse_document(resp.body, url)
         doc.make_links_absolute()
         return doc
 
@@ -102,6 +112,9 @@
             elif el.tag == 'theme':
                 ## FIXME: Add parse error
                 default_theme = Theme.parse_xml(el, source_location)
+            elif el.tag in ('proxy', 'server-settings', Comment):
+                # Handled elsewhere, so we just ignore this element
+                continue
             else:
                 ## FIXME: source location?
                 raise DeliveranceSyntaxError(
@@ -111,8 +124,6 @@
         for rule in rules:
             for class_name in rule.classes:
                 rules_by_class.setdefault(class_name, []).append(rule)
-        if default_theme and default_theme.href:
-            default_theme.href = urlparse.urljoin(doc.base, default_theme.href)
         return cls(matchers, rules_by_class, default_theme=default_theme,
                    source_location=source_location)
 

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/security.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/security.py	Tue Jul  8 21:40:14 2008
@@ -0,0 +1,79 @@
+class SecurityContext(object):
+    """
+    This represents the security context of the Deliverance request.
+    This is stored in ``environ['deliverance.security_context']`` and
+    is local to the request.
+
+    The three primary security-related restrictions are:
+    
+    1. Can Python be executed using pyref attributes?
+    2. Can logging messages be displayed?
+    3. Can local files be displayed?
+
+    Each of these is a method that takes the request.
+
+    When instantiating, the default value of None means that the value
+    should be guessed from the environment.
+
+    This uses the `developer auth spec
+    <http://wsgi.org/wsgi/Specifications/developer_auth>`_ for
+    guessing when a value is None.
+    """
+
+    def __init__(self, execute_pyref=False, display_logging=None,
+                 display_local_files=None):
+        self._execute_pyref = execute_pyref
+        self._display_logging = display_logging
+        self._display_local_files = display_local_files
+    
+    @classmethod
+    def install(cls, environ, **kw):
+        """
+        Instantiate the context and put it into the environment
+        """
+        inst = cls(**kw)
+        environ['deliverance.security_context'] = cls(**kw)
+        return inst
+    
+    def display_logging(self, environ):
+        if self._display_logging is not None:
+            return self._display_logging
+        return self.is_developer_user(environ)
+
+    def display_local_files(self, environ):
+        if self._display_logging is not None:
+            return self._display_logging
+        return self.is_developer_user(environ)
+
+    def execute_pyref(self, environ):
+        return self._execute_pyref
+
+    def is_developer_user(self, environ):
+        if hasattr(environ, 'environ'):
+            # Actually a request
+            environ = environ.environ
+        return bool(environ.get('x-wsgiorg.developer_user'))
+
+    @classmethod
+    def middleware(cls, app, **settings):
+        """
+        Wrap the application with middleware that installs settings
+        with the given configuration values.
+        """
+        def replacement_app(environ, start_response):
+            cls.install(environ, **settings)
+            return app(environ, start_response)
+        return replacement_app
+
+def make_getter(meth_name):
+    def getter(environ):
+        if hasattr(environ, 'environ'):
+            environ = environ.environ
+        ## FIXME: handle case when security context isn't in place?
+        return getattr(environ['deliverance.security_context'], meth_name)(environ)
+    getter.func_name = meth_name
+    return getter
+
+display_logging = make_getter('display_logging')
+display_local_files = make_getter('display_local_files')
+execute_pyref = make_getter('execute_pyref')

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/stringmatch.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/stringmatch.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/stringmatch.py	Tue Jul  8 21:40:14 2008
@@ -72,6 +72,12 @@
 
     name = None
 
+    def strip_prefix(self):
+        """
+        String prefix to strip from a matched string
+        """
+        return None
+
     def __init__(self, pattern):
         self.pattern = pattern
 
@@ -157,6 +163,9 @@
         return (s == self.pattern[:-1]
                 or s.startswith(self.pattern))
 
+    def strip_prefix(self):
+        return self.pattern
+
 _add_matcher(PathMatcher)
 
 class ExactMatcher(Matcher):

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml	Tue Jul  8 21:40:14 2008
@@ -0,0 +1,50 @@
+<ruleset>
+
+  <server-settings>
+    <server>127.0.0.1:8080</server>
+    <execute-pyref>true</execute-pyref>
+    <dev-allow>127.0.0.1</dev-allow>
+    <dev-user username="guest" password="guest" />
+  </server-settings>
+
+  <!-- 
+  <proxy path="/">
+    <dest next="1" />
+    <request header="X-Openplans-Project" content="opencore" />
+  </proxy>
+
+  <proxy path="/blog">
+    <dest href="http://localhost:10003" />
+  </proxy>
+
+  <proxy path="/tasks">
+    <dest href="http://localhost:10004" />
+  </proxy>
+
+  <proxy path="/theme">
+    <dest href="file://{here}/example-files/proxy-theme/" />
+  </proxy>
+
+  <proxy path="/">
+    [!- FIXME: I don't support {vh_SCRIPT_NAME} here -]
+    [!- <dest href="http://localhost:10001/VirtualHostBase/{wsgi.url_scheme}/{HTTP_HOST}/openplans/projects/{X-Openplans-Project}/VirtualHostRoot" /> -]
+    <dest href="http://localhost:10001/VirtualHostBase/{wsgi.url_scheme}/{HTTP_HOST}/openplans/VirtualHostRoot" />
+  </proxy>
+  -->
+
+  <proxy path="/theme-files">
+    <dest href="{here}/example-files/" />
+  </proxy>
+
+  <proxy path="/">
+    <dest href="http://www.openplans.org" />
+    <response rewrite-links="1" />
+  </proxy>
+
+  <theme href="/theme-files/theme.html" />
+
+  <rule>
+    <replace content="children:body" theme="children:#content" nocontent="abort" />
+  </rule>
+
+</ruleset>

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py	Tue Jul  8 21:40:14 2008
@@ -2,7 +2,11 @@
 Represents <theme> elements
 """
 from deliverance.exceptions import DeliveranceSyntaxError, AbortTheme
-from deliverance.pyref import PyReference, PyArgs
+from deliverance.pyref import PyReference
+from deliverance.security import execute_pyref
+from deliverance.util.uritemplate import uri_template_substitute
+from deliverance.util.nesteddict import NestedDict
+import posixpath
 import urlparse
 
 class Theme(object):
@@ -10,41 +14,48 @@
     Represents the <theme> element
     """
 
-    def __init__(self, href=None, pyref=None, pyargs=None, source_location=None):
+    def __init__(self, href=None, pyref=None, source_location=None):
         self.href = href
         self.pyref = pyref
-        self.pyargs = pyargs
         self.source_location = source_location
 
     @classmethod
     def parse_xml(cls, el, source_location):
         assert el.tag == 'theme'
         href = el.get('href')
-        pyref = el.get('pyref')
-        pyargs = PyArgs.from_attrib(el.attrib)
-        if not pyref and pyargs:
-            raise DeliveranceSyntaxError(
-                'You cannot provide arguments (like %s) unless you provide a pyref attribute'
-                % pyargs,
-                element=el)
-        if pyref:
-            pyref = PythonReference.parse(pyref, source_location, 
-                                          default_function='get_theme',
-                                          default_objs=dict(AbortTheme=AbortTheme))
+        pyref = PyReference.parse_xml(el, source_location, default_function='get_theme',
+                                      default_objs=dict(AbortTheme=AbortTheme))
         if not pyref and not href:
             ## FIXME: also warn when pyref and href?
             raise DeliveranceSyntaxError(
                 'You must provide at least one of href, pymodule, or the pyfile attribute',
                 element=el)
         return cls(href=href, pyref=pyref,
-                   pyargs=pyargs, source_location=source_location)
+                   source_location=source_location)
 
     def resolve_href(self, req, resp, log):
+        substitute = True
         if self.pyref:
-            href = self.pyref(req, resp, log, **self.pyargs.dict)
+            if not execute_pyref(req):
+                log.error(
+                    self, "Security disallows executing pyref %s" % self.pyref)
+                ## FIXME: this isn't very good; fatal exception?:
+                href = self.href
+            else:
+                href = self.pyref(req, resp, log)
+                substitute = False
         else:
             href = self.href
+        if substitute:
+            vars = NestedDict(req.environ, req.headers, dict(here=posixpath.dirname(self.source_location)))
+            new_href = uri_template_substitute(href, vars)
+            if new_href != href:
+                log.debug(
+                    self, 'Rewrote theme href="%s" to "%s"' % (href, new_href))
+                href = new_href
         ## FIXME: is this join a good idea?
+        print 'resolved value is %r (from %r) for request %r (base %r)' % (
+            href, self.href, req.url, self.source_location)
         if href:
             href = urlparse.urljoin(req.url, href)
         return href

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/nesteddict.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/nesteddict.py	Tue Jul  8 21:40:14 2008
@@ -0,0 +1,16 @@
+from UserDict import DictMixin
+
+class NestedDict(object):
+    def __init__(self, *dicts):
+        self.dicts = dicts
+    def __getitem__(self, key):
+        for d in self.dicts:
+            if key in d:
+                return d[key]
+        raise KeyError(key)
+    def keys(self):
+        keys = set()
+        for d in self.dicts:
+            keys.update(d.keys())
+        return list(keys)
+

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/uritemplate.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/uritemplate.py	Tue Jul  8 21:40:14 2008
@@ -0,0 +1,20 @@
+"""
+A simple implementation of URI templates.  Note: this is incomplete!
+
+This only implements simple {var} substitution, not any of the other
+operations in the URI template (unfinished) spec.
+"""
+import re
+
+__all__ = ['uri_template_substitute']
+
+_uri_var_re = re.compile(r'\{(.*?)\}')
+
+def uri_template_substitute(uri_template, vars):
+    def subber(match):
+        try:
+            return vars[match.group(1)]
+        except KeyError:
+            raise KeyError('No variable {%s} in uri_template %r'
+                           % (match.group(1), uri_template))
+    return _uri_var_re.sub(subber, uri_template)

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/proxy.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/proxy.txt	Tue Jul  8 21:40:14 2008
@@ -0,0 +1,23 @@
+Proxying can be setup inside the rules as well:
+
+<proxy *match-attrs>
+  <dest href="dest" | pyref="pyref" *pyargs />
+  <transform strip-script-name="1" keep-host="1" />
+  <request header="Some-Header" content="some content" />
+  <request pyref="pyref" *pyargs />
+  <response header="Some-Header" content="some content" />
+  <response pyref="pyref" *pyargs />
+  <response rewrite-links="1">
+</proxy>
+
+
+The <proxy> element takes all the same attributes that <match> does for matching (not including class, abort, last).
+
+The request is proxied to a location given with the <dest> element, either a location to proxy to, or a Python callback.  Any Python callback can raise AbortProxy, and the proxy will be skipped (looking for later matching proxies).  It can proxy to http/https and to file locations.
+
+The <transform> element controls how the request is transformed when it is forwarded.  By default all the standard headers -- X-Forwarded-For, X-Forwarded-Host, X-Forwarded-Scheme -- are added.  The Host header is not preserved by default, but if you use ``keep-host="1"`` it will be.  As a minor matter, ``environ['SCRIPT_NAME']`` is typically just ignored.  You can have it stripped off, and then X-Forwarded-Path will also be set.  (FIXME: check that header name)
+
+You can modify both the request and the response with multiple <request> and <response> tags.  The request can set headers to literal strings, and you can modify the request arbitrarily with ``pyref``.  The response can also have headers added, and arbitrary modification with ``pyref``.  You can also rewrite all links with ``rewrite-links="1"``; this is typically necessary if the X-Forwarded-\* headers aren't used to construct links.
+
+FIXME: should there be a way to avoid theming on a section?
+

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py	Tue Jul  8 21:40:14 2008
@@ -29,12 +29,19 @@
       test_suite='nose.collector', 
       tests_require=['nose'],
       install_requires=[
-        "lxml",
+        "lxml>=2.1alpha1",
         "WebOb",
         "WSGIProxy",
         "Tempita",
         "Pygments",
-      ],
+        "WebError",
+        "DevAuth",
+        ],
+      dependency_links=[
+        "https://svn.openplans.org/svn/DevAuth/trunk#egg=DevAuth-dev",
+        ],
       entry_points="""
+      [console_scripts]
+      deliverance-proxy = deliverance.proxycommand:main
       """,
       )

From ianb at codespeak.net  Tue Jul  8 22:25:53 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  8 Jul 2008 22:25:53 +0200 (CEST)
Subject: [z3-checkins] r56373 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk: . deliverance
	docs docs/_static
Message-ID: <20080708202553.ABE3416A047@codespeak.net>

Author: ianb
Date: Tue Jul  8 22:25:51 2008
New Revision: 56373

Added:
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/_static/
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/conf.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/news.txt   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/pyref.txt   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/regen-docs   (contents, props changed)
Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/   (props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/license.txt
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/page-classes.txt
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/proxy.txt
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/rules.txt
Log:
Setup sphinx

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	Tue Jul  8 22:25:51 2008
@@ -205,7 +205,9 @@
         proxy_req.headers['X-Forwarded-Scheme'] = request.scheme
         proxy_req.headers['X-Forwarded-Server'] = request.host
         ## FIXME: something with path? proxy_req.headers['X-Forwarded-Path']
+        ## (now we are only doing it with strip_script_name)
         if self.strip_script_name:
+            proxy_req.headers['X-Forwarded-Path'] = proxy_req.script_name
             proxy_req.script_name = ''
         try:
             resp = proxy_req.get_response(proxy_exact_request)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py	Tue Jul  8 22:25:51 2008
@@ -100,6 +100,7 @@
         """
         Returns the instantiated module, or a module created from the filename
         """
+        ## FIXME: this should reload the module as necessary.
         if self.module_name:
             if module_name not in self._modules:
                 new_mod = simple_import(self.module_name)

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/conf.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/conf.py	Tue Jul  8 22:25:51 2008
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# Paste documentation build configuration file, created by
+# sphinx-quickstart on Tue Apr 22 22:08:49 2008.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# The contents of this file are pickled, so don't put values in the namespace
+# that aren't pickleable (module imports are okay, they're removed automatically).
+#
+# All configuration values have a default value; values that are commented out
+# serve to show the default value.
+
+import sys
+
+# If your extensions are in another directory, add it here.
+#sys.path.append('some/directory')
+
+# General configuration
+# ---------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+#extensions = ['sphinx.ext.autodoc']
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.txt'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General substitutions.
+project = 'Deliverance'
+copyright = '2008, The Open Planning Project'
+
+# The default replacements for |version| and |release|, also used in various
+# other places throughout the built documents.
+#
+# The short X.Y version.
+version = '0.2'
+# The full version, including alpha/beta/rc tags.
+release = '0.2'
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+unused_docs = []
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# Options for HTML output
+# -----------------------
+
+# The style sheet to use for HTML and HTML Help pages. A file of that name
+# must exist either in Sphinx' static/ path, or in one of the custom paths
+# given in html_static_path.
+html_style = 'default.css'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Content template for the index page.
+#html_index = ''
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_use_modindex = True
+
+# If true, the reST sources are included in the HTML build as _sources/<name>.
+#html_copy_source = True
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Deliverancedoc'
+
+
+# Options for LaTeX output
+# ------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, document class [howto/manual]).
+#latex_documents = []
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_use_modindex = True

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt	Tue Jul  8 22:25:51 2008
@@ -0,0 +1,13 @@
+Deliverance
+===========
+
+Deliverance is a tool to theme HTML, applying a consistent style to applications and static files regardless of how they are implemented, and separating site-wide styling from application-level templating.
+
+.. toctree::
+
+   rules
+   proxy
+   pyref
+   page-classes
+   news
+   license

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/license.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/license.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/license.txt	Tue Jul  8 22:25:51 2008
@@ -1,4 +1,4 @@
-Copyright (c) 2007
+Copyright (c) 2008 The Open Planning Project
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/news.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/news.txt	Tue Jul  8 22:25:51 2008
@@ -0,0 +1,4 @@
+News
+====
+
+Everything is new: no news yet!

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/page-classes.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/page-classes.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/page-classes.txt	Tue Jul  8 22:25:51 2008
@@ -1,4 +1,7 @@
-OK, so I feel okay about a redesign of selectors and rules.  So the other thing is selecting themes and whatnot.
+Page Classes
+============
+
+**Note:** This document was written before the code, and needs to be corrected.
 
 I still like the idea of using names and indirection, ala CSS classes. I'm thinking of calling them page-class, like "page type" that Wichert suggested, but using "class" to suggest that a page isn't exclusively one thing.
 
@@ -6,23 +9,25 @@
 
 Setting these isn't always easy.  You have to poke around in templates to do it, and for something like styling Trac it would be nice to do it completely externally to Trac itself.  So back to path matching.  This seems fairly obvious:
 
+.. code-block:: xml
+
   <match path="/foo" class="foo" />
 
 There will be the <match> tag, and these selectors as attributes:
 
-  path: a path.  If it allows wildcards, I am a little concerned about people mistakenly putting an exact path and assuming it is a prefix.  So I'm thinking this won't take wildcards.  A trailing / won't matter -- /foo/ and /foo will be treated equivalently, and neither would match /foobar.
+  ``path``: a path.  If it allows wildcards, I am a little concerned about people mistakenly putting an exact path and assuming it is a prefix.  So I'm thinking this won't take wildcards.  A trailing / won't matter -- /foo/ and /foo will be treated equivalently, and neither would match /foobar.
 
     Perhaps prefixes could allow different kinds of matches.  E.g.: "wildcard:*/manage*" or "regex:/wp-admin/(post|new-post)\.php".
 
     I'd like a way to indicate whether just PATH_INFO or the full path is being matched.  People are naturally inclined to view it as a full-path match.  Perhaps a leading / would indicate this, and no leading / means match SCRIPT_NAME.  Except that wouldn't work for wildcard and regex matches.  For most installations PATH_INFO will be the full path, so it's a little academic.  But that's not true for openplans, so I have to think about this.
 
-  domain: a domain, with possible wildcard.
+  ``domain``: a domain, with possible wildcard.
 
   request-header: this would be "Header: match", like "X-Requested-With:  regex:(?i)xmlhttprequest".  The header name will be parsed out.  Maybe allow wildcards?  The part after : will be matched like path, except maybe a case-insensitive whitespace-normalized match as the default.  So "X-Requested-With: xmlhttprequest" would match a value of "XMLHttpRequest", but wouldn't match "XMLHttpRequest/foo"
 
-  response-header: just like request header, but the response.
+  ``response-header``: just like request header, but the response.
 
-  last: if this is "1", then if this <match> matches, no other matches will be checked.  (Not sure if this is the best name... it's like [L] in a rewrite rule)
+  ``last``: if this is "1", then if this <match> matches, no other matches will be checked.  (Not sure if this is the best name... it's like [L] in a rewrite rule)
 
 If you provide multiple attributes then they all must match.  I'm not sure how to allow multiple attributes of the same type, like if you want to match multiple request headers.
 
@@ -31,6 +36,8 @@
 
 OK, so then the actual rules.  We introduce a single attribute:
 
+.. code-block:: xml
+
   <rules class="class_name(s)">
     <append ... />
   </rules>
@@ -39,6 +46,8 @@
 
 Maybe better than things like -standard, would be a way of including other classes in a rule.  Like:
 
+.. code-block:: xml
+
   <rules class="my-page">
     <match class="standard" />
   </rules>

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/proxy.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/proxy.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/proxy.txt	Tue Jul  8 22:25:51 2008
@@ -1,23 +1,27 @@
 Proxying can be setup inside the rules as well:
 
-<proxy *match-attrs>
-  <dest href="dest" | pyref="pyref" *pyargs />
-  <transform strip-script-name="1" keep-host="1" />
-  <request header="Some-Header" content="some content" />
-  <request pyref="pyref" *pyargs />
-  <response header="Some-Header" content="some content" />
-  <response pyref="pyref" *pyargs />
-  <response rewrite-links="1">
-</proxy>
+.. code-block:: xml
 
+    <proxy *match-attrs>
+      <dest href="dest" | pyref="pyref" *pyargs />
+      <transform strip-script-name="1" keep-host="1" />
+      <request header="Some-Header" content="some content" />
+      <request pyref="pyref" *pyargs />
+      <response header="Some-Header" content="some content" />
+      <response pyref="pyref" *pyargs />
+      <response rewrite-links="1">
+    </proxy>
 
-The <proxy> element takes all the same attributes that <match> does for matching (not including class, abort, last).
+The ``<proxy>`` element takes all the same attributes that ``<match>`` does for matching (not including ``class``, ``abort``, ``last``).
 
-The request is proxied to a location given with the <dest> element, either a location to proxy to, or a Python callback.  Any Python callback can raise AbortProxy, and the proxy will be skipped (looking for later matching proxies).  It can proxy to http/https and to file locations.
+The request is proxied to a location given with the ``<dest>`` element, either a location to proxy to, or a Python callback.  Any Python callback can raise ``AbortProxy``, and the proxy will be skipped (looking for later matching proxies).  It can proxy to http/https and to file locations.  You can use URI templates for destinations as well, substituting headers and environmental variables as well as ``{here}`` which is the directory location of the rule document (e.g., ``<dest href="{href}/theme-files" />``).
 
 The <transform> element controls how the request is transformed when it is forwarded.  By default all the standard headers -- X-Forwarded-For, X-Forwarded-Host, X-Forwarded-Scheme -- are added.  The Host header is not preserved by default, but if you use ``keep-host="1"`` it will be.  As a minor matter, ``environ['SCRIPT_NAME']`` is typically just ignored.  You can have it stripped off, and then X-Forwarded-Path will also be set.  (FIXME: check that header name)
 
-You can modify both the request and the response with multiple <request> and <response> tags.  The request can set headers to literal strings, and you can modify the request arbitrarily with ``pyref``.  The response can also have headers added, and arbitrary modification with ``pyref``.  You can also rewrite all links with ``rewrite-links="1"``; this is typically necessary if the X-Forwarded-\* headers aren't used to construct links.
+You can modify both the request and the response with multiple ``<request>`` and ``<response>`` tags.  The request can set headers to literal strings, and you can modify the request arbitrarily with ``pyref``.  The response can also have headers added, and arbitrary modification with ``pyref``.  You can also rewrite all links with ``rewrite-links="1"``; this is typically necessary if the X-Forwarded-\* headers aren't used to construct links in the application.  You can also use this to try theming on an existing live site.
 
 FIXME: should there be a way to avoid theming on a section?
 
+FIXME: it would be nice to be able to put in a hard restriction on ``file:`` URLs (both to disallow, or simply to give a base directory that you can't possibly go above).
+
+FIXME: there should be a way of indicating what portion of the path is stripped.  If you do ``<proxy path="/foo">`` then ``/foo`` is stripped (moved to SCRIPT_NAME) before proxying, but for any other kind of matching it doesn't work.  Note you can still do it with ``pyref``.

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/pyref.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/pyref.txt	Tue Jul  8 22:25:51 2008
@@ -0,0 +1,59 @@
+Python References
+=================
+
+Many places in the configuration take ``pyref`` attributes.  These reference a module/function name.  The general pattern is:
+
+``pyref="location:function_name"``
+
+    The ``location`` can be a module name, or ``file:/path/to/filename.py``.  If it is a literal filename, then the file is exec'd and turned into a module that way.  All references are to functions (or callable objects), and so you must give the function name.  Note that the ``file:`` case isn't a URL, just a path.
+
+``pyarg-foo="bar"``
+
+    You can pass extra ad hoc arguments to the function using attributes in this form.  This would add the keyword argument ``foo="bar"`` to the function call.  All arguments have string (well, unicode) values.
+
+Examples
+--------
+
+You can use a ``<theme>`` like:
+
+.. code-block:: xml
+
+    <theme pyref="file:/home/me/src/customize-deliverance.py:get_theme"
+           pyarg-default_theme="default"
+           pyarg-base="/home/me/src/custom-themes" 
+           pyarg-base_url="/custom-themes" />
+
+    <proxy path="/custom-themes">
+        <dest href="file:///home/me/src/custom-themes" />
+    </proxy>
+
+Then your code might look like:
+
+.. code-block:: python
+
+    def get_theme(request, response, log, default_theme, 
+                  base, base_url):
+        host = request.host.split(':')[0]
+        if os.path.exists(os.path.join(base, host, 'theme.html'):
+            return base_url + '/' + host + '/theme.html'
+        elif not default_theme:
+            log.fatal(None, 
+                      "Theme for host %r doesn't exist and no default theme was given"
+                      % host)
+            raise AbortTheme("No theme found")
+        else:
+            log.debug(None,
+                      "Falling back to default theme")
+            return base_url + '/' + default_theme + '/theme.html'
+
+Disabling
+---------
+
+You can disallow Python references using ``deliverance-proxy`` with:
+
+.. code-block:: xml
+
+    <server-settings>
+      <execute-pyref>false</execute-pyref>
+    </server-settings>
+

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/rules.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/rules.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/rules.txt	Tue Jul  8 22:25:51 2008
@@ -1,10 +1,12 @@
 Rule Spec
 =========
 
+**Note:** This document was written before the code, and needs to be corrected (it's close to accurate, though).
+
 Selectors
 ---------
 
-Rules use selectors.  Selectors look like:
+Rules use selectors.  Selectors look like::
 
   type:selector1 || selector2 || ...
 
@@ -44,13 +46,18 @@
 
 These are optional:
 
-  href: a place to find the "content" for this rule.  You can use this to include external content.
-  if-content: a selector; if the selector matches anything then the rule will be run.  If not the rule will be skipped.  No types (elements:, etc) are allowed for this, but you may prefix "not " before the selector.  You may also use "&&", which binds looser than ||.
-  notheme: one of "ignore", "abort", "warn", where the default is "warn".  This is the error action if the theme selector doesn't match anything.
-  nocontent: like notheme.
-  manytheme: if the theme selector matches more than one element, then this is used to decide the action.  The default is "warn:first".  If you use "warn:", then a warning will given.  You can also use "abort" and "ignore", and "first" or "last".  E.g., manytheme="ignore:last", then the last element selected will be used, and no warning given.  If you give just "first" or "last", then ignore: is presumed.  "abort" by itself aborts the theming.
-  manycontent: like manytheme, but for those (few) instances when the content selector must match a single element.
-  move: contains "0" or "1" (default "1").  If "1", then the elements are removed from the content.  You can use this to take pieces out of the content; otherwise if an inner element and outer element from the content are both copied in, then the inner element will show up twice.
+  ``href``: a place to find the "content" for this rule.  You can use this to include external content.
+
+  ``if-content``: a selector; if the selector matches anything then the rule will be run.  If not the rule will be skipped.  No types (elements:, etc) are allowed for this, but you may prefix "not " before the selector.  You may also use "&&", which binds looser than ||.
+
+  ``notheme``: one of "ignore", "abort", "warn", where the default is "warn".  This is the error action if the theme selector doesn't match anything.
+
+  ``nocontent``: like notheme.
+
+  ``manytheme``: if the theme selector matches more than one element, then this is used to decide the action.  The default is "warn:first".  If you use "warn:", then a warning will given.  You can also use "abort" and "ignore", and "first" or "last".  E.g., manytheme="ignore:last", then the last element selected will be used, and no warning given.  If you give just "first" or "last", then ignore: is presumed.  "abort" by itself aborts the theming.
+  ``manycontent``: like manytheme, but for those (few) instances when the content selector must match a single element.
+
+  ``move``: contains "0" or "1" (default "1").  If "1", then the elements are removed from the content.  You can use this to take pieces out of the content; otherwise if an inner element and outer element from the content are both copied in, then the inner element will show up twice.
 
 For drop:
 
@@ -58,7 +65,7 @@
 
 Optional attributes for drop:
 
-  if-content, notheme, nocontent
+  ``if-content``, ``notheme``, ``nocontent``
 
 Behavior: replace
 ~~~~~~~~~~~~~~~~~
@@ -67,31 +74,31 @@
 
 theme=children, content=elements:
 
-The theme element is cleared of any content and text, and replaced with the content elements.
+    The theme element is cleared of any content and text, and replaced with the content elements.
 
 theme=elements, content=elements:
 
-This element is removed, and the content element(s) are put in the same location.
+    This element is removed, and the content element(s) are put in the same location.
 
 theme=elements, content=children:
 
-The theme element is removed, and replaced with all the children of the content elements (including text).
+    The theme element is removed, and replaced with all the children of the content elements (including text).
 
 theme=children, content=children:
 
-The theme element is cleared and replaced with the children of the content elements.
+    The theme element is cleared and replaced with the children of the content elements.
 
 theme=attributes, content=attributes:
 
-The theme element has all its attributes cleared, and replaced with the attributes of the content.
+    The theme element has all its attributes cleared, and replaced with the attributes of the content.
 
-If the theme is attributes, the content must also be attributes (and vice versa).
+    If the theme is attributes, the content must also be attributes (and vice versa).
 
 theme=tag, content=tag:
 
-Both the theme and content selector must match a single element.  The theme tag is removed and replaced with the content tag, but the theme's children are moved into the content tag.
+    Both the theme and content selector must match a single element.  The theme tag is removed and replaced with the content tag, but the theme's children are moved into the content tag.
 
-Both theme and content must be tag.
+    Both theme and content must be tag.
 
 
 Behavior: append/prepend
@@ -101,27 +108,27 @@
 
 theme=children, content=elements:
 
-The theme (a single element) has the content elements appended after all the theme's children.  prepend puts the children of the contents at the start of the theme element, in their original order.  For prepend any leading text in the theme goes after the children elements.
+    The theme (a single element) has the content elements appended after all the theme's children.  prepend puts the children of the contents at the start of the theme element, in their original order.  For prepend any leading text in the theme goes after the children elements.
 
 theme=elements, content=elements:
 
-The theme (a single element) has all the elements from the content put immediate after the theme element.  prepend puts the content elements immediately before the element, but the content elements themselves retain their original order.
+    The theme (a single element) has all the elements from the content put immediate after the theme element.  prepend puts the content elements immediately before the element, but the content elements themselves retain their original order.
 
 theme=elements, content=children:
 
-The theme element has the children of the content elements appended after it.
+    The theme element has the children of the content elements appended after it.
 
 theme=children, content=children:
 
-The theme element has the children of the content appended or prepended before its children.
+    The theme element has the children of the content appended or prepended before its children.
 
 theme=attributes, content=attributes:
 
-The content attributes are added to the theme attributes.  When attributes overlap, the content attribute is dropped when using append.  With prepend, the theme attribute is dropped.  The class attribute is a special case: these are combined (space-separated).
+    The content attributes are added to the theme attributes.  When attributes overlap, the content attribute is dropped when using append.  With prepend, the theme attribute is dropped.  The class attribute is a special case: these are combined (space-separated).
 
 theme=tag, or content=tag:
 
-Disallowed.
+    Disallowed.
 
 Behavior: drop
 ~~~~~~~~~~~~~~
@@ -130,19 +137,19 @@
 
 elements:
 
-Drop the elements.  (Doesn't include tail text)
+    Drop the elements.  (Doesn't include tail text)
 
 children:
 
-Clear the elements of their children (including contained text).
+    Clear the elements of their children (including contained text).
 
 attributes:
 
-Clear the elements' attributes.
+    Clear the elements' attributes.
 
 tag:
 
-Remove the element tags, but keep their children in place.
+    Remove the element tags, but keep their children in place.
 
 
 
@@ -153,6 +160,8 @@
 
 You want to add the class of the content's body element to the class of the resulting page:
 
+.. code-block:: xml
+
   <append content="attribute(class):body"
           theme="attribute:body" />
 
@@ -162,6 +171,8 @@
 
 You want to take content from one of several places (different products put it in different locations).  In all cases it is merged into a single theme element:
 
+.. code-block:: xml
+
   <replace content="#portal-content || children:body"
            theme="children:#content" nocontent="abort" />
 
@@ -171,6 +182,8 @@
 
 You want to copy over content CSS, except for one annoying file:
 
+.. code-block:: xml
+
   <drop content="link[href$='/bad-style.css']" nocontent="ignore" />
   <append theme="children:head" content="link[rel='stylesheet']" />
 
@@ -182,10 +195,14 @@
 
 You want to copy over the title from the content, but if the content has no title then use the title from the theme:
 
+.. code-block:: xml
+
   <drop theme="title" if-content="title" />
   <append theme="children:head" content="title" nocontent="ignore" />
 
-You want to copy over the title from the content.  The theme looks like "<title> | Site Name</title>", and you want to keep "| Site Name" as the tail of the title:
+You want to copy over the title from the content.  The theme looks like ``<title> | Site Name</title>``, and you want to keep ``| Site Name`` as the tail of the title:
+
+.. code-block:: xml
 
   <prepend theme="children:title" content="children:title" />
 
@@ -193,6 +210,8 @@
 
 You want to copy the login button from the content to a known location of the theme.  It might be a login, or it might be the username.
 
+.. code-block:: xml
+
   <replace theme="#place-for-login"
    content="form[action*='login'] || div.user-info"
    nocontent="ignore" />
@@ -201,6 +220,8 @@
 
 The content has a class that uses a CSS style that breaks out of the theme.  You want to just get rid of that class:
 
+.. code-block:: xml
+
   <drop content="attributes(class):.stupid-class"
         nocontent="ignore" />
 

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/regen-docs
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/regen-docs	Tue Jul  8 22:25:51 2008
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+mkdir -p docs/_static docs/_build
+sphinx-build -E -b html docs/ docs/_build || exit 1
+if [ "$1" = "publish" ] ; then
+  cd docs/
+  echo "Uploading files..."
+  #scp -r _build/* <location>
+  echo "(no upload location set)"
+fi

From ianb at codespeak.net  Tue Jul  8 22:39:26 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  8 Jul 2008 22:39:26 +0200 (CEST)
Subject: [z3-checkins] r56374 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance
Message-ID: <20080708203926.A37D816A053@codespeak.net>

Author: ianb
Date: Tue Jul  8 22:39:24 2008
New Revision: 56374

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py
Log:
remove a spurious debug print

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/themeref.py	Tue Jul  8 22:39:24 2008
@@ -54,8 +54,6 @@
                     self, 'Rewrote theme href="%s" to "%s"' % (href, new_href))
                 href = new_href
         ## FIXME: is this join a good idea?
-        print 'resolved value is %r (from %r) for request %r (base %r)' % (
-            href, self.href, req.url, self.source_location)
         if href:
             href = urlparse.urljoin(req.url, href)
         return href

From ianb at codespeak.net  Tue Jul  8 22:40:24 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  8 Jul 2008 22:40:24 +0200 (CEST)
Subject: [z3-checkins] r56375 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance
Message-ID: <20080708204024.8351D16A056@codespeak.net>

Author: ianb
Date: Tue Jul  8 22:40:23 2008
New Revision: 56375

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
Log:
remove default 60min expiration of dev login

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	Tue Jul  8 22:40:23 2008
@@ -436,7 +436,7 @@
     """
     def __init__(self, server_host, execute_pyref=True, display_local_files=True,
                  dev_allow_ips=None, dev_deny_ips=None, dev_htpasswd=None, dev_users=None,
-                 dev_expiration=60,
+                 dev_expiration=0,
                  source_location=None):
         self.server_host = server_host
         self.execute_pyref = execute_pyref
@@ -467,7 +467,7 @@
         dev_allow_ips = []
         dev_deny_ips = []
         dev_htpasswd = None
-        dev_expiration = 60
+        dev_expiration = 0
         dev_users = {}
         for child in el:
             if child.tag is Comment:

From ianb at codespeak.net  Tue Jul  8 22:54:21 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  8 Jul 2008 22:54:21 +0200 (CEST)
Subject: [z3-checkins] r56376 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance: . tests
Message-ID: <20080708205421.BB139169F66@codespeak.net>

Author: ianb
Date: Tue Jul  8 22:54:21 2008
New Revision: 56376

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/security.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example.py
Log:
get example working again; add force_dev_auth to SecurityContext

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/security.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/security.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/security.py	Tue Jul  8 22:54:21 2008
@@ -18,13 +18,18 @@
     This uses the `developer auth spec
     <http://wsgi.org/wsgi/Specifications/developer_auth>`_ for
     guessing when a value is None.
+
+    Also if you use ``force_dev_auth=True`` then DevAuth login will
+    not be required, and at all times you will be logged in as a dev
+    user.
     """
 
     def __init__(self, execute_pyref=False, display_logging=None,
-                 display_local_files=None):
+                 display_local_files=None, force_dev_auth=False):
         self._execute_pyref = execute_pyref
         self._display_logging = display_logging
         self._display_local_files = display_local_files
+        self._force_dev_auth = force_dev_auth
     
     @classmethod
     def install(cls, environ, **kw):
@@ -52,6 +57,8 @@
         if hasattr(environ, 'environ'):
             # Actually a request
             environ = environ.environ
+        if self._force_dev_auth:
+            return True
         return bool(environ.get('x-wsgiorg.developer_user'))
 
     @classmethod

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example.py	Tue Jul  8 22:54:21 2008
@@ -7,10 +7,14 @@
 from paste.httpserver import serve
 from weberror.evalexception import EvalException
 from deliverance.middleware import DeliveranceMiddleware, SubrequestRuleGetter
+from deliverance.security import SecurityContext
 
 base_path = os.path.join(os.path.dirname(__file__), 'example-files')
 app = StaticURLParser(base_path)
 deliv_app = DeliveranceMiddleware(app, SubrequestRuleGetter('/rules.xml'))
+full_app = SecurityContext.middleware(deliv_app, execute_pyref=True, display_logging=True, display_local_files=True,
+                                      force_dev_auth=True)
 
 if __name__ == '__main__':
-    serve(EvalException(deliv_app))
+    print 'See http://127.0.0.1:8080/?deliv_log for the page with log messages'
+    serve(EvalException(full_app))

From ianb at codespeak.net  Tue Jul  8 22:56:29 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue,  8 Jul 2008 22:56:29 +0200 (CEST)
Subject: [z3-checkins] r56377 - z3/deliverance/sandbox/ianb/deliverance/trunk
Message-ID: <20080708205629.D1F8B169F66@codespeak.net>

Author: ianb
Date: Tue Jul  8 22:56:29 2008
New Revision: 56377

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py
Log:
add a requirement

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py	Tue Jul  8 22:56:29 2008
@@ -36,6 +36,7 @@
         "Pygments",
         "WebError",
         "DevAuth",
+        "Paste",
         ],
       dependency_links=[
         "https://svn.openplans.org/svn/DevAuth/trunk#egg=DevAuth-dev",

From ianb at codespeak.net  Wed Jul  9 08:52:30 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Wed,  9 Jul 2008 08:52:30 +0200 (CEST)
Subject: [z3-checkins] r56379 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance
Message-ID: <20080709065230.3DD6B2A013F@codespeak.net>

Author: ianb
Date: Wed Jul  9 08:52:30 2008
New Revision: 56379

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
Log:
Made logging work for <drop> rules.  Handled case of append/prepend when there are no child elements (only text).  Missing import for making nocontent=abort work.

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/rules.py	Wed Jul  9 08:52:30 2008
@@ -3,7 +3,7 @@
 puts them together
 """
 
-from deliverance.exceptions import add_exception_info, DeliveranceSyntaxError
+from deliverance.exceptions import add_exception_info, DeliveranceSyntaxError, AbortTheme
 from deliverance.util.converters import asbool, html_quote
 from deliverance.selector import Selector
 from deliverance.pagematch import AbstractMatch
@@ -297,7 +297,7 @@
         parts = ['&lt;%s' % linked_item(self.source_location, self.name, source=True)]
         if getattr(self, 'content', None):
             body = 'content="%s"' % html_quote(self.content)
-            if self.content_href:
+            if getattr(self, 'content_href', None):
                 if request_url:
                     content_url = urlparse.urljoin(request_url, self.content_href)
                 else:
@@ -688,8 +688,14 @@
                     theme_el.extend(els)
                     pos_text = 'end'
                 else:
-                    add_tail(els[-1], theme_el.text)
-                    theme_el.text = text
+                    if len(els):
+                        add_tail(els[-1], theme_el.text)
+                        theme_el.text = text
+                    else:
+                        old_text = theme_el.text
+                        theme_el.text = text or ''
+                        if old_text:
+                            theme_el.text += old_text
                     theme_el[:0] = els
                     pos_text = 'beginning'
                 if self.move:
@@ -853,7 +859,6 @@
         ## FIXME: proper error:
         assert content is not None or theme is not None
         self.content = content
-        assert theme is not None
         self.theme = theme
         self.if_content = if_content
         self.nocontent = self.convert_error('nocontent', nocontent)

From ianb at codespeak.net  Wed Jul  9 08:53:12 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Wed,  9 Jul 2008 08:53:12 +0200 (CEST)
Subject: [z3-checkins] r56380 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance
Message-ID: <20080709065312.C18A42A013F@codespeak.net>

Author: ianb
Date: Wed Jul  9 08:53:12 2008
New Revision: 56380

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
Log:
Pick up classes from environment.  Catch AbortTheme from more places.  Handle case when there are no rules for a class

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	Wed Jul  9 08:53:12 2008
@@ -29,6 +29,8 @@
             return resp
         if 'X-Deliverance-Page-Class' in resp.headers:
             classes.extend(resp.headers['X-Deliverance-Page-Class'].strip().split())
+        if 'deliverance.page_classes' in req.environ:
+            classes.extend(req.environ['deliverance.page_classes'])
         if not classes:
             classes = ['default']
         rules = []
@@ -36,7 +38,7 @@
         for class_name in classes:
             ## FIXME: handle case of unknown classes
             ## Or do that during compilation?
-            for rule in self.rules_by_class[class_name]:
+            for rule in self.rules_by_class.get(class_name, []):
                 if rule not in rules:
                     rules.append(rule)
                     if rule.theme:
@@ -51,21 +53,21 @@
             theme_href = theme.resolve_href(req, resp, log)
             theme_doc = self.get_theme(theme_href, resource_fetcher, log)
             content_doc = self.parse_document(resp.body, req.url)
+            run_standard = True
+            for rule in rules:
+                if rule.match is not None:
+                    matches = rule.match(req, resp, response_headers, log)
+                    if not matches:
+                        log.debug(rule, "Skipping <rule>")
+                        continue
+                rule.apply(content_doc, theme_doc, resource_fetcher, log)
+                if rule.suppress_standard:
+                    run_standard = False
+            if run_standard:
+                ## FIXME: should it be possible to put the standard rule in the ruleset?
+                standard_rule.apply(content_doc, theme_doc, resource_fetcher, log)
         except AbortTheme:
             return resp
-        run_standard = True
-        for rule in rules:
-            if rule.match is not None:
-                matches = rule.match(req, resp, response_headers, log)
-                if not matches:
-                    log.debug(rule, "Skipping <rule>")
-                    continue
-            rule.apply(content_doc, theme_doc, resource_fetcher, log)
-            if rule.suppress_standard:
-                run_standard = False
-        if run_standard:
-            ## FIXME: should it be possible to put the standard rule in the ruleset?
-            standard_rule.apply(content_doc, theme_doc, resource_fetcher, log)
         remove_content_attribs(theme_doc)
         ## FIXME: handle caching?
         resp.body = tostring(theme_doc)

From ianb at codespeak.net  Wed Jul  9 08:54:06 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Wed,  9 Jul 2008 08:54:06 +0200 (CEST)
Subject: [z3-checkins] r56381 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk: . deliverance
	deliverance/media deliverance/tests
	deliverance/tests/example-files deliverance/util
Message-ID: <20080709065406.2361F2A013F@codespeak.net>

Author: ianb
Date: Wed Jul  9 08:54:05 2008
New Revision: 56381

Added:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/media/
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/media/browser.js
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/deliv-sidebar.html   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/deliv-theme.html   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/fixup_openplans.py   (contents, props changed)
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/filetourl.py   (contents, props changed)
Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxycommand.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt
   z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py
Log:
Lots of bug fixes to make the deliverance combination site work, improving proxying, adding the browser to figure out how to make selections

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/log.py	Wed Jul  9 08:54:05 2008
@@ -64,6 +64,7 @@
       {{endif}}
       | <a href="{{unthemed_url}}" target="_blank">unthemed content</a>
       | <a href="{{content_source}}" target="_blank">content source</a>
+      | <a href="{{content_browse}}" target="_blank">browse content</a>
     </div>
 
     {{if log.messages}}
@@ -98,11 +99,13 @@
 
     def format_html_log(self):
         content_source = self.link_to(self.request.url, source=True)
+        content_browse = self.link_to(self.request.url, browse=True)
         return self.log_template.substitute(
             log=self, middleware=self.middleware, 
             unthemed_url=self._add_notheme(self.request.url),
             theme_url=self._add_notheme(self.theme_url),
             content_source=content_source,
+            content_browse=content_browse,
             **self.tags)
 
     def _add_notheme(self, url):
@@ -137,8 +140,9 @@
             logging.ERROR: ('#fff', '#600'),
             logging.CRITICAL: ('#000', '#f33')}[level]
 
-    def link_to(self, url, source=False, line=None, selector=None):
-        return self.middleware.link_to(self.request, url, source=source, line=line, selector=selector)
+    def link_to(self, url, source=False, line=None, selector=None, browse=False):
+        return self.middleware.link_to(self.request, url, source=source, line=line, 
+                                       selector=selector, browse=browse)
 
 class PrintingLogger(SavingLogger):
 

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/media/browser.js
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/media/browser.js	Wed Jul  9 08:54:05 2008
@@ -0,0 +1,53 @@
+google.load("jquery", "1");
+var deliveranceSelector = null;
+var selectedElement = null;
+
+google.setOnLoadCallback(function() {
+  var selector = $('#deliverance-ids').get(0);
+  deliveranceSelector = selector;
+  var uniqueClasses = {};
+  var dupClasses = {};
+  $("*").get().map(function (el) {
+    if (el.id == 'deliverance-ids' || el.id == 'deliverance-browser') {
+      return;
+    }
+    if (el.id) {
+      var option = document.createElement('option');
+      option.value = '#' + el.id;
+      option.innerHTML = '#' + el.id;
+      selector.appendChild(option);
+    }
+    if (el.className) {
+      var allClasses = el.className.split();
+      allClasses.map(function (className) {
+        if (! dupClasses[className]) {
+          if (uniqueClasses[className]) {
+            dupClasses[className] = true;
+            uniqueClasses[className] = undefined;
+          } else {
+            uniqueClasses[className] = true;
+          }
+        }
+      });
+    }
+  });
+  for (var className in uniqueClasses) {
+    if (! uniqueClasses[className]) {
+      continue;
+    }
+    var option = document.createElement('option');
+    option.value = '.'+className;
+    option.innerHTML = '.'+className;
+    selector.appendChild(option);
+  }
+});
+
+function deliveranceChangeId() {
+  var option = deliveranceSelector.value;
+  if (selectedElement) {
+    selectedElement.removeClass('deliverance-highlight');
+  }
+  selectedElement = $(option);
+  selectedElement.addClass('deliverance-highlight');
+}
+

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/middleware.py	Wed Jul  9 08:54:05 2008
@@ -3,6 +3,7 @@
 from wsgiproxy.exactproxy import proxy_exact_request
 from deliverance.log import SavingLogger
 from deliverance.security import display_logging, display_local_files
+from deliverance.util.filetourl import url_to_filename
 import urllib
 import hmac
 import sha
@@ -64,8 +65,7 @@
                 ## be because of a more valid subrequest than displaying a file
                 return exc.HTTPForbidden(
                     "You cannot access file: URLs (like %r)" % url)
-            filename = '/' + url[len('file:'):].lstrip('/')
-            filename = urllib.unquote(filename)
+            filename = url_to_filename(url)
             if not os.path.exists(filename):
                 return exc.HTTPNotFound(
                     "The file %r was not found" % filename)
@@ -108,7 +108,7 @@
                       url, subresp.status, subresp.content_type)
             return subresp
 
-    def link_to(self, req, url, source=False, line=None, selector=None):
+    def link_to(self, req, url, source=False, line=None, selector=None, browse=False):
         base = req.environ['deliverance.base_url']
         base += '/.deliverance/view'
         source = int(bool(source))
@@ -119,6 +119,8 @@
             args['line'] = str(line)
         if selector:
             args['selector'] = selector
+        if browse:
+            args['browse'] = '1'
         url = base + '?' + urllib.urlencode(args)
         if selector:
             url += '#deliverance-selection'
@@ -140,83 +142,142 @@
         except exc.HTTPException, e:
             return e
 
+    def action_media(self, req, resource_fetcher):
+        ## FIXME: I'm not using this currently, because the Javascript didn't work.  Dunno why.
+        from paste.urlparser import StaticURLParser
+        app = StaticURLParser(os.path.join(os.path.dirname(__file__), 'media'))
+        ## FIXME: need to pop some segments from the req?
+        req.path_info_pop()
+        resp = req.get_response(app)
+        if resp.content_type == 'application/x-javascript':
+            resp.content_type = 'application/javascript'
+        return resp
+
     def action_view(self, req, resource_fetcher):
         url = req.GET['url']
         source = int(req.GET.get('source', '0'))
+        browse = int(req.GET.get('browse', '0'))
         line = int(req.GET.get('line', '0')) or ''
         selector = req.GET.get('selector', '')
         subresp = resource_fetcher(url)
         if source:
-            ct = subresp.content_type
-            if ct.startswith('application/xml'):
-                lexer = XmlLexer()
-            elif ct == 'text/html':
-                lexer = HtmlLexer()
-            else:
-                ## FIXME: what then?
-                lexer = HtmlLexer()
-            text = pygments_highlight(
-                subresp.body, lexer,
-                HtmlFormatter(full=True, linenos=True, lineanchors='code'))
+            return self.view_source(req, subresp)
+        elif browse:
+            return self.view_browse(req, subresp)
+        else:
+            return self.view_selection(req, subresp)
+
+    def view_source(self, req, resp):
+        ct = resp.content_type
+        if ct.startswith('application/xml'):
+            lexer = XmlLexer()
+        elif ct == 'text/html':
+            lexer = HtmlLexer()
+        else:
+            ## FIXME: what then?
+            lexer = HtmlLexer()
+        text = pygments_highlight(
+            resp.body, lexer,
+            HtmlFormatter(full=True, linenos=True, lineanchors='code'))
+        return Response(text)
+
+    def view_browse(self, req, resp):
+        import re
+        body = resp.body
+        f = open(os.path.join(os.path.dirname(__file__), 'media', 'browser.js'))
+        content = f.read()
+        f.close()
+        extra_head = '''
+        <!-- Added by Deliverance for browsing: -->
+        <script src="http://www.google.com/jsapi"></script>
+        <script>
+        %s
+        </script>
+        <style type="text/css">
+          .deliverance-highlight {
+            border: 2px dotted #f00;
+          }
+        </style>
+        <base href="%s">
+        <!-- Begin original page -->
+        ''' % (
+            content, posixpath.dirname(req.GET['url']) + '/')
+        match = re.search(r'<head>', body, re.I)
+        if match:
+            body = body[:match.end()] + extra_head + body[match.end():]
         else:
-            from deliverance.selector import Selector
-            doc = document_fromstring(subresp.body)
-            el = Element('base')
-            el.set('href', posixpath.dirname(url) + '/')
-            doc.head.insert(0, el)
-            selector = Selector.parse(selector)
-            type, elements, attributes = selector(doc)
-            if not elements:
-                template = self._not_found_template
+            body = extra_head + body
+        extra_body = '''
+        <div style="display: block; color: #000; background-color: #dfd; font-family: sans-serif; font-size: 100%; border-bottom: 2px dotted #f00" id="deliverance-browser">
+        <span style="float: right"><button onclick="window.close()">close</button></span>
+        View by id/class: <select onchange="deliveranceChangeId()" name="deliverance-ids" id="deliverance-ids"></select>
+        </div>'''
+        match = re.search('<body.*?>', body, re.I)
+        if match:
+            body = body[:match.end()] + extra_body + body[match.end():]
+        else:
+            body = extra_body + body
+        return Response(body)
+
+    def view_selection(self, req, resp):
+        from deliverance.selector import Selector
+        doc = document_fromstring(resp.body)
+        el = Element('base')
+        el.set('href', posixpath.dirname(req.GET['url']) + '/')
+        doc.head.insert(0, el)
+        selector = Selector.parse(selector)
+        type, elements, attributes = selector(doc)
+        if not elements:
+            template = self._not_found_template
+        else:
+            template = self._found_template
+        all_elements = []
+        els_in_head = False
+        for index, el in enumerate(elements):
+            el_in_head = self._el_in_head(el)
+            if el_in_head:
+                els_in_head = True
+            anchor = 'deliverance-selection'
+            if index:
+                anchor += '-%s' % index
+            if el.get('id'):
+                anchor = el.get('id')
+            ## FIXME: is a <a name> better?
+            if not el_in_head:
+                el.set('id', anchor)
             else:
-                template = self._found_template
-            all_elements = []
-            els_in_head = False
-            for index, el in enumerate(elements):
-                el_in_head = self._el_in_head(el)
-                if el_in_head:
-                    els_in_head = True
-                anchor = 'deliverance-selection'
-                if index:
-                    anchor += '-%s' % index
-                if el.get('id'):
-                    anchor = el.get('id')
-                ## FIXME: is a <a name> better?
-                if not el_in_head:
-                    el.set('id', anchor)
-                else:
-                    anchor = None
-                ## FIXME: add :target CSS rule
-                ## FIXME: or better, some Javascript
-                all_elements.append((anchor, el))
-                if not el_in_head:
-                    style = el.get('style', '')
-                    if style:
-                        style += '; '
-                    style += '/* deliverance */ border: 2px dotted #f00'
-                    el.set('style', style)
-                else:
-                    el.set('DELIVERANCE-MATCH', '1')
-            def highlight(html_code):
-                if isinstance(html_code, _Element):
-                    html_code = tostring(html_code)
-                return html(pygments_highlight(html_code, HtmlLexer(),
-                                               HtmlFormatter(noclasses=True)))
-            def format_tag(tag):
-                return highlight(tostring(tag).split('>')[0]+'>')
-            text = template.substitute(
-                base_url=req.url,
-                els_in_head=els_in_head, doc=doc,
-                elements=all_elements, selector=selector, 
-                format_tag=format_tag, highlight=highlight)
-            message = fromstring(self._message_template.substitute(message=text, url=url))
-            if doc.body.text:
-                message.tail = doc.body.text
-                doc.body.text = ''
-            doc.body.insert(0, message)
-            text = tostring(doc)
-        resp = Response(text)
-        return resp
+                anchor = None
+            ## FIXME: add :target CSS rule
+            ## FIXME: or better, some Javascript
+            all_elements.append((anchor, el))
+            if not el_in_head:
+                style = el.get('style', '')
+                if style:
+                    style += '; '
+                style += '/* deliverance */ border: 2px dotted #f00'
+                el.set('style', style)
+            else:
+                el.set('DELIVERANCE-MATCH', '1')
+        def highlight(html_code):
+            if isinstance(html_code, _Element):
+                html_code = tostring(html_code)
+            return html(pygments_highlight(html_code, HtmlLexer(),
+                                           HtmlFormatter(noclasses=True)))
+        def format_tag(tag):
+            return highlight(tostring(tag).split('>')[0]+'>')
+        text = template.substitute(
+            base_url=req.url,
+            els_in_head=els_in_head, doc=doc,
+            elements=all_elements, selector=selector, 
+            format_tag=format_tag, highlight=highlight)
+        message = fromstring(self._message_template.substitute(message=text, url=url))
+        if doc.body.text:
+            message.tail = doc.body.text
+            doc.body.text = ''
+        doc.body.insert(0, message)
+        text = tostring(doc)
+        return Response(text)
+
 
     def _el_in_head(self, el):
         while el is not None:

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	Wed Jul  9 08:54:05 2008
@@ -10,7 +10,7 @@
 from lxml.etree import tostring as xml_tostring
 from lxml.html import document_fromstring, tostring
 from pyref import PyReference
-from webob import Request
+from webob import Request, Response
 from webob import exc
 import urlparse
 from wsgiproxy.exactproxy import proxy_exact_request
@@ -23,6 +23,8 @@
 from paste.fileapp import FileApp
 import urllib
 import posixpath
+from deliverance.util.filetourl import filename_to_url, url_to_filename
+from lxml.etree import parse
 
 class ProxySet(object):
 
@@ -41,6 +43,12 @@
         ruleset = RuleSet.parse_xml(el, source_location)
         return cls(proxies, ruleset, source_location)
 
+    @classmethod
+    def parse_file(cls, filename):
+        file_url = filename_to_url(filename)
+        el = parse(filename, base_url=file_url).getroot()
+        return cls.parse_xml(el, file_url)
+
     def proxy_app(self, environ, start_response):
         request = Request(environ)
         log = environ['deliverance.log']
@@ -72,8 +80,8 @@
 
     def __init__(self, match, dest,
                  request_modifications, response_modifications,
-                 strip_script_name=False, keep_host=False,
-                 source_location=None):
+                 strip_script_name=True, keep_host=False,
+                 source_location=None, classes=None):
         self.match = match
         self.match.proxy = self
         self.dest = dest
@@ -82,6 +90,7 @@
         self.request_modifications = request_modifications
         self.response_modifications = response_modifications
         self.source_location = source_location
+        self.classes = classes
 
     def log_description(self, log=None):
         parts = []
@@ -89,11 +98,12 @@
             parts.append('&lt;proxy')
         else:
             parts.append('&lt;<a href="%s" target="_blank">proxy</a>' % log.link_to(self.source_location, source=True))
-        if self.strip_script_name:
-            parts.append('strip-script-name="1"')
+        ## FIXME: defaulting to true is bad
+        if not self.strip_script_name:
+            parts.append('strip-script-name="0"')
         if self.keep_host:
             parts.append('keep-host="1"')
-        parts.append('/&gt;<br>\n')
+        parts.append('&gt;<br>\n')
         parts.append('&nbsp;' + self.dest.log_description(log))
         parts.append('<br>\n')
         if self.request_modifications:
@@ -116,7 +126,7 @@
         dest = None
         request_modifications = []
         response_modifications = []
-        strip_script_name = False
+        strip_script_name = True
         keep_host = False
         for child in el:
             if child.tag == 'dest':
@@ -144,9 +154,10 @@
                 raise DeliveranceSyntaxError(
                     "Unknown tag in <proxy>: %s" % xml_tostring(child),
                     element=child, source_location=source_location)
+        classes = el.get('class', '').split() or None
         return cls(match, dest, request_modifications, response_modifications,
                    strip_script_name=strip_script_name, keep_host=keep_host,
-                   source_location=source_location)
+                   source_location=source_location, classes=classes)
 
     def forward_request(self, environ, start_response):
         request = Request(environ)
@@ -155,7 +166,8 @@
             if prefix.endswith('/'):
                 prefix = prefix[:-1]
             path_info = request.path_info
-            if not path_info.startswith(prefix + '/'):
+            if not path_info.startswith(prefix + '/') and not path_info == prefix:
+                log = environ['deliverance.log']
                 log.warn(self, "The match would strip the prefix %r from the request path (%r), but they do not match"
                          % (prefix + '/', path_info))
             else:
@@ -168,6 +180,9 @@
             raise AbortProxy
         dest = self.dest(request, log)
         log.debug(self, '<proxy> matched; forwarding request to %s' % dest)
+        if self.classes:
+            log.debug(self, 'Adding class="%s" to page' % ' '.join(self.classes))
+            request.environ.setdefault('deliverance.page_classes', []).extend(self.classes)
         response, orig_base, proxied_base, proxied_url = self.proxy_to_dest(request, dest)
         for modifier in self.response_modifications:
             response = modifier.modify_response(request, response, orig_base, proxied_base, proxied_url, log)
@@ -195,7 +210,6 @@
             assert 0, "bad scheme: %r (from %r)" % (scheme, dest)
         if not self.keep_host:
             proxy_req.host = netloc
-        proxied_url = '%s://%s%s' % (scheme, netloc, proxy_req.path_qs)
         if query:
             if proxy_req.query_string:
                 proxy_req.query_string += '&'
@@ -204,11 +218,13 @@
         proxy_req.headers['X-Forwarded-For'] = request.remote_addr
         proxy_req.headers['X-Forwarded-Scheme'] = request.scheme
         proxy_req.headers['X-Forwarded-Server'] = request.host
+        proxy_req.scheme = scheme
         ## FIXME: something with path? proxy_req.headers['X-Forwarded-Path']
         ## (now we are only doing it with strip_script_name)
         if self.strip_script_name:
             proxy_req.headers['X-Forwarded-Path'] = proxy_req.script_name
             proxy_req.script_name = ''
+        proxied_url = '%s://%s%s' % (scheme, netloc, proxy_req.path_qs)
         try:
             resp = proxy_req.get_response(proxy_exact_request)
         except socket.error, e:
@@ -227,14 +243,33 @@
         orig_base = request.application_url
         ## FIXME: security restrictions here?
         assert dest.startswith('file:')
-        filename = urllib.unquote('/' + dest[len('file:'):].lstrip('/'))
+        if '?' in dest:
+            dest = dest.split('?', 1)[0]
+        filename = url_to_filename(dest)
         rest = posixpath.normpath(request.path_info)
         proxied_url = dest.lstrip('/') + '/' + urllib.quote(rest.lstrip('/'))
         ## FIXME: handle /->/index.html
         filename = filename.rstrip('/') + '/' + rest.lstrip('/')
-        app = FileApp(filename)
-        # I don't really need a copied request here, because FileApp is so simple:
-        resp = request.get_response(app)
+        if os.path.isdir(filename):
+            if not request.path.endswith('/'):
+                new_url = request.path + '/'
+                if request.query_string:
+                    new_url += '?' + request.query_string
+                resp = exc.HTTPMovedPermanently(location=new_url)
+                return resp, orig_base, dest, proxied_url
+            ## FIXME: configurable?  StaticURLParser?
+            for base in ['index.html', 'index.htm']:
+                if os.path.exists(os.path.join(filename, base)):
+                    filename = os.path.join(filename, base)
+                    break
+            else:
+                resp = exc.HTTPNotFound("There was no index.html file in the directory")
+        if not os.path.exists(filename):
+            resp = exc.HTTPNotFound("The file %s could not be found" % filename)
+        else:
+            app = FileApp(filename)
+            # I don't really need a copied request here, because FileApp is so simple:
+            resp = request.get_response(app)
         return resp, orig_base, dest, proxied_url
         
 class ProxyMatch(AbstractMatch):
@@ -324,11 +359,13 @@
             default_objs=dict(AbortProxy=AbortProxy))
         header = el.get('header')
         content = el.get('content')
+        ## FIXME: the misspelling is annoying :(
         if (not header and content) or (not content and header):
             raise DeliveranceSyntaxError(
                 "If you provide a header attribute you must provide a content attribute, and vice versa",
                 element=el, source_location=source_location)
-        return cls(pyref, header, content, source_location)
+        return cls(pyref, header, content,
+                   source_location=source_location)
         
     def modify_request(self, request, log):
         if self.pyref:
@@ -378,10 +415,10 @@
             proxied_base += '/'
         if not orig_base.endswith('/'):
             orig_base += '/'
-        assert proxied_url.startswith(proxied_base), (
+        assert proxied_url.startswith(proxied_base) or proxied_url.split('?', 1)[0] == proxied_base[:-1], (
             "Unexpected proxied_url %r, doesn't start with proxied_base %r"
             % (proxied_url, proxied_base))
-        assert request.url.startswith(orig_base), (
+        assert request.url.startswith(orig_base) or request.url.split('?', 1)[0] == orig_base[:-1], (
             "Unexpected request.url %r, doesn't start with orig_base %r"
             % (request.url, orig_base))
         if self.pyref:
@@ -395,19 +432,24 @@
         if self.header:
             response.headers[self.header] = self.content
         if self.rewrite_links:
-            if response.content_type != 'text/html':
-                log.debug(self, 'Not rewriting links in response from %s, because Content-Type is %s' % (proxied_url, response.content_type))
-                return response
-            body_doc = document_fromstring(response.body, base_url=proxied_url)
-            body_doc.make_links_absolute()
             def link_repl_func(link):
                 if not link.startswith(proxied_base):
                     # External link, so we don't rewrite it
                     return link
                 new = orig_base + link[len(proxied_base):]
                 return new
-            body_doc.rewrite_links(link_repl_func)
-            response.body = tostring(body_doc)
+            if response.content_type != 'text/html':
+                log.debug(self, 'Not rewriting links in response from %s, because Content-Type is %s' % (proxied_url, response.content_type))
+            else:
+                if not response.charset:
+                    ## FIXME: maybe we should guess the encoding?
+                    body = response.body
+                else:
+                    body = response.unicode_body
+                body_doc = document_fromstring(body, base_url=proxied_url)
+                body_doc.make_links_absolute()
+                body_doc.rewrite_links(link_repl_func)
+                response.body = tostring(body_doc)
             if response.location:
                 ## FIXME: if you give a proxy like http://openplans.org, and it redirects to
                 ## http://www.openplans.org, it won't be rewritten and that can be confusing
@@ -418,7 +460,7 @@
             if response.headers.get('set-cookie'):
                 cook = response.headers['set-cookie']
                 old_domain = urlparse.urlsplit(proxied_url)[1].lower()
-                new_domain = req.host.split(':', 1)[0].lower()
+                new_domain = request.host.split(':', 1)[0].lower()
                 def rewrite_domain(match):
                     domain = match.group(2)
                     if domain == old_domain:
@@ -516,6 +558,12 @@
                    dev_users=dev_users, dev_expiration=dev_expiration,
                    source_location=source_location)
 
+    @classmethod
+    def parse_file(cls, filename):
+        file_url = filename_to_url(filename)
+        el = parse(filename, base_url=file_url).getroot()
+        return cls.parse_xml(el, file_url, traverse=True)
+
     @property
     def host(self):
         return self.server_host.split(':', 1)[0]

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxycommand.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxycommand.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxycommand.py	Wed Jul  9 08:54:05 2008
@@ -4,7 +4,6 @@
 from deliverance.proxy import ProxySet
 from deliverance.proxy import ProxySettings
 
-from lxml.etree import parse
 from paste.httpserver import serve
 import urllib
 
@@ -30,24 +29,54 @@
     dest='interactive_debugger',
     help='Use an interactive debugger (note: security hole when done publically; '
     'if interface is not explicitly given it will be set to 127.0.0.1)')
-
-def run_command(rule_filename, debug=False, interactive_debugger=False):
-    rule_url = 'file://' + urllib.quote(os.path.abspath(rule_filename).replace(os.path.sep, '/'))
-    el = parse(rule_filename, base_url=rule_url).getroot()
-    ## FIXME: rule_filename isn't browsable in the logs
-    ps = ProxySet.parse_xml(el, rule_url)
-    settings = ProxySettings.parse_xml(el, rule_url, traverse=True)
-    app = ps.application
-    app = settings.middleware(app)
+parser.add_option(
+    '--debug-headers',
+    action='count',
+    dest='debug_headers',
+    help='Show (in the console) all the incoming and outgoing headers; use twice for bodies')
+
+def run_command(rule_filename, debug=False, interactive_debugger=False, debug_headers=False):
+    settings = ProxySettings.parse_file(rule_filename)
+    app = ReloadingApp(rule_filename, settings)
     if interactive_debugger:
         from weberror.evalexception import EvalException
         app = EvalException(app, debug=True)
     else:
         from weberror.errormiddleware import ErrorMiddleware
         app = ErrorMiddleware(app, debug=debug)
+    if debug_headers:
+        from wsgifilter.proxyapp import DebugHeaders
+        app = DebugHeaders(app, show_body=debug_headers > 1)
     print 'To see logging, visit %s/.deliverance/login' % settings.base_url
     serve(app, host=settings.host, port=settings.port)
 
+class ReloadingApp(object):
+    """
+    This is a WSGI app that notices when the rule file changes, and
+    reloads it in that case.
+    """
+    def __init__(self, rule_filename, settings):
+        self.rule_filename = rule_filename
+        self.settings = settings
+        self.proxy_set = None
+        self.proxy_set_mtime = None
+        self.application = None
+        # This gives syntax errors earlier:
+        self.load_proxy_set(warn=False)
+        
+    def __call__(self, environ, start_response):
+        if (self.proxy_set is None
+            or self.proxy_set_mtime < os.path.getmtime(self.rule_filename)):
+            self.load_proxy_set()
+        return self.application(environ, start_response)
+
+    def load_proxy_set(self, warn=True):
+        if warn:
+            print 'Reloading rule file %s' % self.rule_filename
+        self.proxy_set = ProxySet.parse_file(self.rule_filename)
+        self.proxy_set_mtime = os.path.getmtime(self.rule_filename)
+        self.application = self.settings.middleware(self.proxy_set.application)
+
 def main(args=None):
     if args is None:
         args = sys.argv[1:]
@@ -59,7 +88,7 @@
     rule_filename = args[0]
     run_command(rule_filename,
                 interactive_debugger=options.interactive_debugger,
-                debug=options.debug)
+                debug=options.debug, debug_headers=options.debug_headers)
 
 if __name__ == '__main__':
     main()

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pyref.py	Wed Jul  9 08:54:05 2008
@@ -6,9 +6,12 @@
 from tempita import html_quote
 import os
 import new
+import urllib
 from UserDict import DictMixin
 from deliverance.exceptions import DeliveranceSyntaxError
 from deliverance.util.importstring import simple_import
+from deliverance.util.nesteddict import NestedDict
+from deliverance.util import filetourl
 
 __all__ = ['PyReference']
 
@@ -91,7 +94,7 @@
                     raise DeliveranceSyntaxError(
                         "The filename %r does not exist" % full_file,
                         element=s, source_location=source_location)
-        return cls(module_name=module, file=filename, function_name=func, args=args, 
+        return cls(module_name=module, filename=filename, function_name=func, args=args, 
                    attr_name=attr_name, default_objs=default_objs,
                    source_location=source_location)
 
@@ -112,7 +115,7 @@
         else:
             filename = self.expand_filename(self.filename, self.source_location)
             if filename not in self._modules:
-                name = self.pyfile.strip('/').strip('\\')
+                name = filename.strip('/').strip('\\')
                 name = os.path.splitext(name)[0]
                 name = name.replace('\\', '_').replace('/', '_')
                 new_mod = new.module(name)
@@ -120,6 +123,7 @@
                 for name, value in self.default_objs.items():
                     if not hasattr(new_mod, name):
                         setattr(new_mod, name, value)
+                execfile(filename, new_mod.__dict__)
                 self._modules[filename] = new_mod
             return self._modules[filename]
 
@@ -131,7 +135,12 @@
         obj = self.module
         for p in self.function_name.split('.'):
             ## FIXME: better error handling:
-            obj = getattr(obj, p)
+            try:
+                obj = getattr(obj, p)
+            except AttributeError, e:
+                raise Exception(
+                    "Could not get function %s: %s; existing attributes: %s"
+                    % (p, e, ', '.join(dir(obj))))
         return obj
     
     def __call__(self, *args, **kw):
@@ -144,10 +153,15 @@
         """
         Expand environmental variables in a filename
         """
-        vars = DefautDict(os.environ)
+        if source_location and source_location.startswith('file:'):
+            here = os.path.dirname(filetourl.url_to_filename(source_location))
+        else:
+            ## FIXME: this is a lousy default:
+            here = ''
+        vars = NestedDict(dict(here=here), DefaultDict(os.environ))
         tmpl = Template(filename)
         try:
-            return tmpl.substitute(os.environ)
+            return tmpl.substitute(vars)
         except ValueError, e:
             raise DeliveranceSyntaxError(
                 "The filename %r contains bad $ substitutions: %s"

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/deliv-sidebar.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/deliv-sidebar.html	Wed Jul  9 08:54:05 2008
@@ -0,0 +1,23 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-15">
+<title>The sidebar</title>
+</head>
+
+<body>
+
+This contains links that go into the sidebar.  Everything in the
+following div goes into the sidebar:
+
+<div id="menu" style="border: 2px solid #000">
+<ul>
+  <li><a href="/">Home/Wiki</a></li>
+  <li><a href="/trac">Trac bug tracker</a></li>
+  <li><a href="/reference">Reference documentation</a></li>
+</ul>
+
+</div>
+
+
+</body> </html>

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/deliv-theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/deliv-theme.html	Wed Jul  9 08:54:05 2008
@@ -0,0 +1,26 @@
+<html>
+<head>
+<title> | Deliverance</title>
+<link rel="stylesheet" href="style.css" type="text/css">
+</head>
+
+<body>
+<div id="header">
+  <h1 id="top-title"><a href="/">Deliverance:</a>
+      <span id="page-title"A Page</span></h1>
+</div>
+<!-- table-based layout, so retro of me! -->
+<table width="100%" id="table-layout">
+  <tr>
+    <td id="sidebar">
+      Some sidebar links.
+    </td>
+    <td id="theme-content">
+      Some content
+    </td>
+  </tr>
+</table>
+
+<div id="footer">Deliverance!</div>
+
+</body> </html>

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/rules.xml	Wed Jul  9 08:54:05 2008
@@ -1,7 +1,7 @@
 <ruleset>
  <theme href="/theme.html" />
  <rule>
-  <replace content="children:body" theme="children:#content" nocontent="abort" />
+  <replace content="children:body" theme="children:#theme-content" nocontent="abort" />
   <replace content="children:title" theme="children:#header" nocontent="ignore" move="0" />
   <replace content="children:#menu" href="/sidebar.html" theme="children:#sidebar" />
  </rule>

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/style.css	Wed Jul  9 08:54:05 2008
@@ -18,19 +18,53 @@
   vertical-align: top;
 }
 
-td#sidebar {
+#sidebar {
   background-color: #bfb;
   border-right: 2px solid #0f0;
   width: 8em;
   padding-top: 1em;
+  font-size: 85%
 }
 
-td#content {
+#sidebar li {
+  list-style: none;
+}
+
+#sidebar ul {
+  padding-left: 0;
+}
+
+#sidebar a, #sidebar a:link{
+  text-decoration: none;
+  width: 100%;
+  display: block;
+}
+
+#sidebar a:hover {
+  text-decoration: underline;
+  background-color: #8f8;
+}
+
+td#theme-content {
   border-top: 2px solid #0f0;
   padding: 0.5em;
 }
 
-h1#header {
+#header {
   background-color: #bfb;
   margin-bottom: 0;
+  width: 100%;
+}
+
+#header h1 {
+  margin: 0;
+}
+
+#header a, #header a:link, #header a:visited {
+  text-decoration: none;
+  color: #000;
+}
+
+#header a:hover {
+  text-decoration: underline;
 }

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-files/theme.html	Wed Jul  9 08:54:05 2008
@@ -12,7 +12,7 @@
     <td id="sidebar">
       Some sidebar links.
     </td>
-    <td id="content">
+    <td id="theme-content">
       Some content
     </td>
   </tr>

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml	Wed Jul  9 08:54:05 2008
@@ -7,44 +7,61 @@
     <dev-user username="guest" password="guest" />
   </server-settings>
 
-  <!-- 
-  <proxy path="/">
-    <dest next="1" />
-    <request header="X-Openplans-Project" content="opencore" />
-  </proxy>
-
-  <proxy path="/blog">
-    <dest href="http://localhost:10003" />
+  <proxy path="/theme-files">
+    <dest href="{here}/example-files/" />
   </proxy>
 
-  <proxy path="/tasks">
-    <dest href="http://localhost:10004" />
+  <proxy path="/trac" class="default trac">
+    <dest href="http://trac.openplans.org/deliverance" />
+    <response rewrite-links="1" />
   </proxy>
 
-  <proxy path="/theme">
-    <dest href="file://{here}/example-files/proxy-theme/" />
+  <proxy path="/reference" class="default sphinx">
+    <dest href="{here}/../../docs/_build/" />
   </proxy>
 
-  <proxy path="/">
-    [!- FIXME: I don't support {vh_SCRIPT_NAME} here -]
-    [!- <dest href="http://localhost:10001/VirtualHostBase/{wsgi.url_scheme}/{HTTP_HOST}/openplans/projects/{X-Openplans-Project}/VirtualHostRoot" /> -]
-    <dest href="http://localhost:10001/VirtualHostBase/{wsgi.url_scheme}/{HTTP_HOST}/openplans/VirtualHostRoot" />
+  <proxy path="/people" class="default openplans">
+    <dest href="http://www.openplans.org/people/" />
+    <response rewrite-links="1" />
+    <response pyref="file:$here/fixup_openplans.py:fixup_openplans_response" />
   </proxy>
-  -->
 
-  <proxy path="/theme-files">
-    <dest href="{here}/example-files/" />
-  </proxy>
+  <proxy path="exact:/login" class="default openplans">
+    <dest href="https://www.openplans.org/login/" />
+    <response rewrite-links="1" />
+    <response pyref="file:$here/fixup_openplans.py:fixup_openplans_response" />
+  </proxy>    
 
-  <proxy path="/">
-    <dest href="http://www.openplans.org" />
+  <proxy path="/" class="default openplans">
+    <dest href="http://www.openplans.org/projects/deliverance/" />
     <response rewrite-links="1" />
+    <response pyref="file:$here/fixup_openplans.py:fixup_openplans_response" />
   </proxy>
 
-  <theme href="/theme-files/theme.html" />
+  <theme href="/theme-files/deliv-theme.html" />
+
+  <rule class="openplans">
+    <replace content="children:#oc-content-main" theme="children:#theme-content" nocontent="abort" />
+    <append content="children:#oc-topnav-context" theme="children:#sidebar" />
+  </rule>
+
+  <rule class="trac">
+    <drop content="link[href$='/trac.css']" />
+    <replace content="children:#content" theme="children:#theme-content" nocontent="abort" />
+    <append content="children:#mainnav" theme="children:#sidebar" />
+    <append content="children:#metanav" theme="children:#sidebar" />
+  </rule>
+
+  <rule class="sphinx">
+    <!-- There's just a few really bad styles in here, unfortunately: -->
+    <drop content="link[href$='_static/default.css']" />
+    <replace content="children:.body" theme="children:#theme-content" nocontent="abort" />
+    <append content=".sphinxsidebar" theme="children:#sidebar" />
+  </rule>
 
   <rule>
-    <replace content="children:body" theme="children:#content" nocontent="abort" />
+    <replace content="children:title" theme="children:#page-title" nocontent="ignore" move="0" />
+    <replace content="children:#menu" href="/theme-files/deliv-sidebar.html" theme="children:#sidebar" />
   </rule>
 
 </ruleset>

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/fixup_openplans.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/fixup_openplans.py	Wed Jul  9 08:54:05 2008
@@ -0,0 +1,6 @@
+def fixup_openplans_response(request, response, orig_base, proxied_base, proxied_url, log):
+    user_base = 'http://www.openplans.org/people'
+    if response.location and response.location.startswith(user_base + '/'):
+        user_rest = response.location[len(user_base):]
+        response.location = '%s/people%s' % (request.application_url, user_rest)
+

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt	Wed Jul  9 08:54:05 2008
@@ -36,15 +36,15 @@
 
     >>> m = make('<match path="/foo" class="a" />')
     >>> match(m, Request.blank('/foo'), Response(), [])
-    True
+    ['a']
     >>> match(m, Request.blank('/foobar'), Response(), [])
     log: Skipping class="a" because request URL (/foobar) does not match path="path:/foo/"
     False
     >>> match(m, Request.blank('/foo/bar'), Response(), [])
-    True
+    ['a']
     >>> m = make('<match response-header="Content-Type: contains: html" class="x" />')
     >>> match(m, Request.blank('/'), Response(content_type='text/plain'), [('content-type', 'text/plain')])
     log: Skipping class="x" because the response headers Content-Type do not match response-header="Content-Type: contains:html"
     False
     >>> match(m, Request.blank('/'), Response(content_type='text/html'), [('content-type', 'text/html')])
-    True
+    ['x']

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/filetourl.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/util/filetourl.py	Wed Jul  9 08:54:05 2008
@@ -0,0 +1,32 @@
+import urllib
+import os
+import re
+
+drive_re = re.compile('^([a-z]):', re.I)
+url_drive_re = re.compile('^([a-z])[|]', re.I)
+
+def filename_to_url(filename):
+    """
+    Convert a path to a file: URL.  The path will be made absolute.
+    """
+    filename = os.path.normcase(os.path.abspath(filename))
+    url = urllib.quote(filename)
+    if drive_re.match(url):
+        url = url[0] + '|' + url[2:]
+    url = url.replace(os.path.sep, '/')
+    url = url.lstrip('/')
+    return 'file:///' + url
+
+def url_to_filename(url):
+    """
+    Convert a file: URL to a path.
+    """
+    assert url.startswith('file:'), (
+        "You can only turn file: urls into filenames (not %r)" % url)
+    filename = url[len('file:'):].lstrip('/')
+    filename = urllib.unquote(filename)
+    if url_drive_re.match(filename):
+        filename = filename[0] + ':' + filename[2:]
+    else:
+        filename = '/' + filename
+    return filename

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/setup.py	Wed Jul  9 08:54:05 2008
@@ -37,6 +37,7 @@
         "WebError",
         "DevAuth",
         "Paste",
+        "WSGIFilter",
         ],
       dependency_links=[
         "https://svn.openplans.org/svn/DevAuth/trunk#egg=DevAuth-dev",

From ianb at codespeak.net  Wed Jul  9 20:57:32 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Wed,  9 Jul 2008 20:57:32 +0200 (CEST)
Subject: [z3-checkins] r56395 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk: deliverance docs
Message-ID: <20080709185732.922492A8006@codespeak.net>

Author: ianb
Date: Wed Jul  9 20:57:30 2008
New Revision: 56395

Added:
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt   (contents, props changed)
Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt
Log:
configuration; the complete docs\!

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	Wed Jul  9 20:57:30 2008
@@ -20,6 +20,7 @@
     def __init__(self, path=None, domain=None,
                  request_header=None, response_header=None, environ=None,
                  pyref=None, source_location=None):
+        ## FIXME: this should add response_status
         self.path = path
         self.domain = domain
         self.request_header = request_header
@@ -35,7 +36,7 @@
         Parses out the match-related arguments
         """
         path = cls._parse_attr(el, 'path', default='path')
-        domain = cls._parse_attr(el, 'domain', default='wildcard')
+        domain = cls._parse_attr(el, 'domain', default='wildcard-insensitive')
         request_header = cls._parse_attr(el, 'request-header', default='exact', header=True)
         response_header = cls._parse_attr(el, 'response-header', default='exact', header=True)
         environ = cls._parse_attr(el, 'environ', default='exact', header=True)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/ruleset.py	Wed Jul  9 20:57:30 2008
@@ -150,6 +150,8 @@
         headers.append((http_equiv, content))
     return headers
 
+# Note: these are included in the documentation; any changes should be
+# reflected there as well.
 standard_rule = Rule.parse_xml(XML('''\
 <rule>
   <!-- FIXME: append-or-replace for title? -->

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt	Wed Jul  9 20:57:30 2008
@@ -0,0 +1,587 @@
+Deliverance Configuration
+=========================
+
+.. contents::
+
+All the configuration for Deliverance goes in an XML file.  This file can configure page matches, transformation rules, proxying, and server settings.  Though it is all in one file, there are several major sections.  A quick overview:
+
+`rule and theme`_:
+    Determine the theme and apply the actual transformations to the page.
+`match and page classes`_:
+    In more complicated sites, page classes allow you to apply rules based on different criteria.
+`proxy and server-settings`_: 
+    Controls the proxy destinations, server, and security settings.
+`request/response matching`_:
+    Several elements can be conditionally used only when the request or response matches conditions.  All these elements share common behavior and attributes.
+`pyref Python references`_:
+    Several elements can call Python hooks.  This describes the general syntax, while the details of the hook are described alongside the element.
+`developer debugging console`_:
+    This gives you information about what Deliverance is doing, on a page-by-page basis.
+
+Everything goes in a ``<ruleset>`` tag.
+
+.. comment: FIXME: should this be <deliverance>?
+
+rule and theme
+--------------
+
+This is the core of what Deliverance does: take a theme along with your content, and apply transformations.
+
+theme
+~~~~~
+
+The ``<theme>`` element defines the theme you'll be using.  The theme is given as a URL.  The basic form is:
+
+.. code-block:: xml
+
+    <theme href="/theme.html" />
+
+This defines the theme as being at ``/theme.html``.  If possible it will be fetched with an internal request, though external requests are also possible (if you host the theme outside of Deliverance).
+
+If you have ``<theme>`` at the global level (in the ``<ruleset>`` element) then that is the default theme.  You can also put it inside ``<rule>`` elements, and then it will apply just to that rule set.  This is useful if you are using `match and page classes`_.
+
+The theme element also supports `pyref Python references`_.  If you use:
+
+.. code-block:: xml
+
+    <theme pyref="mymodule:get_theme" />
+
+Then define a function like:
+
+.. code-bock:: python
+
+    def get_theme(request, response, log):
+        return "/%s/theme.html" % request.host
+
+The ``request`` and ``response`` objects are `WebOb <http://pythonpaste.org/webob/>`_ objects.
+
+The default function name is ``get_theme``.
+
+rule
+~~~~
+
+The ``<rule>`` element defines a set of transformations.  It also supports `page classes`_ and `request/response matching`_, which you can read about in those sections.  Also, as mentioned above, you can include a ``<theme>`` element.  But the most important thing is the transformation actions.
+
+The transformation actions are applied in order.  The starting point for the transformation is the *theme* document, and the actions copy elements from the content into the theme.
+
+There are four actions:
+
+``<replace>``:
+    Replaces something in the theme with elements from the content.
+``<append>``:
+    Appends content to an element in the theme.
+``<prepend>``:
+    Prepends content to an element in the theme.
+``<drop>``:
+    Removes elements from the theme or the content.
+
+Selection and selection types
+++++++++++++++++++++++++++++++
+
+Each rule depends on selecting elements from the theme and content.  The most basic selection is done with CSS 3 selectors.  For instance, this places the element in the content with the id ``content`` *after* the element in the theme with the id ``header``:
+
+.. code-block:: xml
+
+    <append content="#content" theme="#header" />
+
+You can also use `XPath <http://www.w3.org/TR/xpath>`_ selectors.  Any selector starting with ``/`` is treated as an XPath expression, while everything else is treated as CSS.  CSS can only select elements, and while XPath can select text or attributes Deliverance is only interested in elements.  Moving elements around has some limitations, so there are different explicit types of selection:
+
+.. comment: FIXME: a better XPath link would be nice, like to a tutorial.
+
+``elements:``
+    The default, this applies the rules to the elements selected.
+``children:``
+    A common type, this applies rules to the *children* of the elements selected (including text content of the elements).
+``attributes:``
+    This applies the rules to just the attributes.  Also you can apply it to just specifically named attributes, for instance just to the ``class`` and ``style`` attributes with ``attributes(class,style):``.
+``tag:``
+    This applies the rule to the tag, but not the children of the element.  For instance, dropping a tag keeps the children in the document, but removes their enclosing tag.
+
+You can apply any of these like ``content="children:#content"``.  Not all combinations make sense, and some are not allowed.  For instance, ``<replace content="attributes:#content" theme="elements:#header" />`` does not make sense, as you can't replace elements with attributes.  Generally ``elements:`` and ``children:`` work together, ``attributes:`` only works with ``attributes:``, and ``tag:`` only works with ``tag:``.
+
+When selecting elements you can use the ``||`` operator.  This applies to both CSS and XPath selectors, and with the operator you can mix the two.  The ``||`` operator takes the results of the first selector that matches anything.  So ``content="#content || children:body"`` will take the element ``#content`` if there is one, and if there is not one it will take all the children of ``<body>``.  You can mix ``elements:`` and ``children:`` using ``||``, though no other types can be mixed like this.
+
+``<replace>``
++++++++++++++
+
+The ``<replace>`` action replaces something in the theme with something in the content.  Exactly what is replaced depends on the selection type.  Some examples:
+
+.. code-block:: xml
+
+    <replace content="children:#content-wrapper" theme="children:#content" />
+
+this replaces the elements *inside* the theme element ``#content`` with the elements inside the content element ``#content-wrapper``.  The resulting document won't have any element with the id ``#content-wrapper`` (unless the theme already had an element with that id).
+
+.. code-block:: xml
+
+    <replace content="elements:#content-wrapper" theme="elements:#content" />
+    <replace content="#content-wrapper" theme="#content" />
+
+both of these are the same (``elements:`` is the default selection type).  This replaces the theme element ``#content`` with the content element ``#content-wrapper``.  The resulting document has no element ``#content``.
+
+.. code-block:: xml
+
+    <replace content="attributes:body" theme="attributes:body" />
+
+this removes all the attributes (e.g., ``class``, ``onload``) from the ``<body>`` element in the theme, and moves over the attributes from the content body element.
+
+.. code-block:: xml
+
+    <replace content="tag:#content" theme="tag:#content" />
+
+this replaces the tag ``#content`` in the theme with its corresponding tag from the theme.  They might not be the same tag name (e.g., the theme might be a ``<p>`` and the content ``<div>``), and all attributes will be taken from the content.
+
+``<append>`` and ``<prepend>``
+++++++++++++++++++++++++++++++
+
+These actions obvious are very similar; ``<append>`` puts things from te content after things in the theme, and ``<prepend>`` puts things from the content before things in the theme.  Some examples:
+
+.. code-block:: xml
+
+    <append content="children:#sidenav" theme="children:#sidebar" />
+
+this moves the children of the content element ``#sidenav`` to the end of the theme element ``#sidebar``, combining the navigation of the theme and content.  If you wanted the content navigation to go first you'd use:
+
+.. code-block:: xml
+
+    <prepend content="children:#sidenav" theme="children:#sidebar" />
+
+Another example:
+
+.. code-block:: xml
+
+    <append content="children:#sidenav" theme="ol.menulinks" />
+
+this moves the children of ``#sidenav`` *after the element* with the element in the theme ``<ol class="menulinks">``.
+
+.. code-block:: xml
+
+    <append content="li.reference" theme="children:ol.menulinks" />
+
+this moves any element in the content like ``<li class="reference">`` *into* the element in the theme ``<ol class="menulinks">``.
+
+You can also use these with attributes:
+
+.. code-block:: xml
+
+    <append content="attributes:div#content" theme="attributes:div#content-wrapper" />
+
+This adds any attributes from ``div#content`` into the theme element ``div#content-wrapper`` -- but when the attribute already exists in the theme, the theme attribute is kept.  If you use ``<prepend>`` then when there are overlapping attributes the content attribute value is kept.
+
+.. comment: FIXME: should attributes(class) know about how classes can be combined?
+
+You can't use ``tag:`` with these actions.
+
+``<drop>``
+++++++++++
+
+The ``<drop>`` action is used to remove problematic elements from a theme or content.  You can also use it with `if-content`_ for other kinds of conditions.  Since this action doesn't move elements around you don't need to provide both ``content`` and ``theme`` attributes; either will do.
+
+A common example is removing a stylesheet which introduces conflicts:
+
+.. code-block:: xml
+
+    <drop content="link[href $= '/sitestyle.css']" />
+
+This is one of the more advanced selectors that CSS 3 allows.  You can't (confidently) use it in browsers yet, but you can in Deliverance!  The ``$=`` operator means *ends-with*, so this selector could be described as: *all link elements with href attributes that end with '/sitestyle.css'*.  Other operators are the obvious ``=``, ``^=`` which means *starts-with*, ``*=`` which means *contains*.  Note that all comparisons are case-sensitive.
+
+.. comment: FIXME: should those realy be case-sensitive?
+
+Another example:
+
+.. code-block:: xml
+
+    <drop content="attributes(class):a.external-link" />
+
+which removes the class from any ``<a class="external-link">`` elements.
+
+.. code-block:: xml
+
+    <drop content="tag:font" />
+
+which removes all ``<font>`` tags, but doesn't remove any actual text content.
+
+``if-content``
+++++++++++++++
+
+All the actions can take an attribute ``if-content="selector"``.  This selector is attempted on the content, and if it doesn't match anything then the action is skipped.
+
+Zero or multiple matches and error handling
++++++++++++++++++++++++++++++++++++++++++++
+
+Except for ``<drop>``, in actions it is necessary that the ``theme`` selector match exactly one element.  If it doesn't match *any* element, then the action can't be performed.  If it matches many elements then the action is ambiguous: which element should be the target?  Also, when no content element is matched it is a sign something is wrong, and in some cases if multiple content elements are matched it is a problem.
+
+The default handling for all these is to log the problem at the level "warn".  If there are multiple matches when that is not expected, the default handling is to log and use the first element.  There are four attributes to override this:
+
+``notheme``:
+    The handling when no theme element is found.
+``manytheme``:
+    The handling when more than one theme element is found.
+``nocontent``:
+    The handling when no content elements are found.
+``manycontent``:
+    The handling when more than on content element is found (and that doesn't make sense in the context of the action).
+
+A value ``notheme="ignore"`` means that the action is ignored and only a "debug" level logging message is produced.  A value like ``nocontent="abort"`` means that if there is no content then all theming will be aborted, and the unthemed content page will be displayed.  For ``manytheme`` and ``manycontent`` you can indicate ``first`` or ``last`` to select the first (default) or last element.  You can combine this like ``manytheme="ignore:first"``.
+
+An example:
+
+.. code-block:: xml
+
+    <replace content="children:#content" theme="children:#content"
+             nocontent="abort" />
+
+Almost all rules should have at least one action with ``nocontent="abort"``.  This is the action that moves the primary body of the content into the theme.  If that primary body can't be found then when people browse the page they'll see the theme and lose all the useful content.
+
+``manycontent`` is used when you are using selection types ``attributes:`` or ``tag:`` -- in both these cases the theme and content element match up one-to-one.
+
+When using ``<drop>`` multiple theme and content elements are expected, so no error handling is necessary in that case.
+
+Including external content with ``href``
+++++++++++++++++++++++++++++++++++++++++
+
+In addition to manipulating the theme and content documents, you can bring in content from a third source using the ``href`` attribute on a rule.
+
+If you include ``href`` the document at that location is used instead of normal content document.  For example:
+
+.. code-block:: xml
+
+    <append href="/sidebar" content="children:body" 
+            theme="children:#sidebar" />
+
+This appends all the elements in the body of ``/sidebar`` into the theme.  This can be used to make the theme more dynamic.  Deliverance doesn't produce content on its own, or modify content in complex ways, but you can use includes like these to introduce several sources of dynamic content into a single page: ``/sidebar`` can be a dynamically generated page itself.
+
+Moving and copying
+++++++++++++++++++
+
+By default actions *move* elements from the content to the theme.  That is, if you select content with an action, later actions won't be able to access that action.  This is particularly useful in cases like this:
+
+.. code-block:: html
+
+    <body>
+      <div class="navigation">links...</div>
+      Some content
+    </body>
+
+with the rules:
+
+.. code-block:: xml
+
+    <rule>
+      <replace content="children:#navigation" theme="children:#sidebar" />
+      <replace content="children:body" theme="children:#content"
+               nocontent="abort" />
+    </rule>
+
+If elements were copied the navigation would show up twice in the resulting page.  If you *do* want content copied instead of moved then add ``move="0"`` to the action.
+
+Standard actions and ``suppress-standard``
+++++++++++++++++++++++++++++++++++++++++++
+
+There are several actions that are "standard".  These are common for HTML generally.  The actions are:
+
+.. code-block:: xml
+
+  <rule>
+    <replace content="children:/html/head/title"
+             theme="children:/html/head/title" nocontent="ignore" />
+    <append content="elements:/html/head/link"
+            theme="children:/html/head" nocontent="ignore" />
+    <append content="elements:/html/head/script"
+            theme="children:/html/head" nocontent="ignore" />
+    <append content="elements:/html/head/style"
+            theme="children:/html/head" nocontent="ignore" />
+  </rule>
+
+These copy over the title and any links, scripts, or stylesheets from the content into the theme.  These are always applied unless you use ``<rule suppress-standard="1">``.
+
+
+.. _`page classes`:
+.. _`match`:
+
+match and page classes
+----------------------
+
+Note: for a simple site you can mostly ignore this, and just define a single ``<rule>`` that is applied to all requests.  But in many cases you will need to apply specific rules only to parts of the website.  For example, different applications may have navigation in specific parts of the page, and you want rules to move that navigation around the page that is specific to the application.
+
+Often different sets of rules will apply to different parts of the site.  To define what rules go with what requests/responses there is the concept of "page classes".  These are classes that apply to the page, and any rules with those classes are used.  For example, imagine a page has the classes ``trac`` and ``default``.  Then these rules would be used:
+
+.. code-block:: xml
+
+    <rule class="trac">...</rule>
+    <rule class="default">...</rule>
+
+Like CSS any rules that match any of the classes on a page will be run.  Also like CSS you can have multiple classes anywhere, separated by spaces.  Unlike CSS the classes are ordered, and define the order the rules are run in.
+
+.. comment: FIXME: should they be run according to the order of the rules in the ruleset?
+
+There are several ways to define classes on a page.  If you have control of the server you can add the response header ``X-Deliverance-Page-Class`` with the page classes.  You can also add a ``class`` attribute to `proxy`_ elements.
+
+A third way is with ``<match>``.  These elements match requests and add classes.  You can use them like:
+
+.. code-block::
+
+    <match domain="lists.*" class="lists" />
+
+This will add the "lists" class to any requests going to a domain ``lists.*``.  The request matching is described in `request/response matching`_.
+
+proxy and server-settings
+-------------------------
+
+Deliverance comes with a proxy, ``deliverance-proxy``.  This starts a server and proxies HTTP requests to other backend servers (Zope, Apache/PHP, paster, etc).
+
+Settings for the proxying also go in the rule file.  The ``<server-settings>`` element defines aspects of the server, while ``<proxy>`` defines specific servers to proxy to.
+
+.. _`server-settings`:
+
+``<server-settings>``
+~~~~~~~~~~~~~~~~~~~~~
+
+This element looks like:
+
+.. code-block:: xml
+
+    <server-settings>
+      <server>localhost:8080</server>
+      <execute-pyref>false</execute-pyref>
+      <display-local-files>false</display-local-files>
+      <dev-allow>
+        127.0.0.1
+        192.168.0.1/24
+      </dev-allow>
+      <dev-deny>
+        192.168.0.121
+      </dev-deny>
+      <dev-htpasswd>/etc/deliverance-dev-users.htpasswd</dev-htpasswd>
+      <dev-user username="bob" password="uncle" />
+      <dev-expiration>60</dev-expiration>
+    </server-settings>
+
+You can't actually use quite all of these together.  Going over the individual settings:
+
+``<server>``:
+    This gives the host and port to use.  If you use ``localhost`` or ``127.0.0.1`` for the host then only local connections are allowed.  If you use ``0.0.0.0`` then the server is started up on all your interfaces.  The default is ``localhost:8080``.
+
+``<execute-pyref>``:
+    This defaults to true.  If this is true then ``pyref`` attributes are allowed (see `pyref Python References`_).  This gives anyone who can write to your rules the ability to execute arbitrary code for each request, so you should turn it off if untrusted people have that access.
+
+``<display-local-files>``:
+    The `developer debugging console`_ will, by default, display any local files.  If the person accessing that console generally has ssh or other access to the server this isn't a problem.  But if not you should turn this off.
+
+The ``<dev-*>`` tags define access to the `developer console`_.
+
+``<dev-allow>``:
+    This is a list of IP addresses (or IP+mask) that are allowed access.
+
+``<dev-deny>``:
+    These addresses are specifically disallowed.
+
+``<dev-htpasswd>``:
+    This is a list of usernames and passwords created with the Apache ``htpasswd`` program.  You log in using these.
+
+``<dev-user>``:
+    Instead of ``<dev-htpasswd>`` you can put username/passwords directly in.  This isn't a very good idea, but combined with a restrictive ``<dev-allow>`` it's not so bad.  You can't use both this and ``<dev-htpasswd>``.  You must use one of these to get access to the console.
+
+``<dev-expiration>``:
+    The time, in minutes, that a session can last.  You have to re-login after this amount of time.  This defaults to 0, meaning no expiration.
+
+.. comment: FIXME: what's the default IP restriction?
+.. comment: FIXME: say something about variable substitution.
+
+.. _proxy:
+
+``<proxy>``
+~~~~~~~~~~~
+
+The ``<proxy>`` element is what defines what gets routed where.  The basic pattern is:
+
+.. code-block:: xml
+
+    <proxy path="/trac">
+      <dest href="http://localhost:10002" />
+    </proxy>
+
+This proxies any requests to ``/trac`` to the dest address.  A request to something like ``/trac/view/1`` will go to ``http://localhost:10002/view/1``.  If you wanted to keep ``/trac`` you can use ``<proxy strip-script-name="0">``.
+
+If you want to pass the ``Host`` header through without changing it, use ``<proxy keep-host="1">``.  This generally represents the request accurately, but has not been the norm for systems like this, and often the original host name is only available in ``X-Forwarded-Host``.  "Real" proxies like Squid usually preserve the Host header.  Apache does not unless you configure it to do so.  So if you have setup your system for Apache proxying you probably weren't preserving the Host header.
+
+You can also match against things besides the path; anything in `request/response matching`_ is available.  But without a simple ``path`` match the path stripping won't work.  (A path match like ``path="regex:.*/manage$"`` wouldn't work, for instance.)
+
+You can also add `page classes`_ for the proxied request, by using ``<proxy class="page-class">``.
+
+The proxy can contain several elements...
+
+proxy: ``<dest>``
++++++++++++++++++
+
+The ``<dest>`` element defines the destination.  You must have a dest element.
+
+You can give an ``href`` value of both ``http://...`` URLs and ``file:///...`` URLs.  Files are served directly without proxying, though this is seemless to the rest of the process.
+
+The value in ``href`` can be a URI template (though only the simplest form of template).  You can use headers like ``{Host}``, environmental variables like ``{REMOTE_USER}``, or the variable ``{here}`` which points to the directory containing the rule file.  I.e., if the rule file is in ``/etc/deliverance/rules/ruleset.xml`` then ``{here}`` would be ``file:///etc/deliverance/rules``.  This template is substituted for every request, so it can be fully dynamic.
+
+In addition to this you can use `pyref Python references`_, like:
+
+.. code-block:: xml
+
+    <proxy path="/trac">
+      <dest pyref="mymodule:get_proxy_dest" />
+    </proxy>
+
+with the function:
+
+.. code-block:: python
+
+    def get_proxy_dest(request, log):
+        if not request.remote_addr.startswith('192'):
+            raise AbortProxy('Bad remote_addr: %r' % request.remote_addr)
+        return 'http://localhost:10002'
+
+You can return any URL.  URI template substitution is *not* performed.  If you raise ``AbortProxy`` then the ``<proxy>`` will be skipped, and another proxy will be looked for.  (If nothing matches a 404 error is returned.)
+
+If you want to perform request rewriting (as described in the next section) you can use ``<dest next="1">`` and the request modifications will be performed but otherwise the proxy section will be skipped.
+
+proxy: ``<request>``
+++++++++++++++++++++
+
+This modifies the request before it is proxied on.  You can add headers and run Python code against the request.
+
+To add a header:
+
+.. code-block:: xml
+
+    <request header="X-Project-Name" content="My Project" />
+
+You can also use a `pyref Python reference`_, like:
+
+.. code-block:: xml
+
+    <request pyref="mymodule:modify_proxy_request" />
+
+with the code:
+
+.. code-block:: python
+
+    def modify_proxy_request(request, log):
+        request.header['X-Project-Name'] = request.host.split('.')[0]
+        return request
+
+You can modify the request in-place or return a new request.
+
+proxy: ``<response>``
++++++++++++++++++++++
+
+This modifies the response.  Like the request you can add headers:
+
+.. code-block:: xml
+
+    <response header="Cache-Control" content="max-age=0" />
+
+You can also rewrite links, which means you can proxy from a site that doesn't particularly want to be proxied.  So if you proxy to ``http://othersite.com`` then of course all the links it returns will be to ``http://othersite.com``.  Using link rewriting it will rewrite all those links to go back through the proxy.  This includes the ``Location`` header, any references to ``domain`` in cookies, and all links in the HTML.
+
+.. code-block:: xml
+
+    <response rewrite-links="1" />
+
+And of course there's pyref:
+
+.. code-block:: xml
+
+    <response pyref="mymodule:modify_proxy_response" />
+
+with the code:
+
+.. code-block:: python
+
+    def modify_proxy_response(
+        request, response, orig_base, proxied_base, proxied_url, log):
+        # orig_base: the original URL base: http://localhost:8080/trac
+        # proxied_base: where dest sent it to: http://localhost:10001/
+        # proxied_url: the full destination, e.g.,
+        #    http://localhost:10001/view/1
+        # request.url: the full original URL, e.g.,
+        #    http://localhost:8080/trac/view/1
+        response.body += 'look at me!'
+        return response
+
+request/response matching
+-------------------------
+
+Many elements can match requests and responses: the `match`_ element, `rule`_, and `proxy`_.  They all use the same matching.
+
+There are several attributes that match against different parts of the request or response.  If you provide multiple attribute then they must all match.
+
+String Matching
+~~~~~~~~~~~~~~~
+
+These attributes match a string value against a pattern you provide.  There are several kinds of patterns; different attributes default to different types of patterns, whatever is most logical for the attribute value.
+
+First, the attributes:
+
+``path``:
+    This matches the request path.  The default pattern type is ``path:``.
+
+``domain``:
+    This matches the request host, the domain name of the request URL.  The default pattern type is ``wildcard-insensitive:``.
+
+Each pattern string can start with ``patterntype:`` which defines what kind of pattern it is.  Here's a list of the available patterns (these also apply to the key/value matching described in the next section):
+
+``wildcard:``
+    This is a wildcard match, i.e., you can use ``*``.
+
+``wildcard-insensitive:``
+    Wildcards that are case-insensitive.
+
+``regex:``
+    Matches with a `regular expression <http://python.org/doc/current/lib/re-syntax.html>`_.  Note you can use things like ``(?i)`` to make the expression case-insensitive.
+
+``path:``
+    Matches the string as a path prefix.  It's like starts-with, except it is aware of /'s.  So ``path:/some-path`` will match the path ``/some-path/to/a/place``, ``/some-path/`` and ``/some-path``, but it will not match ``/some-path-to-somewhere``.
+
+``exact:``
+    This matches the exact string.
+
+``exact-insensitive:``
+    The exact string, except case.
+
+``contains:``
+    True if the pattern shows up anywhere in the string.
+
+``contains-insensitive:``
+    Contains, case-insensitive.
+
+``boolean:``
+    This tests if the string is "true".  Values "1", "true", "yes", and "on" are true.  Empty, or any other value, is false.  The pattern is fairly insignificant, except ``boolean:not`` inverts the match.
+
+Key/Value Matching
+~~~~~~~~~~~~~~~~~~
+
+There are other matches that match both a key and a value, like matching headers and environmental keys.  These look like ``environ="REMOTE_USER: bob"``, like ``key: pattern``.  The key can be a wildcard, and if any key that matches that wildcard matches the pattern then it is a match.  Only wildcards are allowed for the key.  All patterns default to ``exact:``.
+
+The attributes available:
+
+``request-header``:
+    Matches a request header.  The headers are case-insensitive.
+
+``response-header``:
+    Matches the response headers.
+
+``environ``:
+    Matches keys in the request environment.
+
+.. _`pyref Python reference`:
+
+pyref Python references
+-----------------------
+
+``pyref`` attribute references allow you to hook into Python code.  To read about the details see the `pyref <pyref.html>`_ document.
+
+.. _`developer console`:
+
+Developer Debugging Console
+---------------------------
+
+You must be logged in to view the developer console.  The login page is available at ``/.deliverance/login`` -- you define the users using the `server-settings`_ configuration.
+
+After you log in you can add ``?deliv_log`` to any page URL to get a log at the bottom of the page.  The log describes in detail how the page was transformed, including information about how the proxy was chosen and any subrequests.  It also lets you browse the source involved, see what the selectors select in the content or theme, or get a list of interesting ids and classes in the content.
+

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt	Wed Jul  9 20:57:30 2008
@@ -5,9 +5,12 @@
 
 .. toctree::
 
-   rules
-   proxy
+   configuration
    pyref
-   page-classes
    news
    license
+   page-classes
+   rules
+   proxy
+
+.. comment: FIXME: should those last 3 documents just be deleted?

From ianb at codespeak.net  Wed Jul  9 21:11:30 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Wed,  9 Jul 2008 21:11:30 +0200 (CEST)
Subject: [z3-checkins] r56396 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/docs
Message-ID: <20080709191130.72DE32D8007@codespeak.net>

Author: ianb
Date: Wed Jul  9 21:11:27 2008
New Revision: 56396

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt
Log:
added note about pyref matching

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt	Wed Jul  9 21:11:27 2008
@@ -569,6 +569,20 @@
 ``environ``:
     Matches keys in the request environment.
 
+Python References
+~~~~~~~~~~~~~~~~~
+
+You can also use a ``pyref``, with a function like:
+
+.. code-block:: python
+
+    def match_request(request, response, response_headers, log):
+        if response.headers.get('x-notheme'):
+            raise AbortTheme
+        return True
+
+Note that ``response_headers`` is a list of all the headers, including ``<meta http-equiv>`` headers.
+
 .. _`pyref Python reference`:
 
 pyref Python references

From ianb at codespeak.net  Fri Jul 11 17:46:57 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Fri, 11 Jul 2008 17:46:57 +0200 (CEST)
Subject: [z3-checkins] r56459 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/docs
Message-ID: <20080711154657.8288916A05C@codespeak.net>

Author: ianb
Date: Fri Jul 11 17:46:56 2008
New Revision: 56459

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt
Log:
fix typo

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt	Fri Jul 11 17:46:56 2008
@@ -254,7 +254,7 @@
 Moving and copying
 ++++++++++++++++++
 
-By default actions *move* elements from the content to the theme.  That is, if you select content with an action, later actions won't be able to access that action.  This is particularly useful in cases like this:
+By default actions *move* elements from the content to the theme.  That is, if you select content with an action, later actions won't be able to access those content elements.  This is particularly useful in cases like this:
 
 .. code-block:: html
 

From paul at codespeak.net  Mon Jul 28 23:01:57 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Mon, 28 Jul 2008 23:01:57 +0200 (CEST)
Subject: [z3-checkins] r56823 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080728210157.B7B22398006@codespeak.net>

Author: paul
Date: Mon Jul 28 23:01:55 2008
New Revision: 56823

Added:
   z3/deliverance/sandboxes/paul/xdv/
   z3/deliverance/sandboxes/paul/xdv/README.txt
   z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
   z3/deliverance/sandboxes/paul/xdv/content.html
   z3/deliverance/sandboxes/paul/xdv/rules.xml
   z3/deliverance/sandboxes/paul/xdv/theme.html
Log:
Doing a prototype that can run in Apache using mod_transform XSLT

Added: z3/deliverance/sandboxes/paul/xdv/README.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/README.txt	Mon Jul 28 23:01:55 2008
@@ -0,0 +1,54 @@
+========================================
+xdv - XSLT engine for Deliverance
+========================================
+
+xdv is a mini-project to implement a subset of Deliverance using a
+pure XSLT engine.  With xdv, you "compile" your theme and ruleset in
+one step, then use a superfast/simple transform on each request
+thereafter.  Alternatively, compile your theme during development,
+check it into Subversion, and not touch xdv during deployment.
+
+Usage
+----------------
+
+1) Edit the ``theme.html``, ``rules.xml``, and ``content.html``.
+
+2) xsltproc compiler.xsl theme.html > compiledtheme.xsl
+
+3) xsltproc compiledtheme.xsl content.html > output.html
+
+4) open output.html
+
+Specifying the rules file
+------------------------------
+
+The Deliverance rules file is read into the compiler.xsl file using
+the XSLT ``document()`` function.  This function is given a URI, which
+it loads and parses.
+
+By default it looks for ``rules.xml`` in the same directory as the
+``compiler.xsl``.  However, you can pass a parameter in when compiling
+the theme::
+
+  xsltproc --stringparam rulesuri rules.xml compiler.xsl theme.html > compiledtheme.xsl
+
+
+
+To Do
+-------
+
+- Add support for <drop>
+
+- Make sure attribute merging is handled
+
+
+Not Supported
+--------------------
+
+1) Multiple themes or "page types"
+
+2) Link rewriting
+
+3) CSS Selector syntax
+
+

Added: z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	Mon Jul 28 23:01:55 2008
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+    <xsl:template match="/">
+        <x insert="insert"/>
+    </xsl:template>
+</xsl:stylesheet>

Added: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Mon Jul 28 23:01:55 2008
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns:dv="http://openplans.org/deliverance" xmlns:exsl="http://exslt.org/common"
+    xmlns:dyn="http://exslt.org/dynamic" xmlns:xml="http://www.w3.org/XML/1998/namespace"
+    exclude-result-prefixes="dv dyn exsl xml" version="1.0">
+    <xsl:output indent="yes"/>
+    <xsl:param name="rulesuri">rules.xml</xsl:param>
+    <!-- Multi-stage theme compiler -->
+    <xsl:template match="/">
+        <!-- Put unique xml:id values on all the theme html -->
+        <xsl:variable name="themehtml-rtf">
+            <xsl:apply-templates select="/html" mode="annotate-html"/>
+        </xsl:variable>
+        <xsl:variable name="themehtml" select="exsl:node-set($themehtml-rtf)"/>
+
+        <!-- Include the rules file, adding @xml:id attributes as it is included -->
+        <xsl:variable name="rules-rtf">
+            <xsl:apply-templates select="document($rulesuri)/*" mode="annotate-rules">
+                <xsl:with-param name="themehtml" select="$themehtml"/>
+            </xsl:apply-templates>
+        </xsl:variable>
+        <xsl:variable name="rules" select="exsl:node-set($rules-rtf)"/>
+
+        <!-- Make a pass through all the theme html, filtering 
+            as we go. -->
+        <xsl:variable name="stage1-rtf">
+            <xsl:apply-templates select="$themehtml" mode="apply-rules">
+                <xsl:with-param name="rules" select="$rules"/>
+            </xsl:apply-templates>
+        </xsl:variable>
+        <xsl:variable name="stage1" select="exsl:node-set($stage1-rtf)"/>
+
+        <!-- Stage 2, include the boilerplate and make a compiled
+            XSLT. -->
+        <xsl:variable name="stage2-rtf">
+            <xsl:apply-templates select="document('boilerplate.xsl')" mode="include-boilerplate">
+                <xsl:with-param name="stage1" select="$stage1"/>
+            </xsl:apply-templates>
+        </xsl:variable>
+        <xsl:variable name="stage2" select="exsl:node-set($stage2-rtf)"/>
+
+        <!-- We're done, so generate some output -->
+        <xsl:copy-of select="$stage2"/>
+    </xsl:template>
+    <xsl:template match="node()|@*" mode="include-boilerplate">
+        <xsl:param name="stage1"/>
+        <xsl:choose>
+            <xsl:when test="@insert='insert'">
+                <xsl:apply-templates select="$stage1" mode="include-boilerplate">
+                    <xsl:with-param name="stage1" select="$stage1"/>
+                </xsl:apply-templates>
+            </xsl:when>
+            <xsl:otherwise>
+                <xsl:copy>
+                    <xsl:apply-templates select="node()|@*" mode="include-boilerplate">
+                        <xsl:with-param name="stage1" select="$stage1"/>
+                    </xsl:apply-templates>
+                </xsl:copy>
+            </xsl:otherwise>
+        </xsl:choose>
+
+    </xsl:template>
+    <!-- 
+        Annotate the rules for stage1
+    -->
+    <xsl:template match="dv:rules" mode="annotate-rules">
+        <xsl:param name="themehtml"/>
+        <xsl:copy>
+            <xsl:for-each select="*">
+                <xsl:copy>
+                    <xsl:attribute name="xml:id">r<xsl:value-of select="position()"/></xsl:attribute>
+                    <xsl:copy-of select="@*"/>
+                    <dv:matches>
+                        <xsl:variable name="themexpath" select="@theme"/>
+                        <xsl:for-each select="$themehtml">
+                            <xsl:for-each select="dyn:evaluate($themexpath)">
+                                <dv:xmlid>
+                                    <xsl:value-of select="@xml:id"/>
+                                </dv:xmlid>
+                            </xsl:for-each>
+                        </xsl:for-each>
+                    </dv:matches>
+                </xsl:copy>
+            </xsl:for-each>
+        </xsl:copy>
+    </xsl:template>
+    <xsl:template match="node()|@*" mode="annotate-html">
+        <xsl:copy>
+            <xsl:attribute name="xml:id">
+                <xsl:value-of select="generate-id()"/>
+            </xsl:attribute>
+            <xsl:apply-templates select="node()|@*" mode="annotate-html"/>
+        </xsl:copy>
+    </xsl:template>
+    <xsl:template match="node()|@*" mode="apply-rules">
+        <xsl:param name="rules"/>
+        <xsl:variable name="thisxmlid" select="@xml:id"/>
+        <xsl:variable name="matching-rule" select="$rules//*[dv:matches/dv:xmlid=$thisxmlid]"/>
+        <xsl:choose>
+            <xsl:when test="name($matching-rule)='copy'">
+                <!-- Copy the node and its attributes, but clear 
+                    the children and content.  Just have an 
+                    <xsl:copy-of> for the @content -->
+                <xsl:copy>
+                    <xsl:copy-of select="@*[name()!='xml:id']"/>
+                    <xsl:element name="xsl:copy-of">
+                        <xsl:attribute name="select">
+                            <xsl:value-of select="$matching-rule/@content"/>
+                        </xsl:attribute>
+                    </xsl:element>
+                </xsl:copy>
+            </xsl:when>
+            <xsl:when test="name($matching-rule)='replace'">
+                <!-- Toss out the theme node.  Simply 
+                <xsl:copy-of the @content. -->
+                <xsl:element name="xsl:copy-of">
+                    <xsl:attribute name="select">
+                        <xsl:value-of select="$matching-rule/@content"/>
+                    </xsl:attribute>
+                </xsl:element>
+            </xsl:when>
+            <xsl:when test="name($matching-rule)='append'">
+                <!-- Make the node and all its children, then 
+                    copy in the @content. -->
+                <xsl:copy>
+                    <xsl:apply-templates select="node()|@*" mode="apply-rules">
+                        <xsl:with-param name="rules" select="$rules"/>
+                    </xsl:apply-templates>
+                    <xsl:element name="xsl:copy-of">
+                        <xsl:attribute name="select">
+                            <xsl:value-of select="$matching-rule/@content"/>
+                        </xsl:attribute>
+                    </xsl:element>
+                </xsl:copy>
+            </xsl:when>
+            <xsl:otherwise>
+                <xsl:copy>
+                    <xsl:apply-templates select="node()|@*" mode="apply-rules">
+                        <xsl:with-param name="rules" select="$rules"/>
+                    </xsl:apply-templates>
+                </xsl:copy>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:template>
+    <xsl:template match="@xml:id" mode="apply-rules" priority="5">
+        <!-- Filter this out -->
+    </xsl:template>
+</xsl:stylesheet>

Added: z3/deliverance/sandboxes/paul/xdv/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/content.html	Mon Jul 28 23:01:55 2008
@@ -0,0 +1,10 @@
+<html>
+    <head>
+        <title>A Deeper Look At DVNG</title>
+        <meta name="dv.section" content="DVNG Articles"/>
+    </head>
+    <body>
+        <p>I have some content <em>in here</em> to merge.</p>
+        <p>Perhaps an image will be in here later.</p>
+    </body>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/rules.xml	Mon Jul 28 23:01:55 2008
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <replace theme="/html/head/title" content="/html/head/title"/>
+    <copy theme="//td/h1[@id='pageheading']" content="/html/head/title/text()"/>
+    <copy theme="/html/body/h1" content="/html/head/title/text()"/>
+    <replace theme="//td/div[@id='pagecontent']" content="/html/body/*"/>
+    <append theme="/html/head" content="/html/head/link|/html/head/script|/html/head/meta"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/theme.html	Mon Jul 28 23:01:55 2008
@@ -0,0 +1,19 @@
+<html>
+    <head>
+        <title>Second Theme Page Title</title>
+    </head>
+    <body>
+        <h1 class="somethemeclass">Theme Page Heading</h1>
+        <table border="0" height="300" cellpadding="20">
+            <tr>
+                <td width="200" bgcolor="gray" valign="top">
+                    <h2 align="center">Navigation</h2>
+                </td>
+                <td valign="top">
+                    <h1 id="pageheading">Theme Page Heading</h1>
+                    <div id="pagecontent">Theme content to replace.</div>
+                </td>
+            </tr>
+        </table>
+    </body>
+</html>

From paul at codespeak.net  Mon Jul 28 23:50:11 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Mon, 28 Jul 2008 23:50:11 +0200 (CEST)
Subject: [z3-checkins] r56825 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080728215011.6899C168561@codespeak.net>

Author: paul
Date: Mon Jul 28 23:50:09 2008
New Revision: 56825

Modified:
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
Log:
Parameterize the boilerplate

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Mon Jul 28 23:50:09 2008
@@ -4,7 +4,8 @@
     xmlns:dyn="http://exslt.org/dynamic" xmlns:xml="http://www.w3.org/XML/1998/namespace"
     exclude-result-prefixes="dv dyn exsl xml" version="1.0">
     <xsl:output indent="yes"/>
-    <xsl:param name="rulesuri">rules.xml</xsl:param>
+    <xsl:param name="rulesuri">tests/rules-01.xml</xsl:param>
+    <xsl:param name="boilerplateurl">boilerplate.xsl</xsl:param>
     <!-- Multi-stage theme compiler -->
     <xsl:template match="/">
         <!-- Put unique xml:id values on all the theme html -->
@@ -33,7 +34,7 @@
         <!-- Stage 2, include the boilerplate and make a compiled
             XSLT. -->
         <xsl:variable name="stage2-rtf">
-            <xsl:apply-templates select="document('boilerplate.xsl')" mode="include-boilerplate">
+            <xsl:apply-templates select="document($boilerplateurl)" mode="include-boilerplate">
                 <xsl:with-param name="stage1" select="$stage1"/>
             </xsl:apply-templates>
         </xsl:variable>

From paul at codespeak.net  Tue Jul 29 02:21:06 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Tue, 29 Jul 2008 02:21:06 +0200 (CEST)
Subject: [z3-checkins] r56827 - in z3/deliverance/sandboxes/paul/xdv: . tests
Message-ID: <20080729002106.6E4D4169E19@codespeak.net>

Author: paul
Date: Tue Jul 29 02:21:04 2008
New Revision: 56827

Added:
   z3/deliverance/sandboxes/paul/xdv/tests/
   z3/deliverance/sandboxes/paul/xdv/tests/content-00.html
   z3/deliverance/sandboxes/paul/xdv/tests/content-01.html
   z3/deliverance/sandboxes/paul/xdv/tests/rules-00.xml
   z3/deliverance/sandboxes/paul/xdv/tests/rules-01.xml
   z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
   z3/deliverance/sandboxes/paul/xdv/tests/theme-00.html
   z3/deliverance/sandboxes/paul/xdv/tests/theme-01.html
Modified:
   z3/deliverance/sandboxes/paul/xdv/README.txt
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
Log:
The Python stuff doesn't work just yet

Modified: z3/deliverance/sandboxes/paul/xdv/README.txt
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/README.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/README.txt	Tue Jul 29 02:21:04 2008
@@ -33,7 +33,6 @@
   xsltproc --stringparam rulesuri rules.xml compiler.xsl theme.html > compiledtheme.xsl
 
 
-
 To Do
 -------
 

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Tue Jul 29 02:21:04 2008
@@ -15,33 +15,7 @@
         <xsl:variable name="themehtml" select="exsl:node-set($themehtml-rtf)"/>
 
         <!-- Include the rules file, adding @xml:id attributes as it is included -->
-        <xsl:variable name="rules-rtf">
-            <xsl:apply-templates select="document($rulesuri)/*" mode="annotate-rules">
-                <xsl:with-param name="themehtml" select="$themehtml"/>
-            </xsl:apply-templates>
-        </xsl:variable>
-        <xsl:variable name="rules" select="exsl:node-set($rules-rtf)"/>
 
-        <!-- Make a pass through all the theme html, filtering 
-            as we go. -->
-        <xsl:variable name="stage1-rtf">
-            <xsl:apply-templates select="$themehtml" mode="apply-rules">
-                <xsl:with-param name="rules" select="$rules"/>
-            </xsl:apply-templates>
-        </xsl:variable>
-        <xsl:variable name="stage1" select="exsl:node-set($stage1-rtf)"/>
-
-        <!-- Stage 2, include the boilerplate and make a compiled
-            XSLT. -->
-        <xsl:variable name="stage2-rtf">
-            <xsl:apply-templates select="document($boilerplateurl)" mode="include-boilerplate">
-                <xsl:with-param name="stage1" select="$stage1"/>
-            </xsl:apply-templates>
-        </xsl:variable>
-        <xsl:variable name="stage2" select="exsl:node-set($stage2-rtf)"/>
-
-        <!-- We're done, so generate some output -->
-        <xsl:copy-of select="$stage2"/>
     </xsl:template>
     <xsl:template match="node()|@*" mode="include-boilerplate">
         <xsl:param name="stage1"/>

Added: z3/deliverance/sandboxes/paul/xdv/tests/content-00.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/content-00.html	Tue Jul 29 02:21:04 2008
@@ -0,0 +1,10 @@
+<html>
+    <head>
+        <title>A Deeper Look At DVNG</title>
+        <meta name="dv.section" content="DVNG Articles"/>
+    </head>
+    <body>
+        <p>I have some content <em>in here</em> to merge.</p>
+        <p>Perhaps an image will be in here later.</p>
+    </body>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/content-01.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/content-01.html	Tue Jul 29 02:21:04 2008
@@ -0,0 +1 @@
+<title>Content 01</title>

Added: z3/deliverance/sandboxes/paul/xdv/tests/rules-00.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/rules-00.xml	Tue Jul 29 02:21:04 2008
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <replace theme="/html/head/title" content="/html/head/title"/>
+    <copy theme="//td/h1[@id='pageheading']" content="/html/head/title/text()"/>
+    <copy theme="/html/body/h1" content="/html/head/title/text()"/>
+    <replace theme="//td/div[@id='pagecontent']" content="/html/body/*"/>
+    <append theme="/html/head" content="/html/head/link|/html/head/script|/html/head/meta"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/rules-01.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/rules-01.xml	Tue Jul 29 02:21:04 2008
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <replace theme="title" content="title"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	Tue Jul 29 02:21:04 2008
@@ -0,0 +1,32 @@
+
+from lxml import etree
+
+class XDV:
+
+    def __init__(self, testnumber):
+        testnumber = testnumber
+        themefn = "theme.html"
+        rulesfn = "rules.xml"
+        contentfn = "content.html"
+
+        themedoc = etree.ElementTree(file=themefn)
+        contentdoc = etree.ElementTree(file=contentfn)
+
+        compilerfn = "compiler.xsl"
+        compilerdoc = etree.ElementTree(file=compilerfn)
+        compiler = etree.XSLT(compilerdoc)
+
+        # Make a compiled version
+        params = {
+            'rulesuri': rulesfn,
+            'boilerplate': 'boilerplate.xsl',
+            }
+        ct = compiler(themedoc, **params)
+        print str(ct)
+
+def main():
+    xdv = XDV("01")
+
+
+if __name__ == "__main__":
+    main()

Added: z3/deliverance/sandboxes/paul/xdv/tests/theme-00.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/theme-00.html	Tue Jul 29 02:21:04 2008
@@ -0,0 +1,19 @@
+<html>
+    <head>
+        <title>Second Theme Page Title</title>
+    </head>
+    <body>
+        <h1 class="somethemeclass">Theme Page Heading</h1>
+        <table border="0" height="300" cellpadding="20">
+            <tr>
+                <td width="200" bgcolor="gray" valign="top">
+                    <h2 align="center">Navigation</h2>
+                </td>
+                <td valign="top">
+                    <h1 id="pageheading">Theme Page Heading</h1>
+                    <div id="pagecontent">Theme content to replace.</div>
+                </td>
+            </tr>
+        </table>
+    </body>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/theme-01.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/theme-01.html	Tue Jul 29 02:21:04 2008
@@ -0,0 +1,5 @@
+<div>
+    <head>
+        <title>Theme 01</title>
+    </head>
+</div>

From paul at codespeak.net  Tue Jul 29 02:30:05 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Tue, 29 Jul 2008 02:30:05 +0200 (CEST)
Subject: [z3-checkins] r56828 - in z3/deliverance/sandboxes/paul/xdv: . tests
Message-ID: <20080729003005.6FF75169E2A@codespeak.net>

Author: paul
Date: Tue Jul 29 02:30:04 2008
New Revision: 56828

Modified:
   z3/deliverance/sandboxes/paul/xdv/README.txt
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
   z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
Log:
The python module now runs.

Modified: z3/deliverance/sandboxes/paul/xdv/README.txt
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/README.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/README.txt	Tue Jul 29 02:30:04 2008
@@ -32,6 +32,14 @@
 
   xsltproc --stringparam rulesuri rules.xml compiler.xsl theme.html > compiledtheme.xsl
 
+Running from Python
+-----------------------
+
+There is a little bit of support for generating a theme and printing
+it to stdout using lxml:
+
+  $ cd xdv
+  $ python tests/test_nodes.py
 
 To Do
 -------

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Tue Jul 29 02:30:04 2008
@@ -15,7 +15,33 @@
         <xsl:variable name="themehtml" select="exsl:node-set($themehtml-rtf)"/>
 
         <!-- Include the rules file, adding @xml:id attributes as it is included -->
+        <xsl:variable name="rules-rtf">
+            <xsl:apply-templates select="document($rulesuri)/*" mode="annotate-rules">
+                <xsl:with-param name="themehtml" select="$themehtml"/>
+            </xsl:apply-templates>
+        </xsl:variable>
+        <xsl:variable name="rules" select="exsl:node-set($rules-rtf)"/>
 
+        <!-- Make a pass through all the theme html, filtering 
+            as we go. This is where rule generation happens. -->
+        <xsl:variable name="stage1-rtf">
+            <xsl:apply-templates select="$themehtml" mode="apply-rules">
+                <xsl:with-param name="rules" select="$rules"/>
+            </xsl:apply-templates>
+        </xsl:variable>
+        <xsl:variable name="stage1" select="exsl:node-set($stage1-rtf)"/>
+
+        <!-- Stage 2, include the boilerplate and make a compiled
+            XSLT. -->
+        <xsl:variable name="stage2-rtf">
+            <xsl:apply-templates select="document($boilerplateurl)" mode="include-boilerplate">
+                <xsl:with-param name="stage1" select="$stage1"/>
+            </xsl:apply-templates>
+        </xsl:variable>
+        <xsl:variable name="stage2" select="exsl:node-set($stage2-rtf)"/>
+
+        <!-- We're done, so generate some output -->
+        <xsl:copy-of select="$stage2"/>
     </xsl:template>
     <xsl:template match="node()|@*" mode="include-boilerplate">
         <xsl:param name="stage1"/>
@@ -108,6 +134,9 @@
                     </xsl:element>
                 </xsl:copy>
             </xsl:when>
+            <xsl:when test="name($matching-rule)='drop'">
+                <!-- Do nothing.  We want to get rid of this node. -->
+            </xsl:when>
             <xsl:otherwise>
                 <xsl:copy>
                     <xsl:apply-templates select="node()|@*" mode="apply-rules">

Modified: z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	Tue Jul 29 02:30:04 2008
@@ -5,9 +5,9 @@
 
     def __init__(self, testnumber):
         testnumber = testnumber
-        themefn = "theme.html"
-        rulesfn = "rules.xml"
-        contentfn = "content.html"
+        themefn = "tests/theme-%s.html"      % testnumber
+        rulesfn = "tests/rules-%s.xml"       % testnumber
+        contentfn = "tests/content-%s.html"  % testnumber
 
         themedoc = etree.ElementTree(file=themefn)
         contentdoc = etree.ElementTree(file=contentfn)
@@ -19,13 +19,14 @@
         # Make a compiled version
         params = {
             'rulesuri': rulesfn,
-            'boilerplate': 'boilerplate.xsl',
             }
         ct = compiler(themedoc, **params)
+        for msg in compiler.error_log:
+            print msg
         print str(ct)
 
 def main():
-    xdv = XDV("01")
+    xdv = XDV("00")
 
 
 if __name__ == "__main__":

From paul at codespeak.net  Wed Jul 30 13:31:35 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Wed, 30 Jul 2008 13:31:35 +0200 (CEST)
Subject: [z3-checkins] r56856 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080730113135.C803D169E54@codespeak.net>

Author: paul
Date: Wed Jul 30 13:31:34 2008
New Revision: 56856

Modified:
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
Log:
Fixed default rules file

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Wed Jul 30 13:31:34 2008
@@ -4,7 +4,7 @@
     xmlns:dyn="http://exslt.org/dynamic" xmlns:xml="http://www.w3.org/XML/1998/namespace"
     exclude-result-prefixes="dv dyn exsl xml" version="1.0">
     <xsl:output indent="yes"/>
-    <xsl:param name="rulesuri">tests/rules-01.xml</xsl:param>
+    <xsl:param name="rulesuri">rules.xml</xsl:param>
     <xsl:param name="boilerplateurl">boilerplate.xsl</xsl:param>
     <!-- Multi-stage theme compiler -->
     <xsl:template match="/">

From paul at codespeak.net  Wed Jul 30 13:36:36 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Wed, 30 Jul 2008 13:36:36 +0200 (CEST)
Subject: [z3-checkins] r56857 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080730113636.DB25C169E72@codespeak.net>

Author: paul
Date: Wed Jul 30 13:36:36 2008
New Revision: 56857

Modified:
   z3/deliverance/sandboxes/paul/xdv/content.html
   z3/deliverance/sandboxes/paul/xdv/rules.xml
Log:
Added an example matchin Laurence's <copy> test

Modified: z3/deliverance/sandboxes/paul/xdv/content.html
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/content.html	(original)
+++ z3/deliverance/sandboxes/paul/xdv/content.html	Wed Jul 30 13:36:36 2008
@@ -4,7 +4,9 @@
         <meta name="dv.section" content="DVNG Articles"/>
     </head>
     <body>
+      <div id="content">
         <p>I have some content <em>in here</em> to merge.</p>
         <p>Perhaps an image will be in here later.</p>
+      </div>
     </body>
 </html>

Modified: z3/deliverance/sandboxes/paul/xdv/rules.xml
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/rules.xml	(original)
+++ z3/deliverance/sandboxes/paul/xdv/rules.xml	Wed Jul 30 13:36:36 2008
@@ -3,6 +3,6 @@
     <replace theme="/html/head/title" content="/html/head/title"/>
     <copy theme="//td/h1[@id='pageheading']" content="/html/head/title/text()"/>
     <copy theme="/html/body/h1" content="/html/head/title/text()"/>
-    <replace theme="//td/div[@id='pagecontent']" content="/html/body/*"/>
+    <copy theme="//td/div[@id='pagecontent']" content="/html/body/div[@id='content']/*"/>
     <append theme="/html/head" content="/html/head/link|/html/head/script|/html/head/meta"/>
 </rules>

From paul at codespeak.net  Wed Jul 30 15:11:01 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Wed, 30 Jul 2008 15:11:01 +0200 (CEST)
Subject: [z3-checkins] r56865 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080730131101.65E05169E33@codespeak.net>

Author: paul
Date: Wed Jul 30 15:11:00 2008
New Revision: 56865

Modified:
   z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
   z3/deliverance/sandboxes/paul/xdv/content.html
Log:
Changed the boilerplate to have the initial-stage, apply-theme, and final-stage multipass.

Modified: z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	Wed Jul 30 15:11:00 2008
@@ -1,6 +1,36 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns:dv="http://openplans.org/deliverance" xmlns:exsl="http://exslt.org/common"
+    exclude-result-prefixes="exsl dv" version="1.0">
     <xsl:template match="/">
-        <x insert="insert"/>
+
+        <!-- Pass incoming content through initial-stage filter. -->
+        <xsl:variable name="initial-stage-rtf">
+            <xsl:apply-templates select="/" mode="initial-stage"/>
+        </xsl:variable>
+        <xsl:variable name="initial-stage" select="exsl:node-set($initial-stage-rtf)"/>
+
+        <!-- Now apply the theme to the initial-stage content -->
+        <xsl:variable name="themedcontent-rtf">
+            <xsl:apply-templates select="$initial-stage" mode="apply-theme"/>
+        </xsl:variable>
+        <xsl:variable name="content" select="exsl:node-set($themedcontent-rtf)"/>
+
+        <!-- We're done, so generate some output by passing 
+            through a final stage. -->
+        <xsl:apply-templates select="$content" mode="final-stage"/>
+    </xsl:template>
+    <xsl:template match="node()|@*" mode="initial-stage">
+        <xsl:copy>
+            <xsl:apply-templates select="node()|@*" mode="initial-stage"/>
+        </xsl:copy>
+    </xsl:template>
+    <xsl:template match="/" mode="apply-theme">
+        <dv:insert/>
+    </xsl:template>
+    <xsl:template match="node()|@*" mode="final-stage">
+        <xsl:copy>
+            <xsl:apply-templates select="node()|@*" mode="final-stage"/>
+        </xsl:copy>
     </xsl:template>
 </xsl:stylesheet>

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Wed Jul 30 15:11:00 2008
@@ -42,11 +42,17 @@
 
         <!-- We're done, so generate some output -->
         <xsl:copy-of select="$stage2"/>
+
     </xsl:template>
+    <!-- 
+        
+        All the utility rules
+
+    -->
     <xsl:template match="node()|@*" mode="include-boilerplate">
         <xsl:param name="stage1"/>
         <xsl:choose>
-            <xsl:when test="@insert='insert'">
+            <xsl:when test="name()='dv:insert'">
                 <xsl:apply-templates select="$stage1" mode="include-boilerplate">
                     <xsl:with-param name="stage1" select="$stage1"/>
                 </xsl:apply-templates>

Modified: z3/deliverance/sandboxes/paul/xdv/content.html
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/content.html	(original)
+++ z3/deliverance/sandboxes/paul/xdv/content.html	Wed Jul 30 15:11:00 2008
@@ -1,12 +1,14 @@
 <html>
-    <head>
-        <title>A Deeper Look At DVNG</title>
-        <meta name="dv.section" content="DVNG Articles"/>
-    </head>
-    <body>
-      <div id="content">
-        <p>I have some content <em>in here</em> to merge.</p>
-        <p>Perhaps an image will be in here later.</p>
+  <head>
+    <title>A Deeper Look At DVNG</title>
+    <meta name="dv.section" content="DVNG Articles"/>
+  </head>
+  <body>
+    <div id="content">
+      <div id="byline">To Drop, By <a href="foo">Some Person.</a>
       </div>
-    </body>
+      <p>I have some content <em>in here</em> to merge.</p>
+      <p>Perhaps an image will be in here later.</p>
+    </div>
+  </body>
 </html>

From paul at codespeak.net  Wed Jul 30 17:18:09 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Wed, 30 Jul 2008 17:18:09 +0200 (CEST)
Subject: [z3-checkins] r56866 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080730151809.A091C2A019E@codespeak.net>

Author: paul
Date: Wed Jul 30 17:18:08 2008
New Revision: 56866

Modified:
   z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
   z3/deliverance/sandboxes/paul/xdv/rules.xml
Log:
Added support for <drop content="Something"/>.

Modified: z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	Wed Jul 30 17:18:08 2008
@@ -19,6 +19,16 @@
         <!-- We're done, so generate some output by passing 
             through a final stage. -->
         <xsl:apply-templates select="$content" mode="final-stage"/>
+
+    </xsl:template>
+
+    <!-- 
+    
+        Utility templates
+    -->
+    <xsl:template match="none" mode="insert-drop-rules">
+        <!-- The compiler looks for this and replaces 
+        the @match and @mode. -->
     </xsl:template>
     <xsl:template match="node()|@*" mode="initial-stage">
         <xsl:copy>

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Wed Jul 30 17:18:08 2008
@@ -36,6 +36,7 @@
         <xsl:variable name="stage2-rtf">
             <xsl:apply-templates select="document($boilerplateurl)" mode="include-boilerplate">
                 <xsl:with-param name="stage1" select="$stage1"/>
+                <xsl:with-param name="rules" select="$rules"/>
             </xsl:apply-templates>
         </xsl:variable>
         <xsl:variable name="stage2" select="exsl:node-set($stage2-rtf)"/>
@@ -51,8 +52,21 @@
     -->
     <xsl:template match="node()|@*" mode="include-boilerplate">
         <xsl:param name="stage1"/>
+        <xsl:param name="rules"/>
         <xsl:choose>
+            <xsl:when test="@mode='insert-drop-rules'">
+                <!-- If there are any <drop @content> rules, put it in 
+                here. -->
+                <xsl:for-each select="$rules/dv:rules/dv:drop[@content]">
+                    <xsl:element name="xsl:template">
+                        <xsl:attribute name="match"><xsl:value-of select="@content"/></xsl:attribute>
+                        <xsl:attribute name="mode">initial-stage</xsl:attribute>
+                        <xsl:comment>Do nothing, skip these nodes</xsl:comment>
+                    </xsl:element>
+                </xsl:for-each>
+            </xsl:when>
             <xsl:when test="name()='dv:insert'">
+                <!-- Put the compiled theme in at this spot of the boilerplate -->
                 <xsl:apply-templates select="$stage1" mode="include-boilerplate">
                     <xsl:with-param name="stage1" select="$stage1"/>
                 </xsl:apply-templates>
@@ -61,6 +75,7 @@
                 <xsl:copy>
                     <xsl:apply-templates select="node()|@*" mode="include-boilerplate">
                         <xsl:with-param name="stage1" select="$stage1"/>
+                        <xsl:with-param name="rules" select="$rules"/>
                     </xsl:apply-templates>
                 </xsl:copy>
             </xsl:otherwise>

Modified: z3/deliverance/sandboxes/paul/xdv/rules.xml
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/rules.xml	(original)
+++ z3/deliverance/sandboxes/paul/xdv/rules.xml	Wed Jul 30 17:18:08 2008
@@ -1,5 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <rules xmlns="http://openplans.org/deliverance">
+    <drop content="//div[@id='byline']"/>
+    <drop content="//div[@id='nomatchever']"/>
     <replace theme="/html/head/title" content="/html/head/title"/>
     <copy theme="//td/h1[@id='pageheading']" content="/html/head/title/text()"/>
     <copy theme="/html/body/h1" content="/html/head/title/text()"/>

From ldr at codespeak.net  Thu Jul 31 11:36:25 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Thu, 31 Jul 2008 11:36:25 +0200 (CEST)
Subject: [z3-checkins] r56871 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080731093625.C2D1F168490@codespeak.net>

Author: ldr
Date: Thu Jul 31 11:36:24 2008
New Revision: 56871

Modified:
   z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
Log:
ouput method html

Modified: z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	Thu Jul 31 11:36:24 2008
@@ -2,6 +2,8 @@
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
     xmlns:dv="http://openplans.org/deliverance" xmlns:exsl="http://exslt.org/common"
     exclude-result-prefixes="exsl dv" version="1.0">
+    <xsl:output method="html"/>
+
     <xsl:template match="/">
 
         <!-- Pass incoming content through initial-stage filter. -->

From paul at codespeak.net  Thu Jul 31 15:48:24 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Thu, 31 Jul 2008 15:48:24 +0200 (CEST)
Subject: [z3-checkins] r56877 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080731134824.9FAD6168533@codespeak.net>

Author: paul
Date: Thu Jul 31 15:48:22 2008
New Revision: 56877

Added:
   z3/deliverance/sandboxes/paul/xdv/CHANGES.txt
Modified:
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
   z3/deliverance/sandboxes/paul/xdv/content.html
   z3/deliverance/sandboxes/paul/xdv/rules.xml
   z3/deliverance/sandboxes/paul/xdv/theme.html
Log:
Ugly implementation for handling both <prepend> and <append> on the same theme node, but good enough for now.

Added: z3/deliverance/sandboxes/paul/xdv/CHANGES.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/CHANGES.txt	Thu Jul 31 15:48:22 2008
@@ -0,0 +1,8 @@
+
+July 31 
+------------------
+
+- Add support for <drop theme=""/>
+
+- Add support for <append> and change example rules.xml to demonstrate
+  it

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Thu Jul 31 15:48:22 2008
@@ -59,7 +59,9 @@
                 here. -->
                 <xsl:for-each select="$rules/dv:rules/dv:drop[@content]">
                     <xsl:element name="xsl:template">
-                        <xsl:attribute name="match"><xsl:value-of select="@content"/></xsl:attribute>
+                        <xsl:attribute name="match">
+                            <xsl:value-of select="@content"/>
+                        </xsl:attribute>
                         <xsl:attribute name="mode">initial-stage</xsl:attribute>
                         <xsl:comment>Do nothing, skip these nodes</xsl:comment>
                     </xsl:element>
@@ -119,6 +121,37 @@
         <xsl:variable name="thisxmlid" select="@xml:id"/>
         <xsl:variable name="matching-rule" select="$rules//*[dv:matches/dv:xmlid=$thisxmlid]"/>
         <xsl:choose>
+            <xsl:when test="count($matching-rule) > 1">
+                <!-- Handle the case where an html node in the theme is the 
+                target of more than one @theme XPath in a rule.  Most likely, 
+                this is a mixture of <prepend> and <append>.  Ignore, for now, 
+                any other cases. Algorithm: make the node (e.g. <head>), copy 
+                in its attributes, insert all the <prepend> content, insert 
+                the theme subelements, insert the <append> content, then 
+                close the node. -->
+                <xsl:copy>
+                    <xsl:apply-templates select="@*" mode="apply-rules">
+                        <xsl:with-param name="rules" select="$rules"/>
+                    </xsl:apply-templates>
+                    <xsl:for-each select="$matching-rule[name()='prepend']">
+                        <xsl:element name="xsl:copy-of">
+                            <xsl:attribute name="select">
+                                <xsl:value-of select="@content"/>
+                            </xsl:attribute>
+                        </xsl:element>
+                    </xsl:for-each>
+                    <xsl:apply-templates select="node()" mode="apply-rules">
+                        <xsl:with-param name="rules" select="$rules"/>
+                    </xsl:apply-templates>
+                    <xsl:for-each select="$matching-rule[name()='append']">
+                        <xsl:element name="xsl:copy-of">
+                            <xsl:attribute name="select">
+                                <xsl:value-of select="@content"/>
+                            </xsl:attribute>
+                        </xsl:element>
+                    </xsl:for-each>
+                </xsl:copy>
+            </xsl:when>
             <xsl:when test="name($matching-rule)='copy'">
                 <!-- Copy the node and its attributes, but clear 
                     the children and content.  Just have an 
@@ -143,7 +176,7 @@
             </xsl:when>
             <xsl:when test="name($matching-rule)='append'">
                 <!-- Make the node and all its children, then 
-                    copy in the @content. -->
+                copy in the @content. -->
                 <xsl:copy>
                     <xsl:apply-templates select="node()|@*" mode="apply-rules">
                         <xsl:with-param name="rules" select="$rules"/>
@@ -155,8 +188,27 @@
                     </xsl:element>
                 </xsl:copy>
             </xsl:when>
+            <xsl:when test="name($matching-rule)='prepend'">
+                <!-- Make the node from the theme and the attributes, then 
+                    copy in all the nodes from the content, then add the 
+                    other child nodes for the theme. -->
+                <xsl:copy>
+                    <xsl:apply-templates select="@*" mode="apply-rules">
+                        <xsl:with-param name="rules" select="$rules"/>
+                    </xsl:apply-templates>
+                    <xsl:element name="xsl:copy-of">
+                        <xsl:attribute name="select">
+                            <xsl:value-of select="$matching-rule/@content"/>
+                        </xsl:attribute>
+                    </xsl:element>
+                    <xsl:apply-templates select="node()" mode="apply-rules">
+                        <xsl:with-param name="rules" select="$rules"/>
+                    </xsl:apply-templates>
+                </xsl:copy>
+            </xsl:when>
             <xsl:when test="name($matching-rule)='drop'">
-                <!-- Do nothing.  We want to get rid of this node. -->
+                <!-- Do nothing.  We want to get rid of this node 
+                    in the theme. -->
             </xsl:when>
             <xsl:otherwise>
                 <xsl:copy>

Modified: z3/deliverance/sandboxes/paul/xdv/content.html
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/content.html	(original)
+++ z3/deliverance/sandboxes/paul/xdv/content.html	Thu Jul 31 15:48:22 2008
@@ -2,6 +2,8 @@
   <head>
     <title>A Deeper Look At DVNG</title>
     <meta name="dv.section" content="DVNG Articles"/>
+    <script type="text/javascript">
+            /* */</script>
   </head>
   <body>
     <div id="content">

Modified: z3/deliverance/sandboxes/paul/xdv/rules.xml
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/rules.xml	(original)
+++ z3/deliverance/sandboxes/paul/xdv/rules.xml	Thu Jul 31 15:48:22 2008
@@ -6,5 +6,6 @@
     <copy theme="//td/h1[@id='pageheading']" content="/html/head/title/text()"/>
     <copy theme="/html/body/h1" content="/html/head/title/text()"/>
     <copy theme="//td/div[@id='pagecontent']" content="/html/body/div[@id='content']/*"/>
-    <append theme="/html/head" content="/html/head/link|/html/head/script|/html/head/meta"/>
+    <append theme="/html/head" content="/html/head/link|/html/head/meta"/>
+    <prepend theme="/html/head" content="/html/head/script"/>
 </rules>

Modified: z3/deliverance/sandboxes/paul/xdv/theme.html
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/theme.html	(original)
+++ z3/deliverance/sandboxes/paul/xdv/theme.html	Thu Jul 31 15:48:22 2008
@@ -1,6 +1,7 @@
 <html>
-    <head>
+    <head hoobly="boobly">
         <title>Second Theme Page Title</title>
+        <link rel="stylesheet" href="foo.css"/>
     </head>
     <body>
         <h1 class="somethemeclass">Theme Page Heading</h1>

From ldr at codespeak.net  Tue Aug  5 19:37:05 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Tue,  5 Aug 2008 19:37:05 +0200 (CEST)
Subject: [z3-checkins] r57010 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080805173705.90E63169FAE@codespeak.net>

Author: ldr
Date: Tue Aug  5 19:37:03 2008
New Revision: 57010

Modified:
   z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
Log:
Set default doctype as XHTML transitional

Modified: z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	Tue Aug  5 19:37:03 2008
@@ -1,8 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
     xmlns:dv="http://openplans.org/deliverance" xmlns:exsl="http://exslt.org/common"
     exclude-result-prefixes="exsl dv" version="1.0">
-    <xsl:output method="html"/>
+    <xsl:output method="html" omit-xml-declaration="yes" doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN" doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd" indent="yes"/>
 
     <xsl:template match="/">
 

From ldr at codespeak.net  Tue Aug  5 19:45:12 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Tue,  5 Aug 2008 19:45:12 +0200 (CEST)
Subject: [z3-checkins] r57012 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080805174512.A4D9B169F94@codespeak.net>

Author: ldr
Date: Tue Aug  5 19:45:12 2008
New Revision: 57012

Modified:
   z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
Log:
Whoops, cleanup the boilerplate

Modified: z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	Tue Aug  5 19:45:12 2008
@@ -1,9 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
     xmlns:dv="http://openplans.org/deliverance" xmlns:exsl="http://exslt.org/common"
     exclude-result-prefixes="exsl dv" version="1.0">
-    <xsl:output method="html" omit-xml-declaration="yes" doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN" doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd" indent="yes"/>
+  <xsl:output method="html" indent="yes" omit-xml-declaration="yes"
+      doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN"
+      doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"/>
 
     <xsl:template match="/">
 

From ldr at codespeak.net  Wed Aug  6 20:52:32 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Wed,  6 Aug 2008 20:52:32 +0200 (CEST)
Subject: [z3-checkins] r57042 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080806185232.500D1168532@codespeak.net>

Author: ldr
Date: Wed Aug  6 20:52:31 2008
New Revision: 57042

Modified:
   z3/deliverance/sandboxes/paul/xdv/CHANGES.txt
   z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
Log:
set output media-type

Modified: z3/deliverance/sandboxes/paul/xdv/CHANGES.txt
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/CHANGES.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/CHANGES.txt	Wed Aug  6 20:52:31 2008
@@ -1,3 +1,9 @@
+August 6
+------------------
+
+- Set media-type on transforms. Compiler outputs as text/xml and generated
+  transforms output in text/html, but run in xml mode.
+  (Laurence)
 
 July 31 
 ------------------

Modified: z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	Wed Aug  6 20:52:31 2008
@@ -2,7 +2,7 @@
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
     xmlns:dv="http://openplans.org/deliverance" xmlns:exsl="http://exslt.org/common"
     exclude-result-prefixes="exsl dv" version="1.0">
-  <xsl:output method="html" indent="yes" omit-xml-declaration="yes"
+  <xsl:output indent="yes" omit-xml-declaration="yes" media-type="text/html"
       doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN"
       doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"/>
 

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Wed Aug  6 20:52:31 2008
@@ -3,7 +3,7 @@
     xmlns:dv="http://openplans.org/deliverance" xmlns:exsl="http://exslt.org/common"
     xmlns:dyn="http://exslt.org/dynamic" xmlns:xml="http://www.w3.org/XML/1998/namespace"
     exclude-result-prefixes="dv dyn exsl xml" version="1.0">
-    <xsl:output indent="yes"/>
+    <xsl:output indent="yes" media-type="text/xml"/>
     <xsl:param name="rulesuri">rules.xml</xsl:param>
     <xsl:param name="boilerplateurl">boilerplate.xsl</xsl:param>
     <!-- Multi-stage theme compiler -->

From faassen at codespeak.net  Thu Aug  7 18:58:18 2008
From: faassen at codespeak.net (faassen at codespeak.net)
Date: Thu,  7 Aug 2008 18:58:18 +0200 (CEST)
Subject: [z3-checkins] r57069 - in z3/hurry.file/trunk: .
	src/hurry/file/browser
Message-ID: <20080807165818.B4758169EBD@codespeak.net>

Author: faassen
Date: Thu Aug  7 18:58:15 2008
New Revision: 57069

Added:
   z3/hurry.file/trunk/bootstrap.py
   z3/hurry.file/trunk/buildout.cfg
Removed:
   z3/hurry.file/trunk/INSTALL.txt
   z3/hurry.file/trunk/hurry.file-configure.zcml
Modified:
   z3/hurry.file/trunk/   (props changed)
   z3/hurry.file/trunk/CHANGES.txt
   z3/hurry.file/trunk/CREDITS.txt
   z3/hurry.file/trunk/setup.py
   z3/hurry.file/trunk/src/hurry/file/browser/widget.py
Log:
Lots of packaging touch-ups, so that tests can be run using buildout
generated test runner.


Modified: z3/hurry.file/trunk/CHANGES.txt
==============================================================================
--- z3/hurry.file/trunk/CHANGES.txt	(original)
+++ z3/hurry.file/trunk/CHANGES.txt	Thu Aug  7 18:58:15 2008
@@ -1,6 +1,18 @@
 hurry.file changes
 ==================
 
+1.1 (unreleased)
+----------------
+
+* Add in a buildout.cfg that installs the test runner.
+
+* List dependencies in setup.py.
+
+* Rely on zope.session instead of zope.app.session to stop deprecation
+  warnings.
+
+* Add long description in setup.py based on README.txt and CHANGES.txt.
+
 1.0 (2006-10-25)
 ----------------
 

Modified: z3/hurry.file/trunk/CREDITS.txt
==============================================================================
--- z3/hurry.file/trunk/CREDITS.txt	(original)
+++ z3/hurry.file/trunk/CREDITS.txt	Thu Aug  7 18:58:15 2008
@@ -1,9 +1,11 @@
 Credits
 -------
 
-Martijn Faassen - initial and main developer
-Daniel Nouri - bugfixes
-Jan-Wijbrand Kolman - suggestions and feedback
+* Martijn Faassen - initial and main developer
+
+* Daniel Nouri - bugfixes
+
+* Jan-Wijbrand Kolman - suggestions and feedback
 
 The hurry.file library for Zope 3 was originally developed at Infrae
 (http://www.infrae.com).

Deleted: /z3/hurry.file/trunk/INSTALL.txt
==============================================================================
--- /z3/hurry.file/trunk/INSTALL.txt	Thu Aug  7 18:58:15 2008
+++ (empty file)
@@ -1,8 +0,0 @@
-Installation
-------------
-
-hurry.file requires Zope 3.3.
-
-Make sure hurry.file's src directory is on the Python path somehow,
-and then copy hurry.file-configure.zcml into your Zope's
-`etc/package-includes` directory.

Added: z3/hurry.file/trunk/bootstrap.py
==============================================================================
--- (empty file)
+++ z3/hurry.file/trunk/bootstrap.py	Thu Aug  7 18:58:15 2008
@@ -0,0 +1,55 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+
+$Id: bootstrap.py 75593 2007-05-06 21:11:27Z jim $
+"""
+
+import os, shutil, sys, tempfile, urllib2
+
+tmpeggs = tempfile.mkdtemp()
+
+try:
+    import pkg_resources
+except ImportError:
+    ez = {}
+    exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
+                         ).read() in ez
+    ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
+
+    import pkg_resources
+
+cmd = 'from setuptools.command.easy_install import main; main()'
+if sys.platform == 'win32':
+    cmd = '"%s"' % cmd # work around spawn lamosity on windows
+
+ws = pkg_resources.working_set
+assert os.spawnle(
+    os.P_WAIT, sys.executable, sys.executable,
+    '-c', cmd, '-mqNxd', tmpeggs, 'zc.buildout',
+    dict(os.environ,
+         PYTHONPATH=
+         ws.find(pkg_resources.Requirement.parse('setuptools')).location
+         ),
+    ) == 0
+
+ws.add_entry(tmpeggs)
+ws.require('zc.buildout')
+import zc.buildout.buildout
+zc.buildout.buildout.main(sys.argv[1:] + ['bootstrap'])
+shutil.rmtree(tmpeggs)

Added: z3/hurry.file/trunk/buildout.cfg
==============================================================================
--- (empty file)
+++ z3/hurry.file/trunk/buildout.cfg	Thu Aug  7 18:58:15 2008
@@ -0,0 +1,9 @@
+[buildout]
+develop = . 
+parts = test
+newest = false
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = hurry.file
+defaults = ['--tests-pattern', '^f?tests$', '-v']

Deleted: /z3/hurry.file/trunk/hurry.file-configure.zcml
==============================================================================
--- /z3/hurry.file/trunk/hurry.file-configure.zcml	Thu Aug  7 18:58:15 2008
+++ (empty file)
@@ -1,3 +0,0 @@
-<!-- install this into your Zope 3 instance's etc/package-includes 
-     directory -->
-<include package="hurry.file" />

Modified: z3/hurry.file/trunk/setup.py
==============================================================================
--- z3/hurry.file/trunk/setup.py	(original)
+++ z3/hurry.file/trunk/setup.py	Thu Aug  7 18:58:15 2008
@@ -1,28 +1,51 @@
 from setuptools import setup, find_packages
+import os
+
+def read(*rnames):
+    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+
+long_description = (
+    read('src/hurry/file/README.txt')
+    + '\n' +
+    read('CHANGES.txt')
+    )
 
 setup(
     name="hurry.file",
-    version="1.0",
-    packages=find_packages('src'),
-    
-    package_dir= {'':'src'},
-    
-    namespace_packages=['hurry'],
-    package_data = {
-    '': ['*.txt', '*.zcml'],
-    },
-
-    zip_safe=False,
-    author='Infrae',
-    author_email='faassen at infrae.com',
+    version="1.1dev",
     description="""\
 hurry.file is an advanced Zope 3 file widget which tries its best to behave
 like other widgets, even when the form is redisplayed due to a validation
 error. It also has built-in support for fast Apache-based file uploads
 and downloads through Tramline.
 """,
+    long_description=long_description,
+    classifiers=[
+        "Programming Language :: Python",
+        "Framework :: Zope3",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        ],
+    keywords='zope zope3',
+    author='Martijn Faassen, Infrae',
+    author_email='faassen at startifact.com',
+    url='',
     license='BSD',
-    keywords="zope zope3",
-    classifiers = ['Framework :: Zope3'],
-    install_requires=['setuptools'],
+    packages=find_packages('src'),
+    package_dir= {'':'src'},
+    namespace_packages=['hurry'],
+    include_package_data=True,
+    zip_safe=False,
+    install_requires=[
+       'setuptools',
+       'ZODB3',
+       'zope.interface',
+       'zope.schema',
+       'zope.component',
+       'zope.testing',
+       'zope.publisher',
+       'zope.app.form',
+       'zope.session',
+       'zope.app.testing',
+       'zope.app.container',
+       ],
     )

Modified: z3/hurry.file/trunk/src/hurry/file/browser/widget.py
==============================================================================
--- z3/hurry.file/trunk/src/hurry/file/browser/widget.py	(original)
+++ z3/hurry.file/trunk/src/hurry/file/browser/widget.py	Thu Aug  7 18:58:15 2008
@@ -9,7 +9,7 @@
 from zope.app.form.browser.widget import renderElement
 from zope.app.form.interfaces import ConversionError
 from zope.app.form.browser.textwidgets import escape
-from zope.app.session.interfaces import ISession
+from zope.session.interfaces import ISession
 from zope.app.form.browser import TextWidget
 
 from hurry.file.file import HurryFile

From faassen at codespeak.net  Thu Aug  7 19:03:31 2008
From: faassen at codespeak.net (faassen at codespeak.net)
Date: Thu,  7 Aug 2008 19:03:31 +0200 (CEST)
Subject: [z3-checkins] r57073 - in z3/hurry.file/trunk: . src/hurry/file
	src/hurry/file/browser
Message-ID: <20080807170331.183C4169ECD@codespeak.net>

Author: faassen
Date: Thu Aug  7 19:03:27 2008
New Revision: 57073

Modified:
   z3/hurry.file/trunk/CHANGES.txt
   z3/hurry.file/trunk/setup.py
   z3/hurry.file/trunk/src/hurry/file/README.txt
   z3/hurry.file/trunk/src/hurry/file/browser/file.txt
Log:
Fix up the generation of the long description.


Modified: z3/hurry.file/trunk/CHANGES.txt
==============================================================================
--- z3/hurry.file/trunk/CHANGES.txt	(original)
+++ z3/hurry.file/trunk/CHANGES.txt	Thu Aug  7 19:03:27 2008
@@ -1,7 +1,7 @@
 hurry.file changes
 ==================
 
-1.1 (unreleased)
+1.1 (2008-08-07)
 ----------------
 
 * Add in a buildout.cfg that installs the test runner.
@@ -11,7 +11,8 @@
 * Rely on zope.session instead of zope.app.session to stop deprecation
   warnings.
 
-* Add long description in setup.py based on README.txt and CHANGES.txt.
+* Add long description in setup.py based on README.txt and file.txt
+  doctests, and CHANGES.txt.
 
 1.0 (2006-10-25)
 ----------------
@@ -41,7 +42,7 @@
   which broke hurry.file. This includes a workaround.
 
 0.9.1 (2006-09-22)
------------------
+------------------
 
 * first cheeseshop release.
 

Modified: z3/hurry.file/trunk/setup.py
==============================================================================
--- z3/hurry.file/trunk/setup.py	(original)
+++ z3/hurry.file/trunk/setup.py	Thu Aug  7 19:03:27 2008
@@ -7,6 +7,8 @@
 long_description = (
     read('src/hurry/file/README.txt')
     + '\n' +
+    read('src/hurry/file/browser/file.txt')
+    + '\n' + 
     read('CHANGES.txt')
     )
 

Modified: z3/hurry.file/trunk/src/hurry/file/README.txt
==============================================================================
--- z3/hurry.file/trunk/src/hurry/file/README.txt	(original)
+++ z3/hurry.file/trunk/src/hurry/file/README.txt	Thu Aug  7 19:03:27 2008
@@ -1,5 +1,5 @@
-This document is about file storage. For information about the actual
-widgets, see also browser/file.txt
+hurry.file fields
+=================
 
 The file widget is built on top of the HurryFile object::
 

Modified: z3/hurry.file/trunk/src/hurry/file/browser/file.txt
==============================================================================
--- z3/hurry.file/trunk/src/hurry/file/browser/file.txt	(original)
+++ z3/hurry.file/trunk/src/hurry/file/browser/file.txt	Thu Aug  7 19:03:27 2008
@@ -1,8 +1,11 @@
-Hurry files
-===========
+hurry.file widgets
+==================
 
 This is an infrastructure to create a file widget that behaves as much
-as possible like a normal text widget.
+as possible like a normal text widget in formlib. Normally a file
+widget loses its file data when a form is re-presented for reasons of
+failing form validation. A ``hurry.file`` widget retains the file, for
+example by storing it in a session.
 
 In order to do this, we have a special way to store file data along with
 its filename::

From faassen at codespeak.net  Thu Aug  7 19:04:04 2008
From: faassen at codespeak.net (faassen at codespeak.net)
Date: Thu,  7 Aug 2008 19:04:04 +0200 (CEST)
Subject: [z3-checkins] r57074 - z3/hurry.file/tag/1.1
Message-ID: <20080807170404.33B0E169ED7@codespeak.net>

Author: faassen
Date: Thu Aug  7 19:03:59 2008
New Revision: 57074

Added:
   z3/hurry.file/tag/1.1/
      - copied from r57073, z3/hurry.file/trunk/
Log:
Create 1.1 release tag.


From faassen at codespeak.net  Thu Aug  7 19:04:38 2008
From: faassen at codespeak.net (faassen at codespeak.net)
Date: Thu,  7 Aug 2008 19:04:38 +0200 (CEST)
Subject: [z3-checkins] r57075 - z3/hurry.file/tag/1.1
Message-ID: <20080807170438.9C9E0169EE8@codespeak.net>

Author: faassen
Date: Thu Aug  7 19:04:37 2008
New Revision: 57075

Modified:
   z3/hurry.file/tag/1.1/setup.py
Log:
Release preparations.


Modified: z3/hurry.file/tag/1.1/setup.py
==============================================================================
--- z3/hurry.file/tag/1.1/setup.py	(original)
+++ z3/hurry.file/tag/1.1/setup.py	Thu Aug  7 19:04:37 2008
@@ -14,7 +14,7 @@
 
 setup(
     name="hurry.file",
-    version="1.1dev",
+    version="1.1",
     description="""\
 hurry.file is an advanced Zope 3 file widget which tries its best to behave
 like other widgets, even when the form is redisplayed due to a validation

From faassen at codespeak.net  Thu Aug  7 19:05:26 2008
From: faassen at codespeak.net (faassen at codespeak.net)
Date: Thu,  7 Aug 2008 19:05:26 +0200 (CEST)
Subject: [z3-checkins] r57076 - z3/hurry.file/trunk
Message-ID: <20080807170526.AF8BD169EE9@codespeak.net>

Author: faassen
Date: Thu Aug  7 19:05:22 2008
New Revision: 57076

Modified:
   z3/hurry.file/trunk/CHANGES.txt
   z3/hurry.file/trunk/setup.py
Log:
Post-release fixup.


Modified: z3/hurry.file/trunk/CHANGES.txt
==============================================================================
--- z3/hurry.file/trunk/CHANGES.txt	(original)
+++ z3/hurry.file/trunk/CHANGES.txt	Thu Aug  7 19:05:22 2008
@@ -1,6 +1,11 @@
 hurry.file changes
 ==================
 
+1.2 (unreleased)
+----------------
+
+* ...
+
 1.1 (2008-08-07)
 ----------------
 

Modified: z3/hurry.file/trunk/setup.py
==============================================================================
--- z3/hurry.file/trunk/setup.py	(original)
+++ z3/hurry.file/trunk/setup.py	Thu Aug  7 19:05:22 2008
@@ -14,7 +14,7 @@
 
 setup(
     name="hurry.file",
-    version="1.1dev",
+    version="1.2dev",
     description="""\
 hurry.file is an advanced Zope 3 file widget which tries its best to behave
 like other widgets, even when the form is redisplayed due to a validation

From gintas at codespeak.net  Mon Aug 11 02:23:12 2008
From: gintas at codespeak.net (gintas at codespeak.net)
Date: Mon, 11 Aug 2008 02:23:12 +0200 (CEST)
Subject: [z3-checkins] r57186 - z3/z3reload
Message-ID: <20080811002312.C0A1216A2F6@codespeak.net>

Author: gintas
Date: Mon Aug 11 02:23:11 2008
New Revision: 57186

Removed:
   z3/z3reload/
Log:
z3reload has been moved to svn.zope.org.


From ianb at codespeak.net  Mon Aug 11 21:17:27 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Mon, 11 Aug 2008 21:17:27 +0200 (CEST)
Subject: [z3-checkins] r57198 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/docs
Message-ID: <20080811191727.7941D16A1D0@codespeak.net>

Author: ianb
Date: Mon Aug 11 21:17:25 2008
New Revision: 57198

Added:
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/philosophy.txt   (contents, props changed)
Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt
Log:
Added a philosophy document

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt	Mon Aug 11 21:17:25 2008
@@ -5,6 +5,7 @@
 
 .. toctree::
 
+   philosophy
    configuration
    pyref
    news

Added: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/philosophy.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/philosophy.txt	Mon Aug 11 21:17:25 2008
@@ -0,0 +1,139 @@
+The Philosophy Of Deliverance
+=============================
+
+:author: Ian Bicking <ianb at openplans.org>
+
+Why is Deliverance?  Why was it made, what purpose does it serve, why should you use it, how can it change the way you do web development?
+
+On the Subject of Platforms
+---------------------------
+
+Right now we live in an age of *platforms*.  Developers (or management or coincidence) decides on a platform, and that serves as the basis for all future development.  Usually there's some old things from a previous platform (or a primordial pre-platform age: I'm looking at you ``formmail.pl``!)  The goal is always to eliminate all of these old pieces, rewriting them for the new platform.  That goal is seldom attained in a timely manner, and even before it is accomplished you may be moving to the next platform.
+
+Why do you have to port everything forward to the newest platform?  Well, presumably it is better engineered.  The newest platform is presumably what people are most familiar with.  But if those were the only reasons it would be hard to justify a rewrite of working software.  Often the real push comes because your systems don't work together.  It's hard to keep templates in sync across all the platforms.  Multiple logins may be required.  Navigation is inconsistent and incomplete.  Functionality that cross-cuts pages -- comments, login status, shopping cart status, etc -- isn't universally available.
+
+A similar conflict arises when you consider how to add new functionality to a site.  For example, you may want to add a blog.  Do you:
+
+a. Use the best blogging software available?  
+b. Use something native to your platform?  
+c. Write something yourself?
+
+The answer is probably (b) or (c), because it would be too hard to integrate something foreign to your platform.  This form of choice means that every platform has some kind of "blog", but the users of that blog are likely to only be a subset of the users of the parent platform.  This makes it difficult for winners to emerge, or for a well-developed piece of software to really be successful.  Platform-based software is limited by the adoption of the platform.
+
+Not all software has a platform.  These tend to be the most successful web applications, things like `Trac <http://trac.edgewall.org/>`_, `WordPress <http://wordpress.org>`_, etc.
+
+"Aha!" you think "I'll just use those best-of-breed applications!"  But no!  Those applications themselves turn into platforms.  WordPress is practically a CMS.  Trac too.  Extensible applications, if successful, become their own platform.  This is not to place blame, they aren't necessarily any worse than any other platform, just an acknowledgment that this move to platform can happen anywhere.
+
+Beyond Platforms, or A Better Platform
+--------------------------------------
+
+One of the major goals of Deliverance is to move beyond platforms.  It is an *integration tool*, to allow applications from different frameworks or languages to be integrated gracefully.
+
+There are only a few core reasons that people use platforms:
+
+1. A common look-and-feel across the site.
+2. Cohesive navigation.
+3. Indexing of the entire site.
+4. Shared authentication and user accounts.
+5. Cross-cutting functionality (e.g., commenting).
+
+Deliverance specifically addresses 1, providing a common look-and-feel across a site.  It can provide some help with 2, by allowing navigation to be more centrally managed, without relying purely on per-application navigation (though per-application navigation is still essential to navigating the individual applications).  3, 4, and 5 are not addressed by Deliverance (at least not yet).
+
+Deliverance applies a common theme across all the applications in your site.  It's basic unit of abstraction is **HTML**.  It doesn't use a particular templating language.  It doesn't know what an object is.  HTML is something every web application produces.  Deliverance's means of communication is **HTTP**.  It doesn't call functions or create request objects [*]_.  Again, everything speaks HTTP.
+
+.. [*] This isn't entirely true, Deliverance internally uses `WSGI <http://wsgi.org/wsgi/>`_ which is a Python-level abstraction of HTTP calls.
+
+Deliverance also allows you to include output from multiple locations.  In all cases there's the *theme*, a plain HTML page, and the *content*, whatever the underlying application returns.  You can also include output from other parts of the site, most commonly navigation content that you can manage separately.  All of these pieces can be dynamic -- again, Deliverance only cares about HTML and HTTP, it doesn't worry about what produces the response.
+
+This is all very similar to systems built on XSLT transforms, except without the XSLT [*]_, and without XML.  Strictly speaking you can apply XSLT to any parseable markup, even HTML, but the most common (or at least most talked about) way to apply XSLT is using "semantic" XML output that is transformed into HTML.  Deliverance does not try to understand the semantics of applications, and instead expects them to provide appropriate *presentation* of whatever semantics the underlying application possesses.  Presentation is more universal than semantics.
+
+.. [*] At different times in the past, and potentially again in the future, Deliverance has been compiled down to XSLT rules.  So Deliverance could be seen even as an simple transformation language that compiles down to XSLT.
+
+While Deliverance does its best to work with applications as-they-exist, without making particular demands on those applications, it is not perfect.  Conflicting CSS can be a serious problem.  Some applications don't have very good structure to work with.  You can't generate any content in Deliverance, you can only manipulate existing content, and often that means finding new ways to generate content, or making sure you have a place to store your content (as in the case of navigation).  This is why arguably Deliverance does not *remove* the need for a platform, but is just its own platform.  In so far as this is true, Deliverance tries to be a *better* platform, where "better" is "more universal" rather than "more powerful".  Most templating systems are more powerful than Deliverance transformations.  It can be useful to have access to the underlying objects used to procude the markup.  But Deliverance doesn't give you these things, because it only implements things that can be applied to any source of content.  Static files are entirely workable in Deliverance, just as any application written in Python, PHP, or even an application hosted on an entirely separate service is usable through Deliverance.
+
+The Missing Parts
+-----------------
+
+As mentioned before, two important benefits of a platform are missing from Deliverance.  I'll try to describe what I believe are the essential aspects.  I hope at some time that Deliverance or some complementary application will be able to satisfy these needs.  Also, I suggest some lines of development that might be easier than others.
+
+Indexing The Entire Site
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Typically each application has a notion of what all the interesting pages in that application are.  Most applications have a set of *uninteresting* pages, or transient pages.  A search result is transient, as an example.  An application also knows when new pages appear, and when other pages disappear.  A site-wide index of these pages would allow things like site maps, cross-application search, and cross-application reporting to be done.
+
+An interesting exception to the knowledge an application has of itself: search results are generally boring.  But a search result based on a category might still be interesting.  The difference between a "search" and a "report" is largely in the eye of the beholder.  An important feature is that the application shouldn't be the sole entity allowed to mark interesting pages.  Manually-managed lists of resources that may point to specific applications can allow people to usefully and easily tweak the site.  Ideally even fully external resources could be included, such as a resource on an entirely different site.
+
+To do indexing you need both events (to signal the creation, update, or deletion of an entity/page), and a list of entities (so the index can be completely regenerated).  A simple way of giving a list of entities would be the `Google Site Map XML resource <https://www.google.com/webmasters/tools/docs/en/protocol.html#sitemapXMLFormat>`_.  Signaling events is much more complex, so I won't go into it in any greater depth here, but we're working on a product called `Cabochon <http://www.openplans.org/projects/cabochon/>`_ to handle events.
+
+One thing that indexing can provide is a way to use `microformats <http://microformats.org/>`_.  Right now microformats are interesting, but for most sites they are largely useless.  You can mark up your content, but no one will do anything interesting with that markup.  If you could easily code up an indexer that could keep up-to-date on all the content on your site, you could produce interesting results like cross-application mapping.
+
+Shared Authentication And User Accounts
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Authentication is one of the most common and annoying integration tasks when crossing platform boundaries.  Systems like `Open ID <http://openid.net/>`_ offer the ability to unify *cross-site* authentication, but they don't actually solve the problem of a single site with multiple applications.
+
+There is a basic protocol in HTTP for authentication, one that is workable for a system like Deliverance, and there are already several existing products (like `repoze.who <http://svn.repoze.org/repoze.who/trunk/>`_) that work this way.  It works like this:
+
+* The logged-in username is sent in some header, e.g., ``X-Remote-User``.  Some kind of signing is necessary to really trust this header (Deliverance could filter out that header in incoming requests, but if you removed Deliverance from the stack you'd have a security hole).
+
+* If the user isn't logged in, and the application wants them to log in, the application response with a ``401 Unauthorized`` response.  It is supposed to set the ``WWW-Authenticate`` header, probably to some value indicating that the intermediary should determine the authentication type.  In some cases a kind of HTTP authentication is required (typically `Basic <http://en.wikipedia.org/wiki/Basic_access_authentication>`_ or `Digest <http://en.wikipedia.org/wiki/Digest_access_authentication>`_) because cookie-based logins are too stateful (e.g., in APIs, or for WebDAV access).
+
+* The intermediary catches the 401 and initiates the login process.  This might mean a redirect to a login page, and setting a cookie on successful login.  The login page and setting the cookie could potentially be done by an application outside of the intermediary; the intermediary only has to do the appropriate redirects and setting of headers.
+
+* In the case when a user is logged in but isn't permitted, the application simply sends a ``403 Forbidden`` response.  The intermediary shouldn't actually do anything in this case (though maybe it could usefully add a logout link to that message).  I only mention this because some systems use ``401`` for Forbidden, which causes no end of problems.
+
+While some applications allow for this kind of authentication scheme, many do not.  However, the scheme is general enough that I think it is justifiable that applications could be patched to work like this.
+
+This handles shared authentication, but the only information handed around is a username.  Information about the user -- the real name, email, homepage, permission roles, etc -- are not shared in this model.
+
+You could add something like an internal location to the username.  E.g.: ``X-Remote-User: bob; info_url=http://mysite.com/users/bob.xml``.  It would be the application's responsibility to make a subrequest to fetch that information.  This can be somewhat inefficient, though with appropriate caching perhaps it would be fine.  But many applications want very much to have a complete record of all users.  Changing this is likely to be much harder than changing the authentication scheme.  A more feasible system might be something on the order of what is described in `Indexing the Entire Site`_: provide a complete listing of the site as well as events when users are created, updated, or deleted, and allow applications to maintain their own private but synced databases of users.
+
+A common permission system is another level of integration.  One way of handling this would be if applications had a published set of actions that could be performed, and the person integrating the application could map actions to roles/groups on the system.
+
+Cross-cutting Functionality
+---------------------------
+
+This item requires a bit of explanation.  This is functionality that cuts across multiple parts of the site.  An example might be comments, where you want a commenting system to be applicable to a variety of entities (though probably not all entities).  Or you might want page-update notification, or to provide a feed of changes to the entity.
+
+You might also want to include some request logger like `Google Analytics <http://www.google.com/analytics/>`_ to all pages, but this is already handled well by Deliverance theming.  Deliverance's aggregation handles universal content well, but it doesn't handle content (or subrequests) that should only be present in a portion of pages.
+
+One possible way to address this is `transclusion <http://en.wikipedia.org/wiki/Transclusion>`_, where a page can specifically request some other resource to be included in the page.  A simple subrequest could accomplish this, but many applications make it relatively easy to include some extra markup (e.g., by editing their templates) but not so easy to do something like a subrequest.  We've written a product `Transcluder <http://www.openplans.org/projects/transcluder/>`_ to use an HTML format to indicate transclusion.
+
+It's also possible using Deliverance that you could implement this functionality without any application modification, though it means added configuration -- an application written to be inserted into a page via Deliverance, and a Deliverance rule that plugs everything together (but if written incorrectly would have to be debugged).
+
+Other Conventions
+-----------------
+
+In addition to this, other platform-like conventions would make the life of the integrator much easier.
+
+Template Customization
+~~~~~~~~~~~~~~~~~~~~~~
+
+While Deliverance handles the look-and-feel of a page, it leaves the inner chunk of content to the application.  If you want to tweak something small you will still need to customize the template of the application.
+
+It would be wonderful if applications could report on what files were used in the construction of a request, and used a common search path so you could easily override those files.
+
+Backups and Other Maintenance
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Process management can be handled by something like `Supervisor <http://supervisord.org/>`_, and maybe in the future Deliverance will even embed Supervisor.
+
+But even then, regular backups of the system are important.  Typically each application has its own way of producing a backup.  Conventions for producing backups would be ideal.  Additional conventions for restoring backups would be even better.
+
+Many systems also require periodic maintenance -- compacting databases, checking for any integrity problems, etc.  Some unified cron-like system might be handy, though it's also workable for applications to handle this internally in whatever ad hoc way seems appropriate.
+
+Common Error Reporting
+~~~~~~~~~~~~~~~~~~~~~~
+
+With a system where one of many components can fail, it's important to keep track of these problems.  If errors just end up in one of 10 log files, it's unlikely anyone is closely tracking them.
+
+One product we're working on to help with this is `ErrorEater <https://svn.openplans.org/svn/ErrorEater/trunk/>`_, which works along with `Supervisor <http://supervisord.org/>`_.  Applications have to be modified to emit errors in a specific format that Supervisor understands, but this is generally not too difficult.
+
+Farming
+~~~~~~~
+
+Application farming is when one instance of an application can support many "sites".  These might be sites with their own domains, or just distinct projects.  Examples are `Trac <http://trac.edgewall.org/>`_, which supports multiple projects in one instance, or `WordPress MU <http://mu.wordpress.org/>`_ which supports many WordPress instances running off a single database and code base.
+
+It would be nice if you could add a simple header to a request, like ``X-Project-Name: foo`` and that would be used by all these products to select the site (or sub-site or project or any other organization unit).  Then mapping domain names, paths, or other aspects of a request to the project could be handled once and the applications could all consistently consume it.
+
+(Internally for `openplans.org <http://openplans.org>`_ we're using ``X-OpenPlans-Project`` and custom patches to several projects to support this, but it's all ad hoc.)

From ianb at codespeak.net  Mon Aug 11 21:56:27 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Mon, 11 Aug 2008 21:56:27 +0200 (CEST)
Subject: [z3-checkins] r57199 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/docs
Message-ID: <20080811195627.7CFA816A20B@codespeak.net>

Author: ianb
Date: Mon Aug 11 21:56:24 2008
New Revision: 57199

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt
   z3/deliverance/sandbox/ianb/deliverance/trunk/docs/philosophy.txt
Log:
rearrange the header levels a bit; typos

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/configuration.txt	Mon Aug 11 21:56:24 2008
@@ -48,7 +48,7 @@
 
 Then define a function like:
 
-.. code-bock:: python
+.. code-block:: python
 
     def get_theme(request, response, log):
         return "/%s/theme.html" % request.host
@@ -319,7 +319,7 @@
 
 A third way is with ``<match>``.  These elements match requests and add classes.  You can use them like:
 
-.. code-block::
+.. code-block:: python
 
     <match domain="lists.*" class="lists" />
 

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/index.txt	Mon Aug 11 21:56:24 2008
@@ -10,6 +10,11 @@
    pyref
    news
    license
+
+These are some documents that may or may not be accurate; they are a little out of date:
+
+.. toctree::
+
    page-classes
    rules
    proxy

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/docs/philosophy.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/docs/philosophy.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/docs/philosophy.txt	Mon Aug 11 21:56:24 2008
@@ -1,12 +1,14 @@
 The Philosophy Of Deliverance
-=============================
++++++++++++++++++++++++++++++
 
 :author: Ian Bicking <ianb at openplans.org>
 
+.. contents::
+
 Why is Deliverance?  Why was it made, what purpose does it serve, why should you use it, how can it change the way you do web development?
 
 On the Subject of Platforms
----------------------------
+===========================
 
 Right now we live in an age of *platforms*.  Developers (or management or coincidence) decides on a platform, and that serves as the basis for all future development.  Usually there's some old things from a previous platform (or a primordial pre-platform age: I'm looking at you ``formmail.pl``!)  The goal is always to eliminate all of these old pieces, rewriting them for the new platform.  That goal is seldom attained in a timely manner, and even before it is accomplished you may be moving to the next platform.
 
@@ -14,18 +16,18 @@
 
 A similar conflict arises when you consider how to add new functionality to a site.  For example, you may want to add a blog.  Do you:
 
-a. Use the best blogging software available?  
-b. Use something native to your platform?  
-c. Write something yourself?
+1. Use the best blogging software available?  
+2. Use something native to your platform?  
+3. Write something yourself?
 
-The answer is probably (b) or (c), because it would be too hard to integrate something foreign to your platform.  This form of choice means that every platform has some kind of "blog", but the users of that blog are likely to only be a subset of the users of the parent platform.  This makes it difficult for winners to emerge, or for a well-developed piece of software to really be successful.  Platform-based software is limited by the adoption of the platform.
+The answer is probably 2 or 3, because it would be too hard to integrate something foreign to your platform.  This form of choice means that every platform has some kind of "blog", but the users of that blog are likely to only be a subset of the users of the parent platform.  This makes it difficult for winners to emerge, or for a well-developed piece of software to really be successful.  Platform-based software is limited by the adoption of the platform.
 
 Not all software has a platform.  These tend to be the most successful web applications, things like `Trac <http://trac.edgewall.org/>`_, `WordPress <http://wordpress.org>`_, etc.
 
 "Aha!" you think "I'll just use those best-of-breed applications!"  But no!  Those applications themselves turn into platforms.  WordPress is practically a CMS.  Trac too.  Extensible applications, if successful, become their own platform.  This is not to place blame, they aren't necessarily any worse than any other platform, just an acknowledgment that this move to platform can happen anywhere.
 
 Beyond Platforms, or A Better Platform
---------------------------------------
+======================================
 
 One of the major goals of Deliverance is to move beyond platforms.  It is an *integration tool*, to allow applications from different frameworks or languages to be integrated gracefully.
 
@@ -41,23 +43,19 @@
 
 Deliverance applies a common theme across all the applications in your site.  It's basic unit of abstraction is **HTML**.  It doesn't use a particular templating language.  It doesn't know what an object is.  HTML is something every web application produces.  Deliverance's means of communication is **HTTP**.  It doesn't call functions or create request objects [*]_.  Again, everything speaks HTTP.
 
-.. [*] This isn't entirely true, Deliverance internally uses `WSGI <http://wsgi.org/wsgi/>`_ which is a Python-level abstraction of HTTP calls.
-
 Deliverance also allows you to include output from multiple locations.  In all cases there's the *theme*, a plain HTML page, and the *content*, whatever the underlying application returns.  You can also include output from other parts of the site, most commonly navigation content that you can manage separately.  All of these pieces can be dynamic -- again, Deliverance only cares about HTML and HTTP, it doesn't worry about what produces the response.
 
 This is all very similar to systems built on XSLT transforms, except without the XSLT [*]_, and without XML.  Strictly speaking you can apply XSLT to any parseable markup, even HTML, but the most common (or at least most talked about) way to apply XSLT is using "semantic" XML output that is transformed into HTML.  Deliverance does not try to understand the semantics of applications, and instead expects them to provide appropriate *presentation* of whatever semantics the underlying application possesses.  Presentation is more universal than semantics.
 
-.. [*] At different times in the past, and potentially again in the future, Deliverance has been compiled down to XSLT rules.  So Deliverance could be seen even as an simple transformation language that compiles down to XSLT.
-
 While Deliverance does its best to work with applications as-they-exist, without making particular demands on those applications, it is not perfect.  Conflicting CSS can be a serious problem.  Some applications don't have very good structure to work with.  You can't generate any content in Deliverance, you can only manipulate existing content, and often that means finding new ways to generate content, or making sure you have a place to store your content (as in the case of navigation).  This is why arguably Deliverance does not *remove* the need for a platform, but is just its own platform.  In so far as this is true, Deliverance tries to be a *better* platform, where "better" is "more universal" rather than "more powerful".  Most templating systems are more powerful than Deliverance transformations.  It can be useful to have access to the underlying objects used to procude the markup.  But Deliverance doesn't give you these things, because it only implements things that can be applied to any source of content.  Static files are entirely workable in Deliverance, just as any application written in Python, PHP, or even an application hosted on an entirely separate service is usable through Deliverance.
 
 The Missing Parts
------------------
+=================
 
 As mentioned before, two important benefits of a platform are missing from Deliverance.  I'll try to describe what I believe are the essential aspects.  I hope at some time that Deliverance or some complementary application will be able to satisfy these needs.  Also, I suggest some lines of development that might be easier than others.
 
 Indexing The Entire Site
-~~~~~~~~~~~~~~~~~~~~~~~~
+------------------------
 
 Typically each application has a notion of what all the interesting pages in that application are.  Most applications have a set of *uninteresting* pages, or transient pages.  A search result is transient, as an example.  An application also knows when new pages appear, and when other pages disappear.  A site-wide index of these pages would allow things like site maps, cross-application search, and cross-application reporting to be done.
 
@@ -68,7 +66,7 @@
 One thing that indexing can provide is a way to use `microformats <http://microformats.org/>`_.  Right now microformats are interesting, but for most sites they are largely useless.  You can mark up your content, but no one will do anything interesting with that markup.  If you could easily code up an indexer that could keep up-to-date on all the content on your site, you could produce interesting results like cross-application mapping.
 
 Shared Authentication And User Accounts
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+---------------------------------------
 
 Authentication is one of the most common and annoying integration tasks when crossing platform boundaries.  Systems like `Open ID <http://openid.net/>`_ offer the ability to unify *cross-site* authentication, but they don't actually solve the problem of a single site with multiple applications.
 
@@ -137,3 +135,10 @@
 It would be nice if you could add a simple header to a request, like ``X-Project-Name: foo`` and that would be used by all these products to select the site (or sub-site or project or any other organization unit).  Then mapping domain names, paths, or other aspects of a request to the project could be handled once and the applications could all consistently consume it.
 
 (Internally for `openplans.org <http://openplans.org>`_ we're using ``X-OpenPlans-Project`` and custom patches to several projects to support this, but it's all ad hoc.)
+
+Footnotes
+=========
+
+.. [*] This isn't entirely true, Deliverance internally uses `WSGI <http://wsgi.org/wsgi/>`_ which is a Python-level abstraction of HTTP calls.
+
+.. [*] At different times in the past, and potentially again in the future, Deliverance has been compiled down to XSLT rules.  So Deliverance could be seen even as an simple transformation language that compiles down to XSLT.

From optilude at codespeak.net  Sat Aug 16 23:42:54 2008
From: optilude at codespeak.net (optilude at codespeak.net)
Date: Sat, 16 Aug 2008 23:42:54 +0200 (CEST)
Subject: [z3-checkins] r57335 - z3/deliverance/sandboxes/optilude
Message-ID: <20080816214254.8747B168572@codespeak.net>

Author: optilude
Date: Sat Aug 16 23:42:52 2008
New Revision: 57335

Added:
   z3/deliverance/sandboxes/optilude/
Log:
Make sandbox


From optilude at codespeak.net  Sat Aug 16 23:43:50 2008
From: optilude at codespeak.net (optilude at codespeak.net)
Date: Sat, 16 Aug 2008 23:43:50 +0200 (CEST)
Subject: [z3-checkins] r57336 - in
	z3/deliverance/sandboxes/optilude/dv.xdvserver: . branches
	tags trunk
Message-ID: <20080816214350.898A7168571@codespeak.net>

Author: optilude
Date: Sat Aug 16 23:43:49 2008
New Revision: 57336

Added:
   z3/deliverance/sandboxes/optilude/dv.xdvserver/
   z3/deliverance/sandboxes/optilude/dv.xdvserver/branches/
   z3/deliverance/sandboxes/optilude/dv.xdvserver/tags/
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/
Log:
Space for dv.xdvserver


From optilude at codespeak.net  Sat Aug 16 23:45:15 2008
From: optilude at codespeak.net (optilude at codespeak.net)
Date: Sat, 16 Aug 2008 23:45:15 +0200 (CEST)
Subject: [z3-checkins] r57337 - in
	z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk: . docs
	dv dv/xdvserver
Message-ID: <20080816214515.11660168571@codespeak.net>

Author: optilude
Date: Sat Aug 16 23:45:12 2008
New Revision: 57337

Added:
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/README.txt
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/HISTORY.txt
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/INSTALL.txt
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/LICENSE.GPL
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/LICENSE.txt
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/__init__.py
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/   (props changed)
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/__init__.py
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/setup.py
Modified:
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/   (props changed)
Log:
Check in dv.xvserver


Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/README.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/README.txt	Sat Aug 16 23:45:12 2008
@@ -0,0 +1,56 @@
+Introduction
+============
+
+dv.xdvserver is a simple piece of WSGI middleware that can execute the two
+step compile-and-run XSLT transforms of xdv.
+
+It takes two required parameters:
+
+ - rules: a path to a file containing Deliverance rules
+ - theme_uri: a URI to a theme HTML file
+
+In addition, it can take several optional parameters:
+
+ - live: set to True to recompile the theme on each request, rather than on
+    startup only.
+ - compiler: a path to the XSLT file that can turn theme+rules into a compiled
+    theme. The default, bundled version will probably suffice in most cases.
+ - boilerplate: a path to the XSLT file that contains boilerplate XSLT
+    instructions. The default, bundled version will probably suffice in most
+    cases.
+    
+Configuration
+=============
+
+You can use this middleware in a Paste Deploy pipeline. Here is an example 
+configuration file of an application that themes a Plone site running on
+http://localhost:8080/demo. Static resources are served from /static.
+
+    [server:main]
+    use = egg:Paste#http
+    host = 127.0.0.1
+    port = 5000
+
+    [composite:main]
+    use = egg:Paste#urlmap
+    /static = static
+    / = default
+
+    [app:static]
+    use = egg:Paste#static
+    document_root = %(here)s/static
+
+    [pipeline:default]
+    pipeline = egg:Paste#cgitb
+               egg:Paste#httpexceptions
+               theme.default
+               zope.proxy
+
+    [filter:theme.default]
+    use = egg:dv.xdvserver#xdv
+    theme_uri = file://%(here)s/static/index.html
+    rules = %(here)s/static/rules/default.xml
+
+    [app:zope.proxy]
+    use = egg:Paste#proxy
+    address = http://localhost:8080/VirtualHostBase/http/localhost:5000/demo/VirtualHostRoot/
\ No newline at end of file

Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/HISTORY.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/HISTORY.txt	Sat Aug 16 23:45:12 2008
@@ -0,0 +1,8 @@
+Changelog
+=========
+
+1.0 - Unreleased
+----------------
+
+* Initial release
+

Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/INSTALL.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/INSTALL.txt	Sat Aug 16 23:45:12 2008
@@ -0,0 +1,43 @@
+dv.xdvserver Installation
+==========================
+
+To install dv.xdvserver into the global Python environment (or a workingenv),
+using a traditional Zope 2 instance, you can do this:
+
+ * When you're reading this you have probably already run 
+   ``easy_install dv.xdvserver``. Find out how to install setuptools
+   (and EasyInstall) here:
+   http://peak.telecommunity.com/DevCenter/EasyInstall
+
+ * Create a file called ``dv.xdvserver-configure.zcml`` in the
+   ``/path/to/instance/etc/package-includes`` directory.  The file
+   should only contain this::
+
+       <include package="dv.xdvserver" />
+
+
+Alternatively, if you are using zc.buildout and the plone.recipe.zope2instance
+recipe to manage your project, you can do this:
+
+ * Add ``dv.xdvserver`` to the list of eggs to install, e.g.:
+ 
+    [buildout]
+    ...
+    eggs =
+        ...
+        dv.xdvserver
+        
+  * Tell the plone.recipe.zope2instance recipe to install a ZCML slug:
+  
+    [instance]
+    recipe = plone.recipe.zope2instance
+    ...
+    zcml =
+        dv.xdvserver
+        
+  * Re-run buildout, e.g. with:
+  
+    $ ./bin/buildout
+        
+You can skip the ZCML slug if you are going to explicitly include the package
+from another package's configure.zcml file.

Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/LICENSE.GPL
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/LICENSE.GPL	Sat Aug 16 23:45:12 2008
@@ -0,0 +1,222 @@
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS

Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/LICENSE.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/docs/LICENSE.txt	Sat Aug 16 23:45:12 2008
@@ -0,0 +1,16 @@
+  dv.xdvserver is copyright Martin Aspeli
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 59 Temple Place, Suite 330, Boston, 
+  MA 02111-1307 USA.

Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/__init__.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/__init__.py	Sat Aug 16 23:45:12 2008
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/__init__.py
==============================================================================

Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/setup.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/setup.py	Sat Aug 16 23:45:12 2008
@@ -0,0 +1,38 @@
+from setuptools import setup, find_packages
+import os
+
+version = '1.0'
+
+setup(name='dv.xdvserver',
+      version=version,
+      description="A server for the Deliverance/XSLT compiler",
+      long_description=open("README.txt").read() + "\n" +
+                       open(os.path.join("docs", "HISTORY.txt")).read(),
+      # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers
+      classifiers=[
+        "Framework :: Plone",
+        "Programming Language :: Python",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        ],
+      keywords='Deliverance XSLT WSGI',
+      author='Martin Aspeli',
+      author_email='optilude at gmail.com',
+      url='http://open-plans.org/projects/deliverance',
+      license='LGPL',
+      packages=find_packages(exclude=['ez_setup']),
+      namespace_packages=['dv'],
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=[
+          'setuptools',
+          'PasteDeploy',
+          'PasteScript',
+          'lxml',
+          # -*- Extra requirements: -*-
+      ],
+      entry_points="""
+      [paste.filter_app_factory]
+      xslt = dv.xdvserver.filter:XSLTMiddleware
+      xdv = dv.xdvserver.filter:XDVMiddleware
+      """,
+      )

From optilude at codespeak.net  Sun Aug 17 13:14:27 2008
From: optilude at codespeak.net (optilude at codespeak.net)
Date: Sun, 17 Aug 2008 13:14:27 +0200 (CEST)
Subject: [z3-checkins] r57342 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080817111427.75EC6169E0F@codespeak.net>

Author: optilude
Date: Sun Aug 17 13:14:26 2008
New Revision: 57342

Modified:
   z3/deliverance/sandboxes/paul/xdv/README.txt
Log:
Update docs


Modified: z3/deliverance/sandboxes/paul/xdv/README.txt
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/README.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/README.txt	Sun Aug 17 13:14:26 2008
@@ -44,8 +44,6 @@
 To Do
 -------
 
-- Add support for <drop>
-
 - Make sure attribute merging is handled
 
 

From optilude at codespeak.net  Sun Aug 17 13:18:51 2008
From: optilude at codespeak.net (optilude at codespeak.net)
Date: Sun, 17 Aug 2008 13:18:51 +0200 (CEST)
Subject: [z3-checkins] r57343 -
	z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver
Message-ID: <20080817111851.1D9B4169E2D@codespeak.net>

Author: optilude
Date: Sun Aug 17 13:18:50 2008
New Revision: 57343

Added:
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py
Log:
Add middleware


Added: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py	Sun Aug 17 13:18:50 2008
@@ -0,0 +1,170 @@
+import os.path
+import re
+import urllib2
+
+from lxml import etree
+
+from paste.request import construct_url
+from paste.response import header_value, replace_header
+from paste.wsgilib import intercept_output
+from paste.deploy.converters import asbool
+
+IGNORE_EXTENSIONS = ['js', 'css', 'gif', 'jpg', 'jpeg', 'pdf', 'ps', 'doc',
+                     'png', 'ico', 'mov', 'mpg', 'mpeg', 'mp3', 'm4a', 'txt',
+                     'rtf', 'swf', 'wav', 'zip', 'wmv', 'ppt', 'gz', 'tgz',
+                     'jar', 'xls', 'bmp', 'tif', 'tga', 'hqx', 'avi',
+                    ]
+
+IGNORE_URL_PATTERN = re.compile("^.*\.(%s)$" % '|'.join(IGNORE_EXTENSIONS))
+HTML_DOC_PATTERN = re.compile(r"^.*<\s*html(\s*|>).*$",re.I|re.M)
+
+class XSLTMiddleware(object):
+    """Apply XSLT in middleware
+    """
+    
+    def __init__(self, app, global_conf, xslt_file=None, xslt_source=""):
+        """Initialise, giving a filename or file pointer for an XSLT file.
+        """
+        
+        self.app = app
+        self.global_conf = global_conf
+
+        if xslt_file is not None:
+            xslt_file = open(xslt_file)
+            xslt_source = xslt_file.read()
+            xslt_file.close()
+        
+        xslt_source = xslt_source
+        xslt_tree = etree.fromstring(xslt_source)
+        self.transform = etree.XSLT(xslt_tree)
+        
+    def should_intercept(self, status, headers):
+        """Callback to determine if the content should be intercepted
+        """
+        apply_theme = not header_value(headers, 'x-deliverance-no-theme')
+        if not apply_theme:
+            return False
+
+        content_type = header_value(headers, 'content-type')
+        if content_type is None:
+            return True # 304s can have no content-type 
+
+        return (content_type.startswith('text/html') or
+                content_type.startswith('application/xhtml+xml'))
+    
+    def apply_transform(self, environ, body):
+        content = etree.fromstring(body, parser=etree.HTMLParser())
+        transformed = self.transform(content)
+        return etree.tostring(transformed)
+    
+    def __call__(self, environ, start_response):
+        
+        if not environ['PATH_INFO']:
+            environ['PATH_INFO'] = '/'
+        
+        status, headers, body = intercept_output(environ, self.app,
+                                                 self.should_intercept,
+                                                 start_response)
+                                                 
+        # self.should_intercept returned nada
+        if status is None:
+            return body
+        
+        # short circuit from theming if this is not likely to be HTML
+        content_url = construct_url(environ)
+        if not self.is_html(body) or self.should_ignore_url(content_url):
+            start_response(status, headers)
+            return [body]
+            
+        # short circuit if we have a 3xx, 204 or 401 error code
+        status_code = status.split()[0]
+        if status_code.startswith('3') or status_code == '204' or status_code == '401':
+            start_response(status, headers)
+            return [body]
+        
+        # all good - apply the transform
+        body = self.apply_transform(environ, body)
+        
+        replace_header(headers, 'content-length', str(len(body)))
+        replace_header(headers, 'content-type', 'text/html; charset=utf-8')
+        
+        start_response(status, headers)
+        return [body]
+
+    def should_ignore_url(self, url): 
+        return IGNORE_URL_PATTERN.search(url) is not None
+
+    def is_html(self, body):
+        return HTML_DOC_PATTERN.search(body) is not None
+
+class XDVMiddleware(object):
+    """Invoke the Deliverance xdv transform as middleware
+    """
+    
+    def __init__(self, app, global_conf, theme_uri, rules, compiler=None, boilerplate=None, live=False):
+        """Create the middleware. The parameters are:
+        
+            theme_uri
+                URI from which to pull the theme
+            rules
+                Filename or path to the rules file
+            compiler
+                Filename or path to the compiler XSLT. A bundled version of
+                the xdv compiler.xsl will be used if this is not set.
+            boilerplate
+                Filename or path to the boilerplate XSLT file that the
+                compiler expects to be able to use. A bundled version from
+                xdv will be used if this is not set.
+            live
+                If set to true, the theme will be recompiled on each
+                request. The default is to compile the theme on startup only.
+        """
+        self.app = app
+        self.global_conf = global_conf
+        
+        self.theme_uri = theme_uri
+        
+        self.compiler = compiler
+        self.boilerplate = boilerplate
+        self.rules = rules
+
+        if compiler is None:
+            self.compiler = os.path.join(os.path.split(__file__)[0], 'compiler', 'compiler.xsl')
+
+        if boilerplate is None:
+            self.boilerplate = os.path.join(os.path.split(__file__)[0], 'compiler', 'boilerplate.xsl')
+        
+        if not os.path.isfile(self.compiler):
+            raise ValueError("Compiler XSLT %s does not exist" % self.compiler)
+        if not os.path.isfile(self.boilerplate):
+            raise ValueError("Boilerplate XSLT %s does not exist" % self.boilerplate)
+        if not os.path.isfile(self.rules):
+            raise ValueError("Rules file %s does not exist" % self.rules)
+        
+        self.live = asbool(live)
+        self.transform = None
+        if not live:
+            self.transform = XSLTMiddleware(app, global_conf, xslt_source=self.compile_theme())
+    
+    def compile_theme(self):
+        compiler = open(self.compiler)
+        compiler_transform = etree.XSLT(etree.ElementTree(file=compiler))
+        compiler.close()
+        
+        theme = urllib2.urlopen(self.theme_uri)
+        parser = etree.HTMLParser()
+        theme_tree = etree.ElementTree(file=theme, parser=parser)
+        theme.close()
+        
+        compiled = compiler_transform(theme_tree,
+                                      rulesuri="'%s'" % self.rules,
+                                      boilerplateurl="'%s'" % self.boilerplate)
+        
+        return etree.tostring(compiled)
+    
+    def __call__(self, environ, start_response):
+        transform = self.transform
+        if self.live:
+            transform = XSLTMiddleware(self.app, self.global_conf, xslt_source=self.compile_theme())
+        return transform(environ, start_response)
+        
\ No newline at end of file

From optilude at codespeak.net  Wed Aug 20 01:25:19 2008
From: optilude at codespeak.net (optilude at codespeak.net)
Date: Wed, 20 Aug 2008 01:25:19 +0200 (CEST)
Subject: [z3-checkins] r57491 -
	z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver
Message-ID: <20080819232519.21D8D169FA0@codespeak.net>

Author: optilude
Date: Wed Aug 20 01:25:18 2008
New Revision: 57491

Modified:
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py
Log:
Add some pre-processing to do link rewriting.


Modified: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py
==============================================================================
--- z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py	(original)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py	Wed Aug 20 01:25:18 2008
@@ -17,6 +17,15 @@
 
 IGNORE_URL_PATTERN = re.compile("^.*\.(%s)$" % '|'.join(IGNORE_EXTENSIONS))
 HTML_DOC_PATTERN = re.compile(r"^.*<\s*html(\s*|>).*$",re.I|re.M)
+IMPORT_STYLESHEET_PATTERN = re.compile('@import url\\([\'"](.+)[\'"]\\);', re.I)
+
+def to_absolute(src, prefix):
+    if not (src.startswith('/') or '://' in src):
+        if src.startswith('./'):
+            return "%s/%s" % (prefix, src[:2])
+        else:
+            return "%s/%s" % (prefix, src)
+    return src
 
 class XSLTMiddleware(object):
     """Apply XSLT in middleware
@@ -101,7 +110,8 @@
     """Invoke the Deliverance xdv transform as middleware
     """
     
-    def __init__(self, app, global_conf, theme_uri, rules, compiler=None, boilerplate=None, live=False):
+    def __init__(self, app, global_conf, theme_uri, rules, compiler=None,
+                    boilerplate=None, live=False, absolute_prefix=None):
         """Create the middleware. The parameters are:
         
             theme_uri
@@ -118,6 +128,9 @@
             live
                 If set to true, the theme will be recompiled on each
                 request. The default is to compile the theme on startup only.
+            absolute_prefix
+                If set to a string, then all relative image and CSS references
+                in the theme will be prefixed by this string.
         """
         self.app = app
         self.global_conf = global_conf
@@ -127,6 +140,8 @@
         self.compiler = compiler
         self.boilerplate = boilerplate
         self.rules = rules
+        
+        self.absolute_prefix = absolute_prefix
 
         if compiler is None:
             self.compiler = os.path.join(os.path.split(__file__)[0], 'compiler', 'compiler.xsl')
@@ -156,6 +171,19 @@
         theme_tree = etree.ElementTree(file=theme, parser=parser)
         theme.close()
         
+        if self.absolute_prefix:
+            for node in theme_tree.xpath('*//style | *//img | *//link'):
+                if node.tag == 'img':
+                    src = node.get('src')
+                    if src:
+                        node.set('src', to_absolute(src, self.absolute_prefix))
+                elif node.tag == 'link':
+                    href = node.get('href')
+                    if href:
+                        node.set('href', to_absolute(href, self.absolute_prefix))
+                elif node.tag == 'style':
+                    node.text = IMPORT_STYLESHEET_PATTERN.sub('@import url("%s/\\1");' % self.absolute_prefix, node.text)
+        
         compiled = compiler_transform(theme_tree,
                                       rulesuri="'%s'" % self.rules,
                                       boilerplateurl="'%s'" % self.boilerplate)

From optilude at codespeak.net  Wed Aug 20 01:30:26 2008
From: optilude at codespeak.net (optilude at codespeak.net)
Date: Wed, 20 Aug 2008 01:30:26 +0200 (CEST)
Subject: [z3-checkins] r57492 -
	z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver
Message-ID: <20080819233026.36ED3169FD1@codespeak.net>

Author: optilude
Date: Wed Aug 20 01:30:24 2008
New Revision: 57492

Modified:
   z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py
Log:
Don't forget script tags


Modified: z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py
==============================================================================
--- z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py	(original)
+++ z3/deliverance/sandboxes/optilude/dv.xdvserver/trunk/dv/xdvserver/filter.py	Wed Aug 20 01:30:24 2008
@@ -172,8 +172,8 @@
         theme.close()
         
         if self.absolute_prefix:
-            for node in theme_tree.xpath('*//style | *//img | *//link'):
-                if node.tag == 'img':
+            for node in theme_tree.xpath('*//style | *//script | *//img | *//link'):
+                if node.tag == 'img' or node.tag == 'script':
                     src = node.get('src')
                     if src:
                         node.set('src', to_absolute(src, self.absolute_prefix))

From z3-checkins at codespeak.net  Tue Aug 26 21:12:08 2008
From: z3-checkins at codespeak.net (Microsoft XP)
Date: Tue, 26 Aug 2008 21:12:08 +0200 (CEST)
Subject: ® Official Update 2008! 
Message-ID: <20080826101209.3974.qmail@acyo170.neoplus.adsl.tpnet.pl>

An HTML attachment was scrubbed...
URL: http://codespeak.net/pipermail/z3-checkins/attachments/20080826/dd711d33/attachment.htm 

From paul at codespeak.net  Wed Aug 27 14:25:42 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Wed, 27 Aug 2008 14:25:42 +0200 (CEST)
Subject: [z3-checkins] r57642 - in z3/deliverance/sandboxes/paul/xdv/tests:
	. 001 002 003 004 005 006 007
Message-ID: <20080827122542.49B1E16A0A3@codespeak.net>

Author: paul
Date: Wed Aug 27 14:25:39 2008
New Revision: 57642

Added:
   z3/deliverance/sandboxes/paul/xdv/tests/001/
   z3/deliverance/sandboxes/paul/xdv/tests/001/README.rst
   z3/deliverance/sandboxes/paul/xdv/tests/001/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/001/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/001/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/001/xpaths.txt
   z3/deliverance/sandboxes/paul/xdv/tests/002/
   z3/deliverance/sandboxes/paul/xdv/tests/002/README.rst
   z3/deliverance/sandboxes/paul/xdv/tests/002/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/002/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/002/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/002/xpaths.txt
   z3/deliverance/sandboxes/paul/xdv/tests/003/
   z3/deliverance/sandboxes/paul/xdv/tests/003/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/003/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/003/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/003/xpaths.txt
   z3/deliverance/sandboxes/paul/xdv/tests/004/
   z3/deliverance/sandboxes/paul/xdv/tests/004/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/004/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/004/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/004/xpaths.txt
   z3/deliverance/sandboxes/paul/xdv/tests/005/
   z3/deliverance/sandboxes/paul/xdv/tests/005/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/005/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/005/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/005/xpaths.txt
   z3/deliverance/sandboxes/paul/xdv/tests/006/
   z3/deliverance/sandboxes/paul/xdv/tests/006/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/006/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/006/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/006/xpaths.txt
   z3/deliverance/sandboxes/paul/xdv/tests/007/
   z3/deliverance/sandboxes/paul/xdv/tests/007/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt
Removed:
   z3/deliverance/sandboxes/paul/xdv/tests/content-00.html
   z3/deliverance/sandboxes/paul/xdv/tests/content-01.html
   z3/deliverance/sandboxes/paul/xdv/tests/rules-01.xml
Modified:
   z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
Log:
Temporary cut at a test

Added: z3/deliverance/sandboxes/paul/xdv/tests/001/README.rst
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/001/README.rst	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,6 @@
+Basic replace
+=============
+
+Use <replace> to replace a theme node with a content node.  Put an
+attribute on the theme node to ensure that the result doesn't have the
+attribute.

Added: z3/deliverance/sandboxes/paul/xdv/tests/001/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/001/content.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,7 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>A Deeper Look At xdv</title>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/001/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/001/rules.xml	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <replace theme="/html/head/title" content="/html/head/title"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/001/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/001/theme.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,7 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title id="nothing">Second Theme Page Title</title>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/001/xpaths.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/001/xpaths.txt	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,3 @@
+# XPaths that must be true to "validate" this test
+/html/head/title = 'A Deeper Look At DVNG'
+not(/html/head/title/@id)

Added: z3/deliverance/sandboxes/paul/xdv/tests/002/README.rst
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/002/README.rst	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,5 @@
+Copy and preserve attributes
+============================
+
+Use <copy> to copy the content while preserving the attribute of a
+node in the HTML.

Added: z3/deliverance/sandboxes/paul/xdv/tests/002/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/002/content.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,7 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>A Deeper Look At xdv</title>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/002/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/002/rules.xml	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <copy theme="/html/head/title" content="/html/head/title"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/002/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/002/theme.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,7 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title id="something">Second Theme Page Title</title>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/002/xpaths.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/002/xpaths.txt	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,3 @@
+# XPaths that must be true to "validate" this test
+/html/head/title = 'A Deeper Look At DVNG'
+/html/head/title/@id = 'something'

Added: z3/deliverance/sandboxes/paul/xdv/tests/003/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/003/content.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,8 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>A Deeper Look At xdv</title>
+        <link id="fourth" rel="stylesheet" href="fourth.css"/>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/003/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/003/rules.xml	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <append theme="/html/head" content="/html/head/link"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/003/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/003/theme.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,9 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title id="first">Second Theme Page Title</title>
+        <script id="second" src="second.js" type="text/javascript"/>
+        <link id="third" rel="stylesheet" href="third.css"/>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/003/xpaths.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/003/xpaths.txt	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,4 @@
+# 
+# Basic append.  Copy a <link> from the content and make sure 
+# that it shows up at the end.
+/html/head/*[position()='4']/@id='fourth'

Added: z3/deliverance/sandboxes/paul/xdv/tests/004/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/004/content.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,8 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>A Deeper Look At xdv</title>
+        <link id="fourth" rel="stylesheet" href="fourth.css"/>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/004/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/004/rules.xml	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <prepend theme="/html/head" content="/html/head/link"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/004/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/004/theme.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,9 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title id="first">Second Theme Page Title</title>
+        <script id="second" src="second.js" type="text/javascript"/>
+        <link id="third" rel="stylesheet" href="third.css"/>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/004/xpaths.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/004/xpaths.txt	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,4 @@
+# 
+# Basic prepend.  Copy a <link> from the content and make sure 
+# that it shows up at the beginning of the <head>.
+/html/head/*[position()='1']/@id='fourth'

Added: z3/deliverance/sandboxes/paul/xdv/tests/005/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/005/content.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,13 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>A Deeper Look At xdv</title>
+    </head>
+    <body>
+        <div id="pagecontent">
+            <h1>A Deeper Look....at...xdv</h1>
+            <p>And some content.</p>
+            <div id="contentdrop">Get rid of <em>me!!</em></div>
+        </div>
+    </body>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/005/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/005/rules.xml	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <copy theme="/html/body/div[@id='pagecontent']" content="/html/body/div[@id='pagecontent']/*"/>
+    <drop content="/html/body/div/div[@id='contentdrop']"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/005/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/005/theme.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,9 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>Second Theme Page Title</title>
+    </head>
+    <body>
+        <div id="pagecontent">Some stuff</div>
+    </body>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/005/xpaths.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/005/xpaths.txt	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,4 @@
+# 
+# Basic <drop>.  Drop one node from the content.
+#
+not(//div[@id='contentdrop'])

Added: z3/deliverance/sandboxes/paul/xdv/tests/006/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/006/content.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,9 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>A Deeper Look At xdv</title>
+        <link id="veryfirst" rel="stylesheet" href="veryfirst.css"/>
+        <link id="last" rel="stylesheet" href="last.css"/>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/006/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/006/rules.xml	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <prepend theme="/html/head" content="/html/head/link[@id='veryfirst']"/>
+    <append theme="/html/head" content="/html/head/link[@id='last']"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/006/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/006/theme.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,9 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title id="first">Second Theme Page Title</title>
+        <script id="second" src="second.js" type="text/javascript"/>
+        <link id="third" rel="stylesheet" href="third.css"/>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/006/xpaths.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/006/xpaths.txt	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,6 @@
+# 
+# Make sure we can do a <prepend> and <append> on /html/head.
+# There is a special case for this in the code.
+/html/head/*[position()='1']/@id='veryfirst'
+/html/head/*[position()='5']/@id='last'
+

Added: z3/deliverance/sandboxes/paul/xdv/tests/007/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/007/content.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,13 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>A Deeper Look At xdv</title>
+    </head>
+    <body>
+        <div id="wrapper" class="foo">
+            <div id="content">
+                <p>boo bar baz</p>
+            </div>
+        </div>
+    </body>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <append theme="/html/body" content="/html/body/div[@id='wrapper']/*"/>
+    <attcopy theme="/html/body" content="/html/body/div[@id='wrapper']/@class"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,9 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title id="first">Second Theme Page Title</title>
+    </head>
+    <body>
+        <p>blah</p>
+    </body>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt	Wed Aug 27 14:25:39 2008
@@ -0,0 +1,13 @@
+#
+# Test <attcopy>.  First, use <append> to copy over some stuff 
+# intoy <body>.  Then, use the @class on a <div> in the content
+# to set the body/@class.
+/html/body/@class='foo'
+not(//div[@id='wrapper'])
+/html/body/*[1][name()='p']
+/html/body/div[@id='content']/p/text() = 'boo bar baz'
+
+#
+# XXXX Paul, come back and put <attcopy> into the non-special-case 
+# so it works when not matching on an overlapping <body> 
+# selection.
\ No newline at end of file

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/content-00.html
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/content-00.html	Wed Aug 27 14:25:39 2008
+++ (empty file)
@@ -1,10 +0,0 @@
-<html>
-    <head>
-        <title>A Deeper Look At DVNG</title>
-        <meta name="dv.section" content="DVNG Articles"/>
-    </head>
-    <body>
-        <p>I have some content <em>in here</em> to merge.</p>
-        <p>Perhaps an image will be in here later.</p>
-    </body>
-</html>

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/content-01.html
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/content-01.html	Wed Aug 27 14:25:39 2008
+++ (empty file)
@@ -1 +0,0 @@
-<title>Content 01</title>

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/rules-01.xml
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/rules-01.xml	Wed Aug 27 14:25:39 2008
+++ (empty file)
@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<rules xmlns="http://openplans.org/deliverance">
-    <replace theme="title" content="title"/>
-</rules>

Modified: z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	Wed Aug 27 14:25:39 2008
@@ -1,32 +1,61 @@
+#
+# Simple test runner for validating different xdv scenarios
+#
 
 from lxml import etree
+import os
+import sys
+
+_HERE = os.path.abspath(os.path.dirname(__file__))
 
 class XDV:
 
     def __init__(self, testnumber):
         testnumber = testnumber
-        themefn = "tests/theme-%s.html"      % testnumber
-        rulesfn = "tests/rules-%s.xml"       % testnumber
-        contentfn = "tests/content-%s.html"  % testnumber
-
-        themedoc = etree.ElementTree(file=themefn)
-        contentdoc = etree.ElementTree(file=contentfn)
-
-        compilerfn = "compiler.xsl"
+        themefn = "%s/theme.html"           % testnumber
+        contentfn = "%s/content.html"       % testnumber
+        rulesfn = os.path.join(_HERE, "%s/rules.xml" % testnumber)
+        xpathsfn = "%s/xpaths.txt"            % testnumber
+
+        themedoc = etree.ElementTree(file=themefn, 
+                                     parser=etree.HTMLParser())
+        contentdoc = etree.ElementTree(file=contentfn, 
+                                       parser=etree.HTMLParser())
+        compilerfn = "../compiler.xsl"
         compilerdoc = etree.ElementTree(file=compilerfn)
         compiler = etree.XSLT(compilerdoc)
 
         # Make a compiled version
         params = {
-            'rulesuri': rulesfn,
+            'rulesuri': '"%s"' % rulesfn,
             }
         ct = compiler(themedoc, **params)
+
+        # If there were any messages from <xsl:message> in the
+        # compiler step, print them to the console
         for msg in compiler.error_log:
             print msg
-        print str(ct)
+
+        # Apply the compiled version, then test against desired output
+        processor = etree.XSLT(ct)
+        self.themed_content = processor(contentdoc)
+        xp = "/html/head/*[position()='1']/@id"
+        for xpath in open(xpathsfn).readlines():
+            # Read the XPaths from the file, skipping blank lines and
+            # comments
+            this_xpath = xpath.strip()
+            if not this_xpath or this_xpath[0] == '#':
+                continue
+            if not self.themed_content.xpath(this_xpath):
+                print "FAIL:", this_xpath, "is FALSE"
+
+        # Make a serialization
+        self.themed_string = str(self.themed_content)
 
 def main():
-    xdv = XDV("00")
+    test_num = sys.argv[1]
+    xdv = XDV(test_num)
+#    print xdv.themed_string
 
 
 if __name__ == "__main__":

From paul at codespeak.net  Wed Aug 27 14:28:55 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Wed, 27 Aug 2008 14:28:55 +0200 (CEST)
Subject: [z3-checkins] r57643 - in z3/deliverance/sandboxes/paul/xdv: . tests
Message-ID: <20080827122855.28B9F16A0B4@codespeak.net>

Author: paul
Date: Wed Aug 27 14:28:51 2008
New Revision: 57643

Removed:
   z3/deliverance/sandboxes/paul/xdv/tests/rules-00.xml
   z3/deliverance/sandboxes/paul/xdv/tests/theme-00.html
   z3/deliverance/sandboxes/paul/xdv/tests/theme-01.html
Modified:
   z3/deliverance/sandboxes/paul/xdv/CHANGES.txt
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
   z3/deliverance/sandboxes/paul/xdv/theme.html
Log:
Land support for <attcopy>

Modified: z3/deliverance/sandboxes/paul/xdv/CHANGES.txt
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/CHANGES.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/CHANGES.txt	Wed Aug 27 14:28:51 2008
@@ -1,3 +1,19 @@
+
+To Do
+-------------
+
+- Generate valid XHTML output despite the namespaces of the theme or
+  content nodes.  See NOTES.rst for more info.
+
+August 27
+---------
+
+- Support attribute merging.  For example, getting an attribute from
+  content <body> and adding or replacing the attributes on the theme
+  <body>.  See NOTES.rst for more.
+
+- Work on a test suite.
+
 August 6
 ------------------
 

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Wed Aug 27 14:28:51 2008
@@ -4,10 +4,12 @@
     xmlns:dyn="http://exslt.org/dynamic" xmlns:xml="http://www.w3.org/XML/1998/namespace"
     exclude-result-prefixes="dv dyn exsl xml" version="1.0">
     <xsl:output indent="yes" media-type="text/xml"/>
-    <xsl:param name="rulesuri">rules.xml</xsl:param>
+    <xsl:param name="rulesuri"
+        >/Users/paul/projects/deliverance/sandboxes/paul/xdv/tests/001/rules.xml</xsl:param>
     <xsl:param name="boilerplateurl">boilerplate.xsl</xsl:param>
     <!-- Multi-stage theme compiler -->
     <xsl:template match="/">
+
         <!-- Put unique xml:id values on all the theme html -->
         <xsl:variable name="themehtml-rtf">
             <xsl:apply-templates select="/html" mode="annotate-html"/>
@@ -133,6 +135,15 @@
                     <xsl:apply-templates select="@*" mode="apply-rules">
                         <xsl:with-param name="rules" select="$rules"/>
                     </xsl:apply-templates>
+                    <!-- At this point the theme node and its attributes 
+                    have been created.  To handle <attcopy>, we can 
+                    now copy over attributes, which will silently 
+                    overwrite the theme nodes' attributes. -->
+                    <xsl:for-each select="$matching-rule[name()='attcopy']">
+                        <xsl:element name="xsl:copy-of">
+                            <xsl:attribute name="select"><xsl:value-of select="@content"/></xsl:attribute>
+                        </xsl:element>
+                    </xsl:for-each>
                     <xsl:for-each select="$matching-rule[name()='prepend']">
                         <xsl:element name="xsl:copy-of">
                             <xsl:attribute name="select">

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/rules-00.xml
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/rules-00.xml	Wed Aug 27 14:28:51 2008
+++ (empty file)
@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<rules xmlns="http://openplans.org/deliverance">
-    <replace theme="/html/head/title" content="/html/head/title"/>
-    <copy theme="//td/h1[@id='pageheading']" content="/html/head/title/text()"/>
-    <copy theme="/html/body/h1" content="/html/head/title/text()"/>
-    <replace theme="//td/div[@id='pagecontent']" content="/html/body/*"/>
-    <append theme="/html/head" content="/html/head/link|/html/head/script|/html/head/meta"/>
-</rules>

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/theme-00.html
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/theme-00.html	Wed Aug 27 14:28:51 2008
+++ (empty file)
@@ -1,19 +0,0 @@
-<html>
-    <head>
-        <title>Second Theme Page Title</title>
-    </head>
-    <body>
-        <h1 class="somethemeclass">Theme Page Heading</h1>
-        <table border="0" height="300" cellpadding="20">
-            <tr>
-                <td width="200" bgcolor="gray" valign="top">
-                    <h2 align="center">Navigation</h2>
-                </td>
-                <td valign="top">
-                    <h1 id="pageheading">Theme Page Heading</h1>
-                    <div id="pagecontent">Theme content to replace.</div>
-                </td>
-            </tr>
-        </table>
-    </body>
-</html>

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/theme-01.html
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/theme-01.html	Wed Aug 27 14:28:51 2008
+++ (empty file)
@@ -1,5 +0,0 @@
-<div>
-    <head>
-        <title>Theme 01</title>
-    </head>
-</div>

Modified: z3/deliverance/sandboxes/paul/xdv/theme.html
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/theme.html	(original)
+++ z3/deliverance/sandboxes/paul/xdv/theme.html	Wed Aug 27 14:28:51 2008
@@ -1,9 +1,9 @@
 <html>
-    <head hoobly="boobly">
+    <head>
         <title>Second Theme Page Title</title>
         <link rel="stylesheet" href="foo.css"/>
     </head>
-    <body>
+    <body lang="EN-US">
         <h1 class="somethemeclass">Theme Page Heading</h1>
         <table border="0" height="300" cellpadding="20">
             <tr>

From ianb at codespeak.net  Thu Aug 28 20:14:57 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Thu, 28 Aug 2008 20:14:57 +0200 (CEST)
Subject: [z3-checkins] r57669 - z3/deliverance/trunk/deliverance
Message-ID: <20080828181457.7DBD3169E0E@codespeak.net>

Author: ianb
Date: Thu Aug 28 20:14:56 2008
New Revision: 57669

Modified:
   z3/deliverance/trunk/deliverance/htmlserialize.py
Log:
Make the serialization per-thread instead of a single XSLT transform for all threads

Modified: z3/deliverance/trunk/deliverance/htmlserialize.py
==============================================================================
--- z3/deliverance/trunk/deliverance/htmlserialize.py	(original)
+++ z3/deliverance/trunk/deliverance/htmlserialize.py	Thu Aug 28 20:14:56 2008
@@ -1,5 +1,6 @@
 from lxml import etree
 import re
+import threading
 
 html_xsl = """
 <xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
@@ -13,10 +14,22 @@
 # TODO: this should be xsl for real formatting 
 pretty_html_xsl = html_xsl
 
-html_transform = etree.XSLT(etree.XML(html_xsl))
-pretty_html_transform = etree.XSLT(etree.XML(pretty_html_xsl))
-
+pretty_html_transforms = threading.local()
+html_transforms = threading.local()
 
+def get_pretty_html_transform():
+    try:
+        return pretty_html_transforms.transform
+    except AttributeError:
+        t = pretty_html_transforms.transform = etree.XSLT(etree.XML(pretty_html_xsl))
+        return t
+
+def get_html_transform():
+    try:
+        return html_transforms.transform
+    except AttributeError:
+        t = html_transforms.transform = etree.XSLT(etree.XML(html_xsl))
+        return t
 
 def tostring(doc, pretty = False, doctype_pair=None):
     """
@@ -27,9 +40,9 @@
     """
 
     if pretty:
-        doc = str(pretty_html_transform(doc))
+        doc = str(get_pretty_html_transform()(doc))
     else:
-        doc = str(html_transform(doc))
+        doc = str(get_html_transform()(doc))
 
     if doctype_pair: 
         doc = """<!DOCTYPE html PUBLIC "%s" "%s">\n%s""" % (doctype_pair[0], doctype_pair[1], doc) 

From paul at codespeak.net  Thu Aug 28 21:38:29 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Thu, 28 Aug 2008 21:38:29 +0200 (CEST)
Subject: [z3-checkins] r57670 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080828193829.AE6D0169F88@codespeak.net>

Author: paul
Date: Thu Aug 28 21:38:27 2008
New Revision: 57670

Modified:
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
   z3/deliverance/sandboxes/paul/xdv/content.html
   z3/deliverance/sandboxes/paul/xdv/rules.xml
Log:
Fixed attcopy to handle both cases of multiple rules matching one node, or one match

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Thu Aug 28 21:38:27 2008
@@ -4,8 +4,7 @@
     xmlns:dyn="http://exslt.org/dynamic" xmlns:xml="http://www.w3.org/XML/1998/namespace"
     exclude-result-prefixes="dv dyn exsl xml" version="1.0">
     <xsl:output indent="yes" media-type="text/xml"/>
-    <xsl:param name="rulesuri"
-        >/Users/paul/projects/deliverance/sandboxes/paul/xdv/tests/001/rules.xml</xsl:param>
+    <xsl:param name="rulesuri">rules.xml</xsl:param>
     <xsl:param name="boilerplateurl">boilerplate.xsl</xsl:param>
     <!-- Multi-stage theme compiler -->
     <xsl:template match="/">
@@ -141,7 +140,9 @@
                     overwrite the theme nodes' attributes. -->
                     <xsl:for-each select="$matching-rule[name()='attcopy']">
                         <xsl:element name="xsl:copy-of">
-                            <xsl:attribute name="select"><xsl:value-of select="@content"/></xsl:attribute>
+                            <xsl:attribute name="select">
+                                <xsl:value-of select="@content"/>
+                            </xsl:attribute>
                         </xsl:element>
                     </xsl:for-each>
                     <xsl:for-each select="$matching-rule[name()='prepend']">
@@ -163,6 +164,25 @@
                     </xsl:for-each>
                 </xsl:copy>
             </xsl:when>
+            <xsl:when test="name($matching-rule)='attcopy'">
+                <!-- Merge attributes from the content node into the theme node -->
+                <xsl:copy>
+                    <!-- First copy the existing attributes from the 
+                        theme node. -->
+                    <xsl:apply-templates select="@*" mode="apply-rules">
+                        <xsl:with-param name="rules" select="$rules"/>
+                    </xsl:apply-templates>
+                    <!-- Now put in a rule to copy over nodes, possibly 
+                        overwriting, from the content node at runtime. -->
+                    <xsl:element name="xsl:copy-of">
+                        <xsl:attribute name="select">
+                            <xsl:value-of select="$matching-rule/@content"/>
+                        </xsl:attribute>
+                    </xsl:element>
+                </xsl:copy>
+            </xsl:when>
+            
+            
             <xsl:when test="name($matching-rule)='copy'">
                 <!-- Copy the node and its attributes, but clear 
                     the children and content.  Just have an 

Modified: z3/deliverance/sandboxes/paul/xdv/content.html
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/content.html	(original)
+++ z3/deliverance/sandboxes/paul/xdv/content.html	Thu Aug 28 21:38:27 2008
@@ -6,7 +6,7 @@
             /* */</script>
   </head>
   <body>
-    <div id="content">
+    <div id="content" class="foo">
       <div id="byline">To Drop, By <a href="foo">Some Person.</a>
       </div>
       <p>I have some content <em>in here</em> to merge.</p>

Modified: z3/deliverance/sandboxes/paul/xdv/rules.xml
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/rules.xml	(original)
+++ z3/deliverance/sandboxes/paul/xdv/rules.xml	Thu Aug 28 21:38:27 2008
@@ -8,4 +8,5 @@
     <copy theme="//td/div[@id='pagecontent']" content="/html/body/div[@id='content']/*"/>
     <append theme="/html/head" content="/html/head/link|/html/head/meta"/>
     <prepend theme="/html/head" content="/html/head/script"/>
+<!--    <attcopy theme="/html/body" content="/html/body/div[@id='content']/@class"/>-->
 </rules>

From z3-checkins at codespeak.net  Fri Aug 29 21:00:12 2008
From: z3-checkins at codespeak.net (z3-checkins at codespeak.net)
Date: Fri, 29 Aug 2008 21:00:12 +0200 (CEST)
Subject: [z3-checkins] Dear z3-checkins@codespeak.net 84% 0FF
Message-ID: <20080829100015.13076.qmail@exd127.neoplus.adsl.tpnet.pl>

Dear z3-checkins at codespeak.net, Best Price Only Today.
http://zch.dictionaryobject.com?hvh


From ldr at codespeak.net  Mon Sep  1 11:19:42 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Mon,  1 Sep 2008 11:19:42 +0200 (CEST)
Subject: [z3-checkins] r57711 - in z3/deliverance/sandboxes/paul/xdv/tests:
	. 001 002
Message-ID: <20080901091942.93937169E6C@codespeak.net>

Author: ldr
Date: Mon Sep  1 11:19:41 2008
New Revision: 57711

Modified:
   z3/deliverance/sandboxes/paul/xdv/tests/001/xpaths.txt
   z3/deliverance/sandboxes/paul/xdv/tests/002/xpaths.txt
   z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
Log:
fix tests, improve test runner

Modified: z3/deliverance/sandboxes/paul/xdv/tests/001/xpaths.txt
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/001/xpaths.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/001/xpaths.txt	Mon Sep  1 11:19:41 2008
@@ -1,3 +1,3 @@
 # XPaths that must be true to "validate" this test
-/html/head/title = 'A Deeper Look At DVNG'
+/html/head/title = 'A Deeper Look At xdv'
 not(/html/head/title/@id)

Modified: z3/deliverance/sandboxes/paul/xdv/tests/002/xpaths.txt
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/002/xpaths.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/002/xpaths.txt	Mon Sep  1 11:19:41 2008
@@ -1,3 +1,3 @@
 # XPaths that must be true to "validate" this test
-/html/head/title = 'A Deeper Look At DVNG'
+/html/head/title = 'A Deeper Look At xdv'
 /html/head/title/@id = 'something'

Modified: z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	Mon Sep  1 11:19:41 2008
@@ -5,12 +5,14 @@
 from lxml import etree
 import os
 import sys
+from StringIO import StringIO
 
 _HERE = os.path.abspath(os.path.dirname(__file__))
 
 class XDV:
 
     def __init__(self, testnumber):
+        self.errors = StringIO()
         testnumber = testnumber
         themefn = "%s/theme.html"           % testnumber
         contentfn = "%s/content.html"       % testnumber
@@ -34,7 +36,7 @@
         # If there were any messages from <xsl:message> in the
         # compiler step, print them to the console
         for msg in compiler.error_log:
-            print msg
+            print >>self.errors, msg
 
         # Apply the compiled version, then test against desired output
         processor = etree.XSLT(ct)
@@ -47,15 +49,32 @@
             if not this_xpath or this_xpath[0] == '#':
                 continue
             if not self.themed_content.xpath(this_xpath):
-                print "FAIL:", this_xpath, "is FALSE"
+                print >>self.errors, "FAIL:", this_xpath, "is FALSE"
 
         # Make a serialization
         self.themed_string = str(self.themed_content)
 
 def main():
-    test_num = sys.argv[1]
-    xdv = XDV(test_num)
-#    print xdv.themed_string
+    try:
+        test_num = sys.argv[1]
+    except IndexError:
+        test_num = 1
+        errors = 0
+        while True:
+            directory = '%03d' % test_num
+            if not os.path.isdir(directory):
+                break
+            xdv = XDV(directory)
+            result = xdv.errors.getvalue()
+            if result:
+                print 'Error running test %s...' % directory 
+                print result
+                errors += 1
+            test_num += 1
+        print "Ran %s tests with %s errors." % (test_num, errors)
+    else:
+        xdv = XDV(test_num)
+        print xdv.themed_string
 
 
 if __name__ == "__main__":

From ldr at codespeak.net  Mon Sep  1 13:08:23 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Mon,  1 Sep 2008 13:08:23 +0200 (CEST)
Subject: [z3-checkins] r57714 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080901110823.C762E169E30@codespeak.net>

Author: ldr
Date: Mon Sep  1 13:08:22 2008
New Revision: 57714

Modified:
   z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
Log:
set method=xml for valid xhtml

Modified: z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/boilerplate.xsl	Mon Sep  1 13:08:22 2008
@@ -2,7 +2,7 @@
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
     xmlns:dv="http://openplans.org/deliverance" xmlns:exsl="http://exslt.org/common"
     exclude-result-prefixes="exsl dv" version="1.0">
-  <xsl:output indent="yes" omit-xml-declaration="yes" media-type="text/html"
+  <xsl:output method="xml" indent="yes" omit-xml-declaration="yes" media-type="text/html"
       doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN"
       doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"/>
 

From ldr at codespeak.net  Tue Sep  2 12:25:45 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Tue,  2 Sep 2008 12:25:45 +0200 (CEST)
Subject: [z3-checkins] r57752 - z3/deliverance/sandboxes/paul/xdv/tests
Message-ID: <20080902102545.405B6169FA6@codespeak.net>

Author: ldr
Date: Tue Sep  2 12:25:44 2008
New Revision: 57752

Modified:
   z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
Log:
fix test runner counter

Modified: z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/test_nodes.py	Tue Sep  2 12:25:44 2008
@@ -63,6 +63,7 @@
         while True:
             directory = '%03d' % test_num
             if not os.path.isdir(directory):
+                test_num -= 1
                 break
             xdv = XDV(directory)
             result = xdv.errors.getvalue()

From ldr at codespeak.net  Tue Sep  2 12:27:36 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Tue,  2 Sep 2008 12:27:36 +0200 (CEST)
Subject: [z3-checkins] r57753 - z3/deliverance/sandboxes/paul/xdv/tests/008
Message-ID: <20080902102736.0894F16854F@codespeak.net>

Author: ldr
Date: Tue Sep  2 12:27:36 2008
New Revision: 57753

Added:
   z3/deliverance/sandboxes/paul/xdv/tests/008/
      - copied from r57706, z3/deliverance/sandboxes/paul/xdv/tests/007/
   z3/deliverance/sandboxes/paul/xdv/tests/008/README.txt
   z3/deliverance/sandboxes/paul/xdv/tests/008/content.html
      - copied, changed from r57752, z3/deliverance/sandboxes/paul/xdv/tests/007/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/008/rules.xml
      - copied, changed from r57752, z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/008/theme.html
      - copied, changed from r57752, z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/008/xpaths.txt
      - copied, changed from r57752, z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt
Log:
Add an example of the null xmlns problem. Note this is not run automatically, see tests/008/README.txt

Added: z3/deliverance/sandboxes/paul/xdv/tests/008/README.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/008/README.txt	Tue Sep  2 12:27:36 2008
@@ -0,0 +1,38 @@
+This test demonstrates the problem with xmlns="" and incidentally an issue with 
+the way test_nodes.py works vis-a-vis xsltproc / mod_transform / dv.xdvserver.
+
+xdv$ xsltproc --nonet --html --stringparam rulesuri tests/008/rules.xml compiler.xsl tests/008/theme.html | xsltproc --nonet --html - tests/008/content.html 
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <title xmlns="">A Deeper Look At xdv</title>
+  </head>
+  <body>
+    <div xmlns="" id="wrapper" class="foo">
+            <div id="content">
+                <p>boo bar baz</p>
+            </div>
+        </div>
+  </body>
+</html>
+
+
+xdv/tests$ python test_nodes.py 008
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <title>A Deeper Look At xdv</title>
+  </head>
+  <body>
+    <div id="wrapper" class="foo">
+            <div id="content">
+                <p>boo bar baz</p>
+            </div>
+        </div>
+  </body>
+</html>
+
+
+Note the double xmlns="http://www.w3.org/1999/xhtml" on the html node of the test_nodes.py version and no xmlns="" anywhere.

Copied: z3/deliverance/sandboxes/paul/xdv/tests/008/content.html (from r57752, z3/deliverance/sandboxes/paul/xdv/tests/007/content.html)
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/007/content.html	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/008/content.html	Tue Sep  2 12:27:36 2008
@@ -1,5 +1,5 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
+<html xmlns="http://www.w3.org/1999/xhtml">
     <head>
         <title>A Deeper Look At xdv</title>
     </head>

Copied: z3/deliverance/sandboxes/paul/xdv/tests/008/rules.xml (from r57752, z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml)
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/008/rules.xml	Tue Sep  2 12:27:36 2008
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <rules xmlns="http://openplans.org/deliverance">
-    <append theme="/html/body" content="/html/body/div[@id='wrapper']/*"/>
-    <attcopy theme="/html/body" content="/html/body/div[@id='wrapper']/@class"/>
+    <replace theme="/html/head/title" content="/html/head/title"/>
+    <copy theme="/html/body" content="//div[@id='wrapper']"/>
 </rules>

Copied: z3/deliverance/sandboxes/paul/xdv/tests/008/theme.html (from r57752, z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html)
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/008/theme.html	Tue Sep  2 12:27:36 2008
@@ -1,5 +1,5 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
+<html xmlns="http://www.w3.org/1999/xhtml">
     <head>
         <title id="first">Second Theme Page Title</title>
     </head>

Copied: z3/deliverance/sandboxes/paul/xdv/tests/008/xpaths.txt (from r57752, z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt)
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/tests/008/xpaths.txt	Tue Sep  2 12:27:36 2008
@@ -1,13 +0,0 @@
-#
-# Test <attcopy>.  First, use <append> to copy over some stuff 
-# intoy <body>.  Then, use the @class on a <div> in the content
-# to set the body/@class.
-/html/body/@class='foo'
-not(//div[@id='wrapper'])
-/html/body/*[1][name()='p']
-/html/body/div[@id='content']/p/text() = 'boo bar baz'
-
-#
-# XXXX Paul, come back and put <attcopy> into the non-special-case 
-# so it works when not matching on an overlapping <body> 
-# selection.
\ No newline at end of file

From paul at codespeak.net  Wed Sep  3 15:13:24 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Wed,  3 Sep 2008 15:13:24 +0200 (CEST)
Subject: [z3-checkins] r57793 - in z3/deliverance/sandboxes/paul/xdv: .
	tests/007
Message-ID: <20080903131324.CD712169FDD@codespeak.net>

Author: paul
Date: Wed Sep  3 15:13:22 2008
New Revision: 57793

Removed:
   z3/deliverance/sandboxes/paul/xdv/tests/007/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt
Modified:
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
   z3/deliverance/sandboxes/paul/xdv/rules.xml
Log:
Per Laurence, we do not need attcopy, but can instead just use <prepend>

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Wed Sep  3 15:13:22 2008
@@ -134,17 +134,6 @@
                     <xsl:apply-templates select="@*" mode="apply-rules">
                         <xsl:with-param name="rules" select="$rules"/>
                     </xsl:apply-templates>
-                    <!-- At this point the theme node and its attributes 
-                    have been created.  To handle <attcopy>, we can 
-                    now copy over attributes, which will silently 
-                    overwrite the theme nodes' attributes. -->
-                    <xsl:for-each select="$matching-rule[name()='attcopy']">
-                        <xsl:element name="xsl:copy-of">
-                            <xsl:attribute name="select">
-                                <xsl:value-of select="@content"/>
-                            </xsl:attribute>
-                        </xsl:element>
-                    </xsl:for-each>
                     <xsl:for-each select="$matching-rule[name()='prepend']">
                         <xsl:element name="xsl:copy-of">
                             <xsl:attribute name="select">
@@ -164,25 +153,6 @@
                     </xsl:for-each>
                 </xsl:copy>
             </xsl:when>
-            <xsl:when test="name($matching-rule)='attcopy'">
-                <!-- Merge attributes from the content node into the theme node -->
-                <xsl:copy>
-                    <!-- First copy the existing attributes from the 
-                        theme node. -->
-                    <xsl:apply-templates select="@*" mode="apply-rules">
-                        <xsl:with-param name="rules" select="$rules"/>
-                    </xsl:apply-templates>
-                    <!-- Now put in a rule to copy over nodes, possibly 
-                        overwriting, from the content node at runtime. -->
-                    <xsl:element name="xsl:copy-of">
-                        <xsl:attribute name="select">
-                            <xsl:value-of select="$matching-rule/@content"/>
-                        </xsl:attribute>
-                    </xsl:element>
-                </xsl:copy>
-            </xsl:when>
-            
-            
             <xsl:when test="name($matching-rule)='copy'">
                 <!-- Copy the node and its attributes, but clear 
                     the children and content.  Just have an 

Modified: z3/deliverance/sandboxes/paul/xdv/rules.xml
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/rules.xml	(original)
+++ z3/deliverance/sandboxes/paul/xdv/rules.xml	Wed Sep  3 15:13:22 2008
@@ -8,5 +8,4 @@
     <copy theme="//td/div[@id='pagecontent']" content="/html/body/div[@id='content']/*"/>
     <append theme="/html/head" content="/html/head/link|/html/head/meta"/>
     <prepend theme="/html/head" content="/html/head/script"/>
-<!--    <attcopy theme="/html/body" content="/html/body/div[@id='content']/@class"/>-->
 </rules>

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/007/content.html
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/007/content.html	Wed Sep  3 15:13:22 2008
+++ (empty file)
@@ -1,13 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-    <head>
-        <title>A Deeper Look At xdv</title>
-    </head>
-    <body>
-        <div id="wrapper" class="foo">
-            <div id="content">
-                <p>boo bar baz</p>
-            </div>
-        </div>
-    </body>
-</html>

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml	Wed Sep  3 15:13:22 2008
+++ (empty file)
@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<rules xmlns="http://openplans.org/deliverance">
-    <append theme="/html/body" content="/html/body/div[@id='wrapper']/*"/>
-    <attcopy theme="/html/body" content="/html/body/div[@id='wrapper']/@class"/>
-</rules>

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html	Wed Sep  3 15:13:22 2008
+++ (empty file)
@@ -1,9 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-    <head>
-        <title id="first">Second Theme Page Title</title>
-    </head>
-    <body>
-        <p>blah</p>
-    </body>
-</html>

Deleted: /z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt
==============================================================================
--- /z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt	Wed Sep  3 15:13:22 2008
+++ (empty file)
@@ -1,13 +0,0 @@
-#
-# Test <attcopy>.  First, use <append> to copy over some stuff 
-# intoy <body>.  Then, use the @class on a <div> in the content
-# to set the body/@class.
-/html/body/@class='foo'
-not(//div[@id='wrapper'])
-/html/body/*[1][name()='p']
-/html/body/div[@id='content']/p/text() = 'boo bar baz'
-
-#
-# XXXX Paul, come back and put <attcopy> into the non-special-case 
-# so it works when not matching on an overlapping <body> 
-# selection.
\ No newline at end of file

From paul at codespeak.net  Wed Sep  3 15:15:19 2008
From: paul at codespeak.net (paul at codespeak.net)
Date: Wed,  3 Sep 2008 15:15:19 +0200 (CEST)
Subject: [z3-checkins] r57794 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080903131519.1FF1516A006@codespeak.net>

Author: paul
Date: Wed Sep  3 15:15:17 2008
New Revision: 57794

Modified:
   z3/deliverance/sandboxes/paul/xdv/README.txt
Log:
Document usage of <prepend> for attribute merging

Modified: z3/deliverance/sandboxes/paul/xdv/README.txt
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/README.txt	(original)
+++ z3/deliverance/sandboxes/paul/xdv/README.txt	Wed Sep  3 15:15:17 2008
@@ -41,11 +41,12 @@
   $ cd xdv
   $ python tests/test_nodes.py
 
-To Do
--------
+Usage
+------------
 
-- Make sure attribute merging is handled
+- Attribute merging can be performed using <prepend>.  For example::
 
+  <prepend theme="/html/body" content="/html/body/@class" />
 
 Not Supported
 --------------------

From ldr at codespeak.net  Thu Sep 11 23:28:07 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Thu, 11 Sep 2008 23:28:07 +0200 (CEST)
Subject: [z3-checkins] r58073 - z3/deliverance/sandboxes/paul/xdv
Message-ID: <20080911212807.0BDD516A555@codespeak.net>

Author: ldr
Date: Thu Sep 11 23:28:06 2008
New Revision: 58073

Modified:
   z3/deliverance/sandboxes/paul/xdv/compiler.xsl
Log:
Copy comments to output for conditional IE comments etc

Modified: z3/deliverance/sandboxes/paul/xdv/compiler.xsl
==============================================================================
--- z3/deliverance/sandboxes/paul/xdv/compiler.xsl	(original)
+++ z3/deliverance/sandboxes/paul/xdv/compiler.xsl	Thu Sep 11 23:28:06 2008
@@ -109,6 +109,9 @@
             </xsl:for-each>
         </xsl:copy>
     </xsl:template>
+    <xsl:template match="comment()" mode="annotate-html">
+	<xsl:element name="xsl:comment"><xsl:value-of select="."/></xsl:element>
+    </xsl:template>
     <xsl:template match="node()|@*" mode="annotate-html">
         <xsl:copy>
             <xsl:attribute name="xml:id">

From ldr at codespeak.net  Thu Sep 11 23:41:09 2008
From: ldr at codespeak.net (ldr at codespeak.net)
Date: Thu, 11 Sep 2008 23:41:09 +0200 (CEST)
Subject: [z3-checkins] r58074 - z3/deliverance/sandboxes/paul/xdv/tests/007
Message-ID: <20080911214109.BA20C16A37F@codespeak.net>

Author: ldr
Date: Thu Sep 11 23:41:07 2008
New Revision: 58074

Added:
   z3/deliverance/sandboxes/paul/xdv/tests/007/content.html
   z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml
   z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html
   z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt
Log:
add test for comment copying

Added: z3/deliverance/sandboxes/paul/xdv/tests/007/content.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/007/content.html	Thu Sep 11 23:41:07 2008
@@ -0,0 +1,8 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title>A Deeper Look At xdv</title>
+        <link id="fourth" rel="stylesheet" href="fourth.css"/>
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/007/rules.xml	Thu Sep 11 23:41:07 2008
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rules xmlns="http://openplans.org/deliverance">
+    <append theme="/html/head" content="/html/head/link"/>
+</rules>

Added: z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/007/theme.html	Thu Sep 11 23:41:07 2008
@@ -0,0 +1,12 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+    <head>
+        <title id="first">Ninth Theme Page Title</title>
+        <script id="second" src="second.js" type="text/javascript"/>
+        <link id="third" rel="stylesheet" href="third.css"/>
+	<!--[if lte IE 6]>
+	    <script type="text/javascript" src="ie-only.js"></script>
+	<![endif]-->
+    </head>
+    <body/>
+</html>

Added: z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt
==============================================================================
--- (empty file)
+++ z3/deliverance/sandboxes/paul/xdv/tests/007/xpaths.txt	Thu Sep 11 23:41:07 2008
@@ -0,0 +1,3 @@
+# 
+# Comments. Check that comments make it through to the output.
+/html/head/comment()

From faassen at codespeak.net  Tue Sep 16 23:53:57 2008
From: faassen at codespeak.net (faassen at codespeak.net)
Date: Tue, 16 Sep 2008 23:53:57 +0200 (CEST)
Subject: [z3-checkins] r58186 - z3/hurry.file/trunk
Message-ID: <20080916215357.C7C66169EDD@codespeak.net>

Author: faassen
Date: Tue Sep 16 23:53:54 2008
New Revision: 58186

Added:
   z3/hurry.file/trunk/THIS_PACKAGE_MOVED.txt
Log:
Note that this package moved to svn.zope.org.


Added: z3/hurry.file/trunk/THIS_PACKAGE_MOVED.txt
==============================================================================
--- (empty file)
+++ z3/hurry.file/trunk/THIS_PACKAGE_MOVED.txt	Tue Sep 16 23:53:54 2008
@@ -0,0 +1 @@
+The further maintenance of this package will take place on svn.zope.org

From faassen at codespeak.net  Wed Sep 17 00:03:06 2008
From: faassen at codespeak.net (faassen at codespeak.net)
Date: Wed, 17 Sep 2008 00:03:06 +0200 (CEST)
Subject: [z3-checkins] r58188 - z3/hurry.query/trunk
Message-ID: <20080916220306.D84D5169EC8@codespeak.net>

Author: faassen
Date: Wed Sep 17 00:03:04 2008
New Revision: 58188

Added:
   z3/hurry.query/trunk/THIS_PACKAGE_MOVED.txt
Log:
Mark this package as moved over to svn.zope.org.



Added: z3/hurry.query/trunk/THIS_PACKAGE_MOVED.txt
==============================================================================
--- (empty file)
+++ z3/hurry.query/trunk/THIS_PACKAGE_MOVED.txt	Wed Sep 17 00:03:04 2008
@@ -0,0 +1 @@
+The further maintenance of this package will take place on svn.zope.org

From ianb at codespeak.net  Fri Sep 26 22:06:03 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Fri, 26 Sep 2008 22:06:03 +0200 (CEST)
Subject: [z3-checkins] r58447 - z3/deliverance/tags/0.2.1
Message-ID: <20080926200603.21DE916A158@codespeak.net>

Author: ianb
Date: Fri Sep 26 22:06:01 2008
New Revision: 58447

Added:
   z3/deliverance/tags/0.2.1/
      - copied from r58446, z3/deliverance/trunk/
Log:
Tagging 0.2.1 version

From ianb at codespeak.net  Fri Sep 26 22:06:14 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Fri, 26 Sep 2008 22:06:14 +0200 (CEST)
Subject: [z3-checkins] r58448 - z3/deliverance/trunk
Message-ID: <20080926200614.082CF16A15E@codespeak.net>

Author: ianb
Date: Fri Sep 26 22:06:13 2008
New Revision: 58448

Modified:
   z3/deliverance/trunk/setup.py
Log:
Auto-update of version strings

Modified: z3/deliverance/trunk/setup.py
==============================================================================
--- z3/deliverance/trunk/setup.py	(original)
+++ z3/deliverance/trunk/setup.py	Fri Sep 26 22:06:13 2008
@@ -1,4 +1,4 @@
-version = '0.2.1'
+version = '0.2.2'
 
 from setuptools import setup, find_packages
 

From ianb at codespeak.net  Fri Sep 26 22:08:04 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Fri, 26 Sep 2008 22:08:04 +0200 (CEST)
Subject: [z3-checkins] r58449 - z3/deliverance/branches/nextgen
Message-ID: <20080926200804.578FF16A159@codespeak.net>

Author: ianb
Date: Fri Sep 26 22:08:03 2008
New Revision: 58449

Removed:
   z3/deliverance/branches/nextgen/
Log:
Removing defunct branch

From ianb at codespeak.net  Mon Sep 29 22:39:44 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Mon, 29 Sep 2008 22:39:44 +0200 (CEST)
Subject: [z3-checkins] r58485 - in
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance: . tests
Message-ID: <20080929203944.8730116A038@codespeak.net>

Author: ianb
Date: Mon Sep 29 22:39:42 2008
New Revision: 58485

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt
Log:
Fix the logging of <match> elements

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/pagematch.py	Mon Sep 29 22:39:42 2008
@@ -86,7 +86,7 @@
             ('response-header', self.response_header),
             ('environ', self.environ)]:
             if value:
-                parts.append(u'%s="%s"' % (attr, html_quote(unicode(self.path))))
+                parts.append(u'%s="%s"' % (attr, html_quote(unicode(value))))
         if self.pyref:
             parts.append(unicode(self.pyref))
         parts.extend(self._uni_late_args())
@@ -149,11 +149,14 @@
         if self.response_header:
             result, headers = self.response_header(response_headers)
             if not result:
+                header_debug = []
+                for header in headers:
+                    header_debug.append('%s: %s' % (header, response_headers.get(header, '(empty)')))
                 ## FIXME: maybe distinguish <meta> headers and real headers?
                 log.debug(
                     debug_context, 'Skipping %s because the response headers %s '
                     'do not match response-header="%s"',
-                    debug_name, ', '.join(headers), self.response_header)
+                    debug_name, ', '.join(header_debug), self.response_header)
                 return False
         if self.environ:
             result, keys = self.environ(request.environ)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_pagematch.txt	Mon Sep 29 22:39:42 2008
@@ -44,7 +44,9 @@
     ['a']
     >>> m = make('<match response-header="Content-Type: contains: html" class="x" />')
     >>> match(m, Request.blank('/'), Response(content_type='text/plain'), [('content-type', 'text/plain')])
-    log: Skipping class="x" because the response headers Content-Type do not match response-header="Content-Type: contains:html"
+    log: Skipping class="x" because the response headers Content-Type: text/plain do not match response-header="Content-Type: contains:html"
     False
     >>> match(m, Request.blank('/'), Response(content_type='text/html'), [('content-type', 'text/html')])
     ['x']
+    >>> print m
+    <match class="x" response-header="Content-Type: contains:html" />

From ianb at codespeak.net  Mon Sep 29 22:51:08 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Mon, 29 Sep 2008 22:51:08 +0200 (CEST)
Subject: [z3-checkins] r58487 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests
Message-ID: <20080929205108.65722169F7D@codespeak.net>

Author: ianb
Date: Mon Sep 29 22:51:07 2008
New Revision: 58487

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_middleware.txt
Log:
just expanding the tests a bit (rechecking stuff that worked already)

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_middleware.txt
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_middleware.txt	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/test_middleware.txt	Mon Sep 29 22:51:07 2008
@@ -83,7 +83,7 @@
     ... <html><head></head><body>A simple page</body></html>''')
     >>> app['/magic'].headers['x-no-deliverate'] = '1'
     >>> app['/magic2'] = Response('''\
-    ... <html><head><meta http-equiv="X-No-Deliverate" content="1"></head><body>something</body></html>''')
+    ... <html><head><meta http-equiv="x-no-deliverate" content="1" /></head><body>something</body></html>''')
 
 Now to deliverate:
 
@@ -171,9 +171,9 @@
     <html><head></head><body>A simple page</body></html>
     >>> compare_request('/magic2')
     Original content:
-    <html><head><meta http-equiv="X-No-Deliverate" content="1"></head><body>something</body></html>
+    <html><head><meta http-equiv="x-no-deliverate" content="1" /></head><body>something</body></html>
     Themed content:
-    <html><head><meta http-equiv="X-No-Deliverate" content="1"></head><body>something</body></html>
+    <html><head><meta http-equiv="x-no-deliverate" content="1" /></head><body>something</body></html>
 
 
 Other rule formats

From ianb at codespeak.net  Tue Sep 30 07:21:05 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue, 30 Sep 2008 07:21:05 +0200 (CEST)
Subject: [z3-checkins] r58489 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance
Message-ID: <20080930052105.E2C73169EFE@codespeak.net>

Author: ianb
Date: Tue Sep 30 07:21:01 2008
New Revision: 58489

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/stringmatch.py
Log:
Deal better with dispatching and rewriting links when there is an exact:/path match (stripping the exact path, and rewriting links back to self).  Rewrite cookies properly (previously it would clobber repeated set-cookie headers).

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/proxy.py	Tue Sep 30 07:21:01 2008
@@ -480,8 +480,11 @@
         """
         Modify the response however the user wanted.
         """
+        # This might not have a trailing /:
+        exact_proxied_base = proxied_base
         if not proxied_base.endswith('/'):
             proxied_base += '/'
+        exact_orig_base = orig_base
         if not orig_base.endswith('/'):
             orig_base += '/'
         assert (proxied_url.startswith(proxied_base) 
@@ -506,6 +509,8 @@
         if self.rewrite_links:
             def link_repl_func(link):
                 """Rewrites a link to point to this proxy"""
+                if link == exact_proxied_base:
+                    return exact_orig_base
                 if not link.startswith(proxied_base):
                     # External link, so we don't rewrite it
                     return link
@@ -535,20 +540,22 @@
                 loc = urlparse.urljoin(proxied_url, response.location)
                 loc = link_repl_func(loc)
                 response.location = loc
-            if response.headers.get('set-cookie'):
-                cook = response.headers['set-cookie']
-                old_domain = urlparse.urlsplit(proxied_url)[1].lower()
-                new_domain = request.host.split(':', 1)[0].lower()
-                def rewrite_domain(match):
-                    """Rewrites domains to point to this proxy"""
-                    domain = match.group(2)
-                    if domain == old_domain:
-                        ## FIXME: doesn't catch wildcards and the sort
-                        return match.group(1) + new_domain + match.group(3)
-                    else:
-                        return match.group(0)
-                cook = self._cookie_domain_re.sub(rewrite_domain, cook)
-                response.headers['set-cookie'] = cook
+            if 'set-cookie' in response.headers:
+                cookies = response.headers.getall('set-cookie')
+                del response.headers['set-cookie']
+                for cook in cookies:
+                    old_domain = urlparse.urlsplit(proxied_url)[1].lower()
+                    new_domain = request.host.split(':', 1)[0].lower()
+                    def rewrite_domain(match):
+                        """Rewrites domains to point to this proxy"""
+                        domain = match.group(2)
+                        if domain == old_domain:
+                            ## FIXME: doesn't catch wildcards and the sort
+                            return match.group(1) + new_domain + match.group(3)
+                        else:
+                            return match.group(0)
+                    cook = self._cookie_domain_re.sub(rewrite_domain, cook)
+                    response.headers.add('set-cookie', cook)
         return response
 
 class ProxySettings(object):

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/stringmatch.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/stringmatch.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/stringmatch.py	Tue Sep 30 07:21:01 2008
@@ -180,6 +180,9 @@
     def __call__(self, s):
         return s == self.pattern
 
+    def strip_prefix(self):
+        return self.pattern
+
 _add_matcher(ExactMatcher)
         
 class ExactInsensitiveMatcher(Matcher):

From ianb at codespeak.net  Tue Sep 30 07:21:40 2008
From: ianb at codespeak.net (ianb at codespeak.net)
Date: Tue, 30 Sep 2008 07:21:40 +0200 (CEST)
Subject: [z3-checkins] r58490 -
	z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests
Message-ID: <20080930052140.44E97169EFE@codespeak.net>

Author: ianb
Date: Tue Sep 30 07:21:39 2008
New Revision: 58490

Modified:
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml
   z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/fixup_openplans.py
Log:
update the example proxy stuff for the new https layout

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/example-proxy-rule.xml	Tue Sep 30 07:21:39 2008
@@ -21,13 +21,13 @@
   </proxy>
 
   <proxy path="/people" class="default openplans">
-    <dest href="http://www.openplans.org/people/" />
+    <dest href="https://www.openplans.org/people/" />
     <response rewrite-links="1" />
     <response pyref="file:$here/fixup_openplans.py:fixup_openplans_response" />
   </proxy>
 
   <proxy path="exact:/login" class="default openplans">
-    <dest href="https://www.openplans.org/login/" />
+    <dest href="https://www.openplans.org/login" />
     <response rewrite-links="1" />
     <response pyref="file:$here/fixup_openplans.py:fixup_openplans_response" />
   </proxy>    

Modified: z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/fixup_openplans.py
==============================================================================
--- z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/fixup_openplans.py	(original)
+++ z3/deliverance/sandbox/ianb/deliverance/trunk/deliverance/tests/fixup_openplans.py	Tue Sep 30 07:21:39 2008
@@ -1,6 +1,12 @@
+import urlparse
+import posixpath
+
 def fixup_openplans_response(request, response, orig_base, proxied_base, proxied_url, log):
-    user_base = 'http://www.openplans.org/people'
+    user_base = 'https://www.openplans.org/people'
     if response.location and response.location.startswith(user_base + '/'):
+        print 'incoming: %s (%r)' % (response.location, request.application_url)
         user_rest = response.location[len(user_base):]
-        response.location = '%s/people%s' % (request.application_url, user_rest)
+        location = posixpath.join('/people', user_rest.lstrip('/'))
+        response.location = urlparse.urljoin(request.application_url, location)
+        print 'outgoing: %s' % response.location, [location, user_rest]