From ejucovy at codespeak.net Fri Oct 1 16:40:44 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Fri, 1 Oct 2010 16:40:44 +0200 (CEST) Subject: [z3-checkins] r77528 - z3/deliverance/trunk/deliverance Message-ID: <20101001144044.2320D282BDC@codespeak.net> Author: ejucovy Date: Fri Oct 1 16:40:42 2010 New Revision: 77528 Modified: z3/deliverance/trunk/deliverance/proxycommand.py Log: fix #51: deliverance-proxy --version reporting wrong version number Modified: z3/deliverance/trunk/deliverance/proxycommand.py ============================================================================== --- z3/deliverance/trunk/deliverance/proxycommand.py (original) +++ z3/deliverance/trunk/deliverance/proxycommand.py Fri Oct 1 16:40:42 2010 @@ -4,6 +4,7 @@ import os import optparse from paste.httpserver import serve +from pkg_resources import get_distribution from deliverance.proxy import ProxySet from deliverance.proxy import ProxySettings @@ -11,10 +12,11 @@ Starts up a proxy server using the given rule file. """ +version = get_distribution("deliverance").version + parser = optparse.OptionParser( usage='%prog [OPTIONS] RULE.xml', - ## FIXME: get from pkg_resources: - version='0.1', + version=version, description=description, ) ## FIXME: these should be handled by the settings (or just picked up from devauth): From ejucovy at codespeak.net Sat Oct 2 14:05:13 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Sat, 2 Oct 2010 14:05:13 +0200 (CEST) Subject: [z3-checkins] r77547 - in z3/deliverance/trunk/deliverance: . tests Message-ID: <20101002120513.E76A6282BAD@codespeak.net> Author: ejucovy Date: Sat Oct 2 14:05:12 2010 New Revision: 77547 Added: z3/deliverance/trunk/deliverance/tests/test_doctests.py Modified: z3/deliverance/trunk/deliverance/middleware.py Log: fix #53: a lame way to make nose find the doctests. nose has a --with-doctest option, but when trying to use it i am running into the problem described here: http://www.protocolostomy.com/2010/04/30/nose-hates-me/ -- seems to have something to do with optparse, i can't figure it out Modified: z3/deliverance/trunk/deliverance/middleware.py ============================================================================== --- z3/deliverance/trunk/deliverance/middleware.py (original) +++ z3/deliverance/trunk/deliverance/middleware.py Sat Oct 2 14:05:12 2010 @@ -730,8 +730,3 @@ execute_pyref=execute_pyref) return app - -if __name__ == '__main__': - import doctest - doctest.testfile('tests/test_middleware.txt') - Added: z3/deliverance/trunk/deliverance/tests/test_doctests.py ============================================================================== --- (empty file) +++ z3/deliverance/trunk/deliverance/tests/test_doctests.py Sat Oct 2 14:05:12 2010 @@ -0,0 +1,19 @@ + +import doctest +from nose.tools import assert_equals + +def test_middleware(): + failures, tests = doctest.testfile('test_middleware.txt') + assert_equals(failures, 0) + +def test_pagematch(): + failures, tests = doctest.testfile('test_pagematch.txt') + assert_equals(failures, 0) + +def test_selection(): + failures, tests = doctest.testfile('test_selection.txt') + assert_equals(failures, 0) + +def test_stringmatch(): + failures, tests = doctest.testfile('test_stringmatch.txt') + assert_equals(failures, 0) From ejucovy at codespeak.net Sat Oct 2 15:06:10 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Sat, 2 Oct 2010 15:06:10 +0200 (CEST) Subject: [z3-checkins] r77548 - z3/deliverance/trunk/deliverance Message-ID: <20101002130610.17A7A282BAD@codespeak.net> Author: ejucovy Date: Sat Oct 2 15:06:08 2010 New Revision: 77548 Modified: z3/deliverance/trunk/deliverance/middleware.py Log: RulesetGetter doesn't exist anymore; remove it from __all__ Modified: z3/deliverance/trunk/deliverance/middleware.py ============================================================================== --- z3/deliverance/trunk/deliverance/middleware.py (original) +++ z3/deliverance/trunk/deliverance/middleware.py Sat Oct 2 15:06:08 2010 @@ -27,8 +27,10 @@ from deliverance.ruleset import RuleSet -__all__ = ['DeliveranceMiddleware', 'RulesetGetter', 'SubrequestRuleGetter', - 'FileRuleGetter', 'make_deliverance_middleware' ] +__all__ = ['DeliveranceMiddleware', + 'SubrequestRuleGetter', + 'FileRuleGetter', + 'make_deliverance_middleware' ] class DeliveranceMiddleware(object): From ejucovy at codespeak.net Sat Oct 2 15:07:03 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Sat, 2 Oct 2010 15:07:03 +0200 (CEST) Subject: [z3-checkins] r77549 - z3/deliverance/trunk/deliverance/tests Message-ID: <20101002130703.3E943282BAD@codespeak.net> Author: ejucovy Date: Sat Oct 2 15:07:01 2010 New Revision: 77549 Modified: z3/deliverance/trunk/deliverance/tests/example.py Log: fix #52: broken import in tests/example.py; also, extend it to allow port/host declaration from command-line arguments Modified: z3/deliverance/trunk/deliverance/tests/example.py ============================================================================== --- z3/deliverance/trunk/deliverance/tests/example.py (original) +++ z3/deliverance/trunk/deliverance/tests/example.py Sat Oct 2 15:07:01 2010 @@ -3,18 +3,27 @@ """ import os +import sys from paste.urlparser import StaticURLParser from paste.httpserver import serve from weberror.evalexception import EvalException -from deliverance.middleware import DeliveranceMiddleware, RulesetGetter +from deliverance.middleware import DeliveranceMiddleware, FileRuleGetter from deliverance.security import SecurityContext base_path = os.path.join(os.path.dirname(__file__), 'example-files') app = StaticURLParser(base_path) -deliv_app = DeliveranceMiddleware(app, RulesetGetter('/rules.xml')) +rules_file = os.path.join(base_path, 'rules.xml') +deliv_app = DeliveranceMiddleware(app, FileRuleGetter(rules_file)) full_app = SecurityContext.middleware(deliv_app, execute_pyref=True, display_logging=True, display_local_files=True, force_dev_auth=True) if __name__ == '__main__': - print 'See http://127.0.0.1:8080/?deliv_log for the page with log messages' - serve(EvalException(full_app)) + try: + port = sys.argv[1] + except IndexError: + port = '8080' + host = '127.0.0.1' + if ':' in port: + host, port = port.split(':') + print 'See http://%s:%s/?deliv_log for the page with log messages' % (host, port) + serve(EvalException(full_app), port=port, host=host) From ldr at codespeak.net Sat Oct 9 13:51:52 2010 From: ldr at codespeak.net (ldr at codespeak.net) Date: Sat, 9 Oct 2010 13:51:52 +0200 (CEST) Subject: [z3-checkins] r77743 - z3/xdv/trunk/lib/xdv/tests/copy-xsl Message-ID: <20101009115152.C9505282BDD@codespeak.net> Author: ldr Date: Sat Oct 9 13:51:50 2010 New Revision: 77743 Added: z3/xdv/trunk/lib/xdv/tests/copy-xsl/ z3/xdv/trunk/lib/xdv/tests/copy-xsl/content.html z3/xdv/trunk/lib/xdv/tests/copy-xsl/output.html z3/xdv/trunk/lib/xdv/tests/copy-xsl/rules.xml z3/xdv/trunk/lib/xdv/tests/copy-xsl/theme.html Log: demonstration of str:replace Added: z3/xdv/trunk/lib/xdv/tests/copy-xsl/content.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/copy-xsl/content.html Sat Oct 9 13:51:50 2010 @@ -0,0 +1,6 @@ + + + Site – Page title + + + Added: z3/xdv/trunk/lib/xdv/tests/copy-xsl/output.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/copy-xsl/output.html Sat Oct 9 13:51:50 2010 @@ -0,0 +1,8 @@ + + + + + Site : Page title + + + Added: z3/xdv/trunk/lib/xdv/tests/copy-xsl/rules.xml ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/copy-xsl/rules.xml Sat Oct 9 13:51:50 2010 @@ -0,0 +1,17 @@ + + + + + + + + + + + Added: z3/xdv/trunk/lib/xdv/tests/copy-xsl/theme.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/copy-xsl/theme.html Sat Oct 9 13:51:50 2010 @@ -0,0 +1,7 @@ + + + + Title + + + From ldr at codespeak.net Tue Oct 19 16:39:00 2010 From: ldr at codespeak.net (ldr at codespeak.net) Date: Tue, 19 Oct 2010 16:39:00 +0200 (CEST) Subject: [z3-checkins] r78084 - z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src Message-ID: <20101019143900.13CFC282B90@codespeak.net> Author: ldr Date: Tue Oct 19 16:38:58 2010 New Revision: 78084 Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/content.html z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/output.html z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/rules.xml z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/theme.html Log: add an inline xsl example for manipulating attributes Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/content.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/content.html Tue Oct 19 16:38:58 2010 @@ -0,0 +1,5 @@ +
+ Text + + Some more text +
Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/output.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/output.html Tue Oct 19 16:38:58 2010 @@ -0,0 +1,10 @@ + + + +
+ Text + + Some more text +
+ + Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/rules.xml ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/rules.xml Tue Oct 19 16:38:58 2010 @@ -0,0 +1,12 @@ + + + + + + + /@@/images/image/thumb + + + Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/theme.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/theme.html Tue Oct 19 16:38:58 2010 @@ -0,0 +1,3 @@ +
+
Content
+
From ldr at codespeak.net Tue Oct 19 16:59:04 2010 From: ldr at codespeak.net (ldr at codespeak.net) Date: Tue, 19 Oct 2010 16:59:04 +0200 (CEST) Subject: [z3-checkins] r78087 - z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text Message-ID: <20101019145904.3EC01282B90@codespeak.net> Author: ldr Date: Tue Oct 19 16:59:02 2010 New Revision: 78087 Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/content.html z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/output.html z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/rules.xml z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/theme.html Log: add an inline xsl example for modifying text Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/content.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/content.html Tue Oct 19 16:59:02 2010 @@ -0,0 +1,6 @@ +
+

Some heading text

+ Text +

Some more heading text

+ Text +
Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/output.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/output.html Tue Oct 19 16:59:02 2010 @@ -0,0 +1,11 @@ + + + +
+

Some heading text - Some extra text

+ Text +

Some more heading text - Some extra text

+ Text +
+ + Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/rules.xml ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/rules.xml Tue Oct 19 16:59:02 2010 @@ -0,0 +1,10 @@ + + + + + + - Some extra text + + Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/theme.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-text/theme.html Tue Oct 19 16:59:02 2010 @@ -0,0 +1,3 @@ +
+
Content
+
From ldr at codespeak.net Tue Oct 19 17:01:10 2010 From: ldr at codespeak.net (ldr at codespeak.net) Date: Tue, 19 Oct 2010 17:01:10 +0200 (CEST) Subject: [z3-checkins] r78088 - in z3/xdv/trunk/lib/xdv/tests: inline-xsl-example-img-src inline-xsl-example-modify-attribute Message-ID: <20101019150110.42B34282B90@codespeak.net> Author: ldr Date: Tue Oct 19 17:01:08 2010 New Revision: 78088 Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-modify-attribute/ - copied from r78084, z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/ Removed: z3/xdv/trunk/lib/xdv/tests/inline-xsl-example-img-src/ Log: rename test From ldr at codespeak.net Wed Oct 20 14:09:21 2010 From: ldr at codespeak.net (ldr at codespeak.net) Date: Wed, 20 Oct 2010 14:09:21 +0200 (CEST) Subject: [z3-checkins] r78128 - z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates Message-ID: <20101020120921.8459B282BE3@codespeak.net> Author: ldr Date: Wed Oct 20 14:09:19 2010 New Revision: 78128 Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/ z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/content.html z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/output.html z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/rules.xml z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/theme.html Log: Show that xsl:apply-templates works in inline xsl Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/content.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/content.html Wed Oct 20 14:09:19 2010 @@ -0,0 +1,6 @@ +
+

Some heading text

+ Text +

Some more heading text

+ Text +
Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/output.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/output.html Wed Oct 20 14:09:19 2010 @@ -0,0 +1,11 @@ + + + +
+

Some heading text - Some extra text

+ Text +

Some more heading text - Some extra text

+ Text +
+ + Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/rules.xml ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/rules.xml Wed Oct 20 14:09:19 2010 @@ -0,0 +1,12 @@ + + + + + + + + - Some extra text + + Added: z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/theme.html ============================================================================== --- (empty file) +++ z3/xdv/trunk/lib/xdv/tests/inline-xsl-apply-templates/theme.html Wed Oct 20 14:09:19 2010 @@ -0,0 +1,3 @@ +
+
Content
+
From z3-checkins at codespeak.net Mon Oct 25 03:27:33 2010 From: z3-checkins at codespeak.net (z3-checkins at codespeak.net) Date: Mon, 25 Oct 2010 03:27:33 +0200 (CEST) Subject: z3-checkins@codespeak.net VIAGRA ® Official Site ID6979071 Message-ID: <20101025012733.123BB282BF5@codespeak.net> An HTML attachment was scrubbed... URL: http://codespeak.net/pipermail/z3-checkins/attachments/20101025/6bf6cfd4/attachment.htm From ejucovy at codespeak.net Thu Oct 28 16:53:43 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Thu, 28 Oct 2010 16:53:43 +0200 (CEST) Subject: [z3-checkins] r78405 - z3/deliverance/branches/egj-wsgi-sandbox Message-ID: <20101028145343.56358282C1E@codespeak.net> Author: ejucovy Date: Thu Oct 28 16:53:42 2010 New Revision: 78405 Added: z3/deliverance/branches/egj-wsgi-sandbox/ (props changed) - copied from r78404, z3/deliverance/trunk/ Log: sandbox branch From ejucovy at codespeak.net Thu Oct 28 16:54:45 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Thu, 28 Oct 2010 16:54:45 +0200 (CEST) Subject: [z3-checkins] r78406 - z3/deliverance/branches/egj-wsgi-sandbox/deliverance Message-ID: <20101028145445.6557F282C1E@codespeak.net> Author: ejucovy Date: Thu Oct 28 16:54:43 2010 New Revision: 78406 Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py Log: add support for a directive that proxies a request to an in-process wsgi application configured by a paste ini file Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py ============================================================================== --- z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py (original) +++ z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py Thu Oct 28 16:54:43 2010 @@ -15,6 +15,7 @@ from wsgiproxy.exactproxy import proxy_exact_request from tempita import html_quote from paste.fileapp import FileApp +from paste.deploy import loadwsgi from lxml.etree import tostring as xml_tostring, Comment, parse from lxml.html import document_fromstring, tostring from deliverance.exceptions import DeliveranceSyntaxError, AbortProxy @@ -126,7 +127,8 @@ def __init__(self, match, dest, request_modifications, response_modifications, strip_script_name=True, keep_host=False, - source_location=None, classes=None, editable=False): + source_location=None, classes=None, editable=False, + wsgi=None): self.match = match self.match.proxy = self self.dest = dest @@ -137,6 +139,11 @@ self.source_location = source_location self.classes = classes self.editable = editable + self.wsgi = wsgi + + def get_endpoint(self): + ## FIXME: should we assert that one of these is not None? I think so + return self.dest or self.wsgi def log_description(self, log=None): """The debugging description for use in log display""" @@ -154,7 +161,7 @@ if self.editable: parts.append('editable="1"') parts.append('>
\n') - parts.append(' ' + self.dest.log_description(log)) + parts.append(' ' + self.get_endpoint().log_description(log)) parts.append('
\n') if self.request_modifications: if len(self.request_modifications) > 1: @@ -177,11 +184,16 @@ assert el.tag == 'proxy' match = ProxyMatch.parse_xml(el, source_location) dest = None + wsgi = None request_modifications = [] response_modifications = [] strip_script_name = True keep_host = False editable = asbool(el.get('editable')) + rewriting_links = None + + ## FIXME: this inline validation is a bit brittle because it is + ## order-dependent, but validation errors generally aren't for child in el: if child.tag == 'dest': if dest is not None: @@ -189,7 +201,30 @@ "You cannot have more than one tag (second tag: %s)" % xml_tostring(child), element=child, source_location=source_location) + if wsgi is not None: + raise DeliveranceSyntaxError( + "You cannot have both a tag and a tag (second tag: %s)" + % xml_tostring(child), + element=child, source_location=source_location) dest = ProxyDest.parse_xml(child, source_location) + elif child.tag == 'wsgi': + if wsgi is not None: + raise DeliveranceSyntaxError( + "You cannot have more than one tag (second tag: %s)" + % xml_tostring(child), + element=child, source_location=source_location) + if dest is not None: + raise DeliveranceSyntaxError( + "You cannot have both a tag and a tag (second tag: %s)" + % xml_tostring(child), + element=child, source_location=source_location) + if rewriting_links is not None: + raise DeliveranceSyntaxError( + "You cannot use ```` in a proxy with a ```` tag", + element=child, source_location=source_location) + + wsgi = ProxyWsgi.parse_xml(child, source_location) + elif child.tag == 'transform': if child.get('strip-script-name'): strip_script_name = asbool(child.get('strip-script-name')) @@ -200,8 +235,17 @@ request_modifications.append( ProxyRequestModification.parse_xml(child, source_location)) elif child.tag == 'response': - response_modifications.append( - ProxyResponseModification.parse_xml(child, source_location)) + mod = ProxyResponseModification.parse_xml(child, source_location) + if mod.rewrite_links == True: + rewriting_links = mod + + if wsgi is not None: + raise DeliveranceSyntaxError( + "You cannot use ```` in a proxy with a ```` tag", + element=child, source_location=source_location) + + response_modifications.append(mod) + elif child.tag is Comment: continue else: @@ -228,7 +272,7 @@ inst = cls(match, dest, request_modifications, response_modifications, strip_script_name=strip_script_name, keep_host=keep_host, source_location=source_location, classes=classes, - editable=editable) + editable=editable, wsgi=wsgi) match.proxy = inst return inst @@ -255,15 +299,29 @@ log = request.environ['deliverance.log'] for modifier in self.request_modifications: request = modifier.modify_request(request, log) - if self.dest.next: + if self.dest and self.dest.next: raise AbortProxy - dest = self.dest(request, log) - log.debug(self, ' matched; forwarding request to %s' % dest) + + dest, wsgiapp = None, None + if self.dest: + dest = self.dest(request, log) + log.debug(self, ' matched; forwarding request to %s' % dest) + else: + wsgi_app = self.wsgi(request, log) + log.debug(self, ' matched; forwarding request to %s' % wsgi_app) + if self.classes: log.debug(self, 'Adding class="%s" to page' % ' '.join(self.classes)) existing_classes = request.environ.setdefault('deliverance.page_classes', []) existing_classes.extend(self.classes) - response, orig_base, proxied_base, proxied_url = self.proxy_to_dest(request, dest) + + if dest is not None: + response, orig_base, proxied_base, proxied_url = self.proxy_to_dest(request, dest) + else: + ## FIXME: proxied_base and proxied_url don't really have a meaning here, + ## but the modifier signature expects them + response, orig_base, proxied_base, proxied_url = self.proxy_to_wsgi(request, wsgi_app) + for modifier in self.response_modifications: response = modifier.modify_response(request, response, orig_base, proxied_base, proxied_url, log) @@ -318,6 +376,16 @@ return proxy_req + def proxy_to_wsgi(self, request, wsgi_app): + """ Forward a request to an inner wsgi app """ + orig_base = url_normalize(request.application_url) + + ## FIXME: should this be request.copy()? + proxy_req = Request(request.environ.copy()) + resp = proxy_req.get_response(wsgi_app) + + return resp, orig_base, None, None + def proxy_to_dest(self, request, dest): """Do the actual proxying, without applying any transformations""" # Not using request.copy because I don't want to copy wsgi.input: @@ -449,6 +517,41 @@ return self.path.strip_prefix() return None +class ProxyWsgi(object): + """ Represents the ```` element """ + + def __init__(self, app=None, source_location=None): + if not app.startswith("config:") and not app.startswith("egg:"): + app = "config:%s" % app + self.app_string = app + self.app = loadwsgi.loadapp(app) + self.source_location = source_location + + @classmethod + def parse_xml(cls, el, source_location): + """ Parse an instance from an etree XML element """ + app = el.get('app') + if not app: + raise DeliveranceSyntaxError( + "A ```` tag must have an ``app`` attribute", + element=el, source_location=source_location) + return cls(app, source_location=source_location) + + def __call__(self, request, log): + """ + Determine the destination given the request, + returning a WSGI callable + """ + return self.app + + def log_description(self, log=None): + """The text to show when this is the context of a log message""" + parts = ['<wsgi'] + if self.app_string: + parts.append('app="%s"' % html_quote(self.app_string)) + parts.append('/>') + return ' '.join(parts) + class ProxyDest(object): """Represents the ```` element""" @@ -592,21 +695,23 @@ """ Modify the response however the user wanted. """ - # This might not have a trailing /: - exact_proxied_base = proxied_base - if not proxied_base.endswith('/'): - proxied_base += '/' - exact_orig_base = orig_base - if not orig_base.endswith('/'): - orig_base += '/' - assert (proxied_url.startswith(proxied_base) - or proxied_url.split('?', 1)[0] == proxied_base[:-1]), ( - "Unexpected proxied_url %r, doesn't start with proxied_base %r" - % (proxied_url, proxied_base)) - assert (request.url.startswith(orig_base) - or request.url.split('?', 1)[0] == orig_base[:-1]), ( - "Unexpected request.url %r, doesn't start with orig_base %r" - % (request.url, orig_base)) + if proxied_base is not None and proxied_url is not None: + # This might not have a trailing /: + exact_proxied_base = proxied_base + if not proxied_base.endswith('/'): + proxied_base += '/' + exact_orig_base = orig_base + if not orig_base.endswith('/'): + orig_base += '/' + assert (proxied_url.startswith(proxied_base) + or proxied_url.split('?', 1)[0] == proxied_base[:-1]), ( + "Unexpected proxied_url %r, doesn't start with proxied_base %r" + % (proxied_url, proxied_base)) + assert (request.url.startswith(orig_base) + or request.url.split('?', 1)[0] == orig_base[:-1]), ( + "Unexpected request.url %r, doesn't start with orig_base %r" + % (request.url, orig_base)) + if self.pyref: if not execute_pyref(request): log.error( @@ -618,6 +723,7 @@ response = result if self.header: response.headers[self.header] = self.content + if self.rewrite_links: def link_repl_func(link): """Rewrites a link to point to this proxy""" From ejucovy at codespeak.net Thu Oct 28 16:56:34 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Thu, 28 Oct 2010 16:56:34 +0200 (CEST) Subject: [z3-checkins] r78407 - z3/deliverance/branches/egj-wsgi-sandbox/deliverance Message-ID: <20101028145634.65240282C1E@codespeak.net> Author: ejucovy Date: Thu Oct 28 16:56:33 2010 New Revision: 78407 Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/middleware.py Log: #62 -- * Don't theme the response if X-Deliverance-Notheme is present in the request headers * After retrieving the response, check again whether we should return it without theming -- in case a directive added the X-Deliverance-Notheme header, for example Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/middleware.py ============================================================================== --- z3/deliverance/branches/egj-wsgi-sandbox/deliverance/middleware.py (original) +++ z3/deliverance/branches/egj-wsgi-sandbox/deliverance/middleware.py Thu Oct 28 16:56:33 2010 @@ -69,6 +69,8 @@ def notheme_request(self, req): if 'deliv_notheme' in req.GET: return True + if req.headers.get("X-Deliverance-Notheme", False): + return True def __call__(self, environ, start_response): req = Request(environ) @@ -102,6 +104,11 @@ else: log.debug(self, 'Not doing clientside theming because jsEnabled cookie not set') resp = req.get_response(self.app) + + if self.notheme_request(req): + log.debug(self, "Not theming the request") + return resp(environ, start_response) + ## FIXME: also XHTML? if resp.content_type != 'text/html': ## FIXME: remove from known_html? From ejucovy at codespeak.net Thu Oct 28 17:37:51 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Thu, 28 Oct 2010 17:37:51 +0200 (CEST) Subject: [z3-checkins] r78413 - z3/deliverance/branches/egj-wsgi-sandbox/deliverance Message-ID: <20101028153751.3D39A36E076@codespeak.net> Author: ejucovy Date: Thu Oct 28 17:37:49 2010 New Revision: 78413 Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py Log: #63: don't let dest-next=1 proxies consume parts of path info Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py ============================================================================== --- z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py (original) +++ z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py Thu Oct 28 17:37:49 2010 @@ -282,6 +282,8 @@ This also applies all the request and response transformations. """ request = Request(environ) + original_script_name = request.script_name + original_path_info = request.path_info prefix = self.match.strip_prefix() if prefix: if prefix.endswith('/'): @@ -300,6 +302,8 @@ for modifier in self.request_modifications: request = modifier.modify_request(request, log) if self.dest and self.dest.next: + request.script_name = original_script_name + request.path_info = original_path_info raise AbortProxy dest, wsgiapp = None, None @@ -382,6 +386,7 @@ ## FIXME: should this be request.copy()? proxy_req = Request(request.environ.copy()) + resp = proxy_req.get_response(wsgi_app) return resp, orig_base, None, None From ejucovy at codespeak.net Thu Oct 28 17:59:11 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Thu, 28 Oct 2010 17:59:11 +0200 (CEST) Subject: [z3-checkins] r78416 - z3/deliverance/branches/egj-wsgi-sandbox/deliverance Message-ID: <20101028155911.7568136E076@codespeak.net> Author: ejucovy Date: Thu Oct 28 17:59:09 2010 New Revision: 78416 Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py Log: Add more logging Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py ============================================================================== --- z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py (original) +++ z3/deliverance/branches/egj-wsgi-sandbox/deliverance/proxy.py Thu Oct 28 17:59:09 2010 @@ -304,7 +304,8 @@ if self.dest and self.dest.next: request.script_name = original_script_name request.path_info = original_path_info - raise AbortProxy + log.debug(self, "Aborting proxy, ") + raise AbortProxy, '' dest, wsgiapp = None, None if self.dest: @@ -658,9 +659,16 @@ elif isinstance(result, Request): request = result if self.header: + log.info(self, "Setting request header %s = %s" % + (self.header, self.content)) request.headers[self.header] = self.content return request + def log_description(self, log=None): + if self.pyref: + return '<request pyref="%s" />' % self.pyref + return '<request header="%s" content="%s" />' % (self.header, self.content) + class ProxyResponseModification(object): """Represents the ```` element in ````""" From ejucovy at codespeak.net Thu Oct 28 23:07:49 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Thu, 28 Oct 2010 23:07:49 +0200 (CEST) Subject: [z3-checkins] r78423 - z3/deliverance/branches/egj-wsgi-sandbox/deliverance Message-ID: <20101028210749.0FB91282C1E@codespeak.net> Author: ejucovy Date: Thu Oct 28 23:07:48 2010 New Revision: 78423 Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/middleware.py Log: don't theme ajax requests Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/middleware.py ============================================================================== --- z3/deliverance/branches/egj-wsgi-sandbox/deliverance/middleware.py (original) +++ z3/deliverance/branches/egj-wsgi-sandbox/deliverance/middleware.py Thu Oct 28 23:07:48 2010 @@ -71,6 +71,8 @@ return True if req.headers.get("X-Deliverance-Notheme", False): return True + if req.headers.get("X-Requested-With", None) == "XMLHttpRequest": + return True def __call__(self, environ, start_response): req = Request(environ) From ejucovy at codespeak.net Wed Nov 10 17:26:26 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Wed, 10 Nov 2010 17:26:26 +0100 (CET) Subject: [z3-checkins] r78972 - z3/deliverance/branches/egj-wsgi-sandbox/deliverance Message-ID: <20101110162626.DA076282B9E@codespeak.net> Author: ejucovy Date: Wed Nov 10 17:26:25 2010 New Revision: 78972 Added: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/http_proxy.py Log: First stab at extracting deliverance proxy functionality into a standalone wsgi app, deliverance.http_proxy.Proxy Added: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/http_proxy.py ============================================================================== --- (empty file) +++ z3/deliverance/branches/egj-wsgi-sandbox/deliverance/http_proxy.py Wed Nov 10 17:26:25 2010 @@ -0,0 +1,232 @@ +from deliverance.util.urlnormalize import url_normalize +import socket +from wsgiproxy.exactproxy import proxy_exact_request +from lxml.html import document_fromstring, tostring +import re +import urllib +import urlparse +from webob import Request +import logging + +class Proxy(object): + + rewrite_links = True + keep_host = False + strip_script_name = True + + def __init__(self, dest): + self._dest = dest + + def dest(self, request): + return self._dest + + def log(self, request, msg, level='warn'): + pass + + def strip_prefix(self): + pass + + def __call__(self, environ, start_response): + """Forward this request to the remote server, or serve locally. + + This also applies all the request and response transformations. + """ + request, original_script_name, original_path_info = \ + self.prepare_request(environ) + + response, orig_base, proxied_base, proxied_url = self.proxy_to_dest( + request, self.dest(request)) + + if self.rewrite_links: + response = self.rewritten(request, response, + orig_base, proxied_base, proxied_url) + + return response(environ, start_response) + + def prepare_request(self, environ): + request = Request(environ) + original_script_name = request.script_name + original_path_info = request.path_info + prefix = self.strip_prefix() + if prefix: + if prefix.endswith('/'): + prefix = prefix[:-1] + path_info = request.path_info + if not path_info.startswith(prefix + '/') and not path_info == prefix: + self.log(request, + "The match would strip the prefix %r from the request " + "path (%r), but they do not match" + % (prefix + '/', path_info)) + else: + request.script_name = request.script_name + prefix + request.path_info = path_info[len(prefix):] + + return request, original_script_name, original_path_info + + def construct_proxy_request(self, request, dest): + """ + returns a new Request object constructed by copying `request` + and replacing its url with the url passed in as `dest` + + @raises TypeError if `dest` is a file:// url; this can be + caught by the caller and handled accordingly + """ + + dest = url_normalize(dest) + scheme, netloc, path, query, fragment = urlparse.urlsplit(dest) + path = urllib.unquote(path) + + assert not fragment, ( + "Unexpected fragment: %r" % fragment) + + proxy_req = Request(request.environ.copy()) + + proxy_req.path_info = path + + proxy_req.server_name = netloc.split(':', 1)[0] + if ':' in netloc: + proxy_req.server_port = netloc.split(':', 1)[1] + elif scheme == 'http': + proxy_req.server_port = '80' + elif scheme == 'https': + proxy_req.server_port = '443' + elif scheme == 'file': + raise TypeError ## FIXME: is TypeError too general? + else: + assert 0, "bad scheme: %r (from %r)" % (scheme, dest) + if not self.keep_host: + proxy_req.host = netloc + + proxy_req.query_string = query + proxy_req.scheme = scheme + + proxy_req.headers['X-Forwarded-For'] = request.remote_addr + proxy_req.headers['X-Forwarded-Scheme'] = request.scheme + proxy_req.headers['X-Forwarded-Server'] = request.host + + ## FIXME: something with path? proxy_req.headers['X-Forwarded-Path'] + ## (now we are only doing it with strip_script_name) + if self.strip_script_name: + proxy_req.headers['X-Forwarded-Path'] = proxy_req.script_name + proxy_req.script_name = '' + + return proxy_req + + def proxy_to_dest(self, request, dest): + """Do the actual proxying, without applying any transformations""" + proxy_req = self.construct_proxy_request(request, dest) + + proxy_req.path_info += request.path_info + + if proxy_req.query_string and request.query_string: + proxy_req.query_string = '%s&%s' % \ + (proxy_req.query_string, request.query_string) + elif request.query_string: + proxy_req.query_string = request.query_string + + proxy_req.accept_encoding = None + try: + resp = proxy_req.get_response(proxy_exact_request) + if resp.status_int == 500: + print 'Request:' + print proxy_req + print 'Response:' + print resp + except socket.error, e: + ## FIXME: really wsgiproxy should handle this + ## FIXME: which error? + ## 502 HTTPBadGateway, 503 HTTPServiceUnavailable, 504 HTTPGatewayTimeout? + if isinstance(e.args, tuple) and len(e.args) > 1: + error = e.args[1] + else: + error = str(e) + resp = exc.HTTPServiceUnavailable( + 'Could not proxy the request to %s:%s : %s' + % (proxy_req.server_name, proxy_req.server_port, error)) + + dest = url_normalize(dest) + orig_base = url_normalize(request.application_url) + proxied_url = url_normalize('%s://%s%s' % (proxy_req.scheme, + proxy_req.host, + proxy_req.path_qs)) + + return resp, orig_base, dest, proxied_url + + _cookie_domain_re = re.compile(r'(domain="?)([a-z0-9._-]*)("?)', re.I) + + def rewritten(self, request, response, orig_base, proxied_base, proxied_url): + + if proxied_base is not None and proxied_url is not None: + # This might not have a trailing /: + exact_proxied_base = proxied_base + if not proxied_base.endswith('/'): + proxied_base += '/' + exact_orig_base = orig_base + if not orig_base.endswith('/'): + orig_base += '/' + assert (proxied_url.startswith(proxied_base) + or proxied_url.split('?', 1)[0] == proxied_base[:-1]), ( + "Unexpected proxied_url %r, doesn't start with proxied_base %r" + % (proxied_url, proxied_base)) + assert (request.url.startswith(orig_base) + or request.url.split('?', 1)[0] == orig_base[:-1]), ( + "Unexpected request.url %r, doesn't start with orig_base %r" + % (request.url, orig_base)) + + def link_repl_func(link): + """Rewrites a link to point to this proxy""" + if link == exact_proxied_base: + return exact_orig_base + if not link.startswith(proxied_base): + # External link, so we don't rewrite it + return link + new = orig_base + link[len(proxied_base):] + return new + if response.content_type != 'text/html': + self.log(request, + 'Not rewriting links in response from %s, because Content-Type is %s' + % (proxied_url, response.content_type), + 'debug') + else: + if not response.charset: + body = response.body + else: + body = response.unicode_body + body_doc = document_fromstring(body, base_url=proxied_url) + body_doc.make_links_absolute() + body_doc.rewrite_links(link_repl_func) + response.body = tostring(body_doc) + + if response.location: + ## FIXME: if you give a proxy like + ## http://openplans.org, and it redirects to + ## http://www.openplans.org, it won't be rewritten and + ## that can be confusing -- it *shouldn't* be + ## rewritten, but some better log message is required + loc = urlparse.urljoin(proxied_url, response.location) + loc = link_repl_func(loc) + response.location = loc + if 'set-cookie' in response.headers: + cookies = response.headers.getall('set-cookie') + del response.headers['set-cookie'] + for cook in cookies: + old_domain = urlparse.urlsplit(proxied_url)[1].lower() + new_domain = request.host.split(':', 1)[0].lower() + def rewrite_domain(match): + """Rewrites domains to point to this proxy""" + domain = match.group(2) + if domain == old_domain: + ## FIXME: doesn't catch wildcards and the sort + return match.group(1) + new_domain + match.group(3) + else: + return match.group(0) + cook = self._cookie_domain_re.sub(rewrite_domain, cook) + response.headers.add('set-cookie', cook) + return response + +if __name__ == '__main__': + req = Request.blank("http://nytimes.com/about") + app = Proxy("http://www.google.com") + resp = req.get_response(app) + print resp.location + From ejucovy at codespeak.net Wed Nov 10 18:38:08 2010 From: ejucovy at codespeak.net (ejucovy at codespeak.net) Date: Wed, 10 Nov 2010 18:38:08 +0100 (CET) Subject: [z3-checkins] r78978 - z3/deliverance/branches/egj-wsgi-sandbox/deliverance Message-ID: <20101110173808.46773282B90@codespeak.net> Author: ejucovy Date: Wed Nov 10 18:38:05 2010 New Revision: 78978 Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/http_proxy.py Log: missing import Modified: z3/deliverance/branches/egj-wsgi-sandbox/deliverance/http_proxy.py ============================================================================== --- z3/deliverance/branches/egj-wsgi-sandbox/deliverance/http_proxy.py (original) +++ z3/deliverance/branches/egj-wsgi-sandbox/deliverance/http_proxy.py Wed Nov 10 18:38:05 2010 @@ -7,6 +7,7 @@ import urlparse from webob import Request import logging +from webob import exc class Proxy(object):