From philikon at codespeak.net Sun Apr 1 17:33:36 2007 From: philikon at codespeak.net (philikon at codespeak.net) Date: Sun, 1 Apr 2007 17:33:36 +0200 (CEST) Subject: [z3-checkins] r41764 - z3/NudgeNudge/trunk Message-ID: <20070401153336.D0D171007C@code0.codespeak.net> Author: philikon Date: Sun Apr 1 17:33:35 2007 New Revision: 41764 Modified: z3/NudgeNudge/trunk/buildout.cfg Log: cosmetics Modified: z3/NudgeNudge/trunk/buildout.cfg ============================================================================== --- z3/NudgeNudge/trunk/buildout.cfg (original) +++ z3/NudgeNudge/trunk/buildout.cfg Sun Apr 1 17:33:35 2007 @@ -1,3 +1,7 @@ +[buildout] +parts = data instance test +develop = . deliverance + [test] working-directory = parts/instance eggs = nudgenudge @@ -13,10 +17,6 @@ [zope3] location = /usr/local/Zope-3.3.1 -[buildout] -parts = data instance test -develop = . deliverance - [instance] database = data eggs = setuptools From kobold at codespeak.net Sun Apr 1 19:24:56 2007 From: kobold at codespeak.net (kobold at codespeak.net) Date: Sun, 1 Apr 2007 19:24:56 +0200 (CEST) Subject: [z3-checkins] r41771 - in z3/sqlos/trunk/src/sqlos: . ftests Message-ID: <20070401172456.098E21007C@code0.codespeak.net> Author: kobold Date: Sun Apr 1 19:24:55 2007 New Revision: 41771 Modified: z3/sqlos/trunk/src/sqlos/connection.py z3/sqlos/trunk/src/sqlos/ftests/localutilities.txt Log: Let's try to fix the new code: it didn't work with multiple threads. Modified: z3/sqlos/trunk/src/sqlos/connection.py ============================================================================== --- z3/sqlos/trunk/src/sqlos/connection.py (original) +++ z3/sqlos/trunk/src/sqlos/connection.py Sun Apr 1 19:24:55 2007 @@ -21,6 +21,7 @@ """ __metaclass__ = type +import thread import warnings from zope.app import zapi @@ -66,58 +67,41 @@ # This code was heavily based on ZODB.Transaction connCache = {} -try: - import thread -except: - def getConnection(context, name): - global connCache - if not connCache.get(name): - newconn = zapi.queryUtility(IZopeDatabaseAdapter, name, - default=None, - context=context) - if newconn is None: - warnings.warn("Couldn't find a rdb connection by the " - "name %s. Please verify your setup." % name, - SQLOSWarning, 3) - connCache[name] = IZopeSQLConnection(newconn()) - return connCache[name] - - def releaseConnection(name): - global connCache - if connCache.has_key(name): - connCache[name].close() - del connCache[name] -else: - def getConnection(context, name): - global connCache - tid = thread.get_ident() - key = (tid, name) - if not connCache.get(key): - newconn = zapi.queryUtility(IZopeDatabaseAdapter, name, - default=None, - context=context) - if newconn is None: - warnings.warn("Couldn't find a rdb connection by the " - "name %s. Please verify your setup." % name, - SQLOSWarning, 3) - conn = IZopeSQLConnection(newconn()) - if conn.supportTransactions: - connCache[key] = conn.transaction() - else: # At least MySQL does not support transactions - connCache[key] = conn - - return connCache[key] - - def releaseConnection(name): - global connCache - tid = thread.get_ident() - key = (tid, name) - if connCache.has_key(key): - connCache[key].close() - del connCache[key] + +def getConnection(context, name): + global connCache + tid = thread.get_ident() + key = (tid, name) + if not connCache.get(key): + newconn = zapi.queryUtility(IZopeDatabaseAdapter, name, + default=None, + context=context) + if newconn is None: + warnings.warn("Couldn't find a rdb connection by the " + "name %s. Please verify your setup." % name, + SQLOSWarning, 3) + conn = IZopeSQLConnection(newconn()) + if conn.supportTransactions: + connCache[key] = conn.transaction() + else: # At least MySQL does not support transactions + connCache[key] = conn + + return connCache[key] + + +def releaseConnection(name): + global connCache + tid = thread.get_ident() + key = (tid, name) + if connCache.has_key(key): + connCache[key].close() + del connCache[key] def clearCacheSubscriber(*args): """A subscriber to clear the connection cache at site boundaries.""" - for connection in connCache.values(): - connection.cache.clear() + tid = thread.get_ident() + for name in connCache.keys(): + if name[0] != tid: + continue + connCache[name].cache.clear() Modified: z3/sqlos/trunk/src/sqlos/ftests/localutilities.txt ============================================================================== --- z3/sqlos/trunk/src/sqlos/ftests/localutilities.txt (original) +++ z3/sqlos/trunk/src/sqlos/ftests/localutilities.txt Sun Apr 1 19:24:55 2007 @@ -41,9 +41,6 @@ >>> localUtility is dbAdapter True - >>> localUtility is dbAdapter - True - Ok. Now we have a site in root/testsite with a registered sqlite adapter. We make sure that our localUtility is not identical to the global sqlite utility that has been registered through ftesting.zcml. From philikon at codespeak.net Mon Apr 2 01:02:32 2007 From: philikon at codespeak.net (philikon at codespeak.net) Date: Mon, 2 Apr 2007 01:02:32 +0200 (CEST) Subject: [z3-checkins] r41782 - in z3/NudgeNudge/trunk: . etc Message-ID: <20070401230232.4ED9710080@code0.codespeak.net> Author: philikon Date: Mon Apr 2 01:02:29 2007 New Revision: 41782 Modified: z3/NudgeNudge/trunk/ (props changed) z3/NudgeNudge/trunk/buildout.cfg z3/NudgeNudge/trunk/etc/paste.ini Log: * Use development version (checkout for now) of zope.paste which supports running Zope 3 completely on PasteDeploy * New buildout part that installs PasteScript and lets you run NudgeNudge like so: $ bin/paster serve etc/paste.ini Modified: z3/NudgeNudge/trunk/buildout.cfg ============================================================================== --- z3/NudgeNudge/trunk/buildout.cfg (original) +++ z3/NudgeNudge/trunk/buildout.cfg Mon Apr 2 01:02:29 2007 @@ -1,6 +1,6 @@ [buildout] -parts = data instance test -develop = . deliverance +parts = data server instance test +develop = . deliverance zope.paste [test] working-directory = parts/instance @@ -17,6 +17,13 @@ [zope3] location = /usr/local/Zope-3.3.1 +[server] +recipe = zc.recipe.egg +eggs = ${instance:eggs} + PasteScript + WSGIUtils +extra-paths = ${zope3:location}/lib/python + [instance] database = data eggs = setuptools Modified: z3/NudgeNudge/trunk/etc/paste.ini ============================================================================== --- z3/NudgeNudge/trunk/etc/paste.ini (original) +++ z3/NudgeNudge/trunk/etc/paste.ini Mon Apr 2 01:02:29 2007 @@ -1,8 +1,20 @@ -[filter-app:Paste.Main] +[composite:main] +use = egg:Paste#urlmap +/ = themed +/notheme = zope + +[filter-app:themed] use = egg:Deliverance theme_uri = /@@/nudgenudge/www.python.org/pypi/index.html rule_uri = /@@/nudgenudge/rules.xml next = zope [app:zope] -paste.app_factory = zope.paste.application:zope_publisher_app_factory +use = egg:zope.paste +site_definition = ../parts/instance/etc/site.zcml +file_storage = ../parts/data/Data.fs + +[server:main] +use = egg:PasteScript#wsgiutils +host = 127.0.0.1 +port = 8080 From philikon at codespeak.net Mon Apr 2 01:04:42 2007 From: philikon at codespeak.net (philikon at codespeak.net) Date: Mon, 2 Apr 2007 01:04:42 +0200 (CEST) Subject: [z3-checkins] r41783 - in z3/NudgeNudge/trunk: . etc Message-ID: <20070401230442.4626D10080@code0.codespeak.net> Author: philikon Date: Mon Apr 2 01:04:40 2007 New Revision: 41783 Added: z3/NudgeNudge/trunk/nudge.ini - copied, changed from r41782, z3/NudgeNudge/trunk/etc/paste.ini Removed: z3/NudgeNudge/trunk/etc/ Log: Move PasteDeploy configuration out of etc and give it a better name. You now start NudgeNudge like so: $ bin/paster serve nudge.ini Copied: z3/NudgeNudge/trunk/nudge.ini (from r41782, z3/NudgeNudge/trunk/etc/paste.ini) ============================================================================== --- z3/NudgeNudge/trunk/etc/paste.ini (original) +++ z3/NudgeNudge/trunk/nudge.ini Mon Apr 2 01:04:40 2007 @@ -11,8 +11,8 @@ [app:zope] use = egg:zope.paste -site_definition = ../parts/instance/etc/site.zcml -file_storage = ../parts/data/Data.fs +site_definition = parts/instance/etc/site.zcml +file_storage = parts/data/Data.fs [server:main] use = egg:PasteScript#wsgiutils From philikon at codespeak.net Mon Apr 2 01:08:40 2007 From: philikon at codespeak.net (philikon at codespeak.net) Date: Mon, 2 Apr 2007 01:08:40 +0200 (CEST) Subject: [z3-checkins] r41784 - z3/NudgeNudge/trunk Message-ID: <20070401230840.7A87810080@code0.codespeak.net> Author: philikon Date: Mon Apr 2 01:08:37 2007 New Revision: 41784 Modified: z3/NudgeNudge/trunk/buildout.cfg Log: We no longer need the PasteDeploy/Deliverance stuff in the instance recipe. Modified: z3/NudgeNudge/trunk/buildout.cfg ============================================================================== --- z3/NudgeNudge/trunk/buildout.cfg (original) +++ z3/NudgeNudge/trunk/buildout.cfg Mon Apr 2 01:08:37 2007 @@ -20,6 +20,9 @@ [server] recipe = zc.recipe.egg eggs = ${instance:eggs} + zope.paste + PasteDeploy + Deliverance PasteScript WSGIUtils extra-paths = ${zope3:location}/lib/python @@ -29,9 +32,6 @@ eggs = setuptools grok zc.catalog - zope.paste - PasteDeploy - Deliverance nudgenudge recipe = zc.recipe.zope3instance user = grok:grok @@ -55,7 +55,6 @@ zope.app.keyreference zope.app.session zope.app.twisted - zope.paste grok grok-meta nudgenudge From philikon at codespeak.net Mon Apr 2 01:14:22 2007 From: philikon at codespeak.net (philikon at codespeak.net) Date: Mon, 2 Apr 2007 01:14:22 +0200 (CEST) Subject: [z3-checkins] r41785 - z3/NudgeNudge/trunk Message-ID: <20070401231422.6118610080@code0.codespeak.net> Author: philikon Date: Mon Apr 2 01:14:20 2007 New Revision: 41785 Modified: z3/NudgeNudge/trunk/INSTALL.txt Log: Update installation docs Modified: z3/NudgeNudge/trunk/INSTALL.txt ============================================================================== --- z3/NudgeNudge/trunk/INSTALL.txt (original) +++ z3/NudgeNudge/trunk/INSTALL.txt Mon Apr 2 01:14:20 2007 @@ -17,30 +17,28 @@ $ python2.4 bootstrap/bootstrap.py 4. Run the buildout to install required packages (grok, deliverance, - paste.deploy, zope.paste, etc.) and create the Zope 3 instance - automatically:: + paste.deploy, zope.paste, etc.) and create the Zope 3 configuration + for NudgeNudge automatically:: $ bin/buildout ...lots of output here... -5. Start Zope:: +5. We are now able start up Zope using the following command:: $ parts/instance/bin/runzope - You will now be able to create a NudgeNudge application object by - pointing your webbrowser to http://localhost:8080. + Then we can create a NudgeNudge application object by pointing our + webbrowser to http://localhost:8080. Let's call it "nudge". -6. To switch on the theming middleware, edit ``parts/etc/zope.conf`` - and add the following server definition:: - - - type Paste.Main - address 8081 - - - You'll have to restart Zope after this change. You'll then be able - to access the unthemed application on port 8080 like before and the - themed one on port 8081. +6. To run the NudgeNudge application using the PasteDeploy stack and + therefore themed using Deliverance, start the server using the + following command:: + + $ bin/paster serve nudge.ini + + You'll then be able to access the themed application at + http://localhost:8080/nudge and unthemed at + http://localhost:8080/notheme/nudge. Note that the Deliverance middleware requires lxml to do the theming which is known to have problems on certain platforms, From philikon at codespeak.net Mon Apr 2 01:22:17 2007 From: philikon at codespeak.net (philikon at codespeak.net) Date: Mon, 2 Apr 2007 01:22:17 +0200 (CEST) Subject: [z3-checkins] r41786 - z3/NudgeNudge/trunk Message-ID: <20070401232217.8064F10080@code0.codespeak.net> Author: philikon Date: Mon Apr 2 01:22:15 2007 New Revision: 41786 Modified: z3/NudgeNudge/trunk/buildout.cfg Log: Move and fix [test] part (it's not really used anyway, but why not) Modified: z3/NudgeNudge/trunk/buildout.cfg ============================================================================== --- z3/NudgeNudge/trunk/buildout.cfg (original) +++ z3/NudgeNudge/trunk/buildout.cfg Mon Apr 2 01:22:15 2007 @@ -2,15 +2,6 @@ parts = data server instance test develop = . deliverance zope.paste -[test] -working-directory = parts/instance -eggs = nudgenudge -recipe = zc.recipe.testrunner -extra-paths = parts/zope3/src -defaults = ['--tests-pattern', '^f?tests$', - '-v' - ] - [data] recipe = zc.recipe.filestorage @@ -59,3 +50,11 @@ grok-meta nudgenudge +[test] +working-directory = parts/instance +eggs = nudgenudge +recipe = zc.recipe.testrunner +extra-paths = ${zope3:location}/lib/python +defaults = ['--tests-pattern', '^f?tests$', + '-v' + ] From philikon at codespeak.net Mon Apr 2 01:23:05 2007 From: philikon at codespeak.net (philikon at codespeak.net) Date: Mon, 2 Apr 2007 01:23:05 +0200 (CEST) Subject: [z3-checkins] r41787 - z3/NudgeNudge/trunk Message-ID: <20070401232305.2A07F10080@code0.codespeak.net> Author: philikon Date: Mon Apr 2 01:23:03 2007 New Revision: 41787 Modified: z3/NudgeNudge/trunk/buildout.cfg Log: Move [data] part, just cosmetics Modified: z3/NudgeNudge/trunk/buildout.cfg ============================================================================== --- z3/NudgeNudge/trunk/buildout.cfg (original) +++ z3/NudgeNudge/trunk/buildout.cfg Mon Apr 2 01:23:03 2007 @@ -2,9 +2,6 @@ parts = data server instance test develop = . deliverance zope.paste -[data] -recipe = zc.recipe.filestorage - [zope3] location = /usr/local/Zope-3.3.1 @@ -50,6 +47,9 @@ grok-meta nudgenudge +[data] +recipe = zc.recipe.filestorage + [test] working-directory = parts/instance eggs = nudgenudge From philikon at codespeak.net Mon Apr 2 01:23:52 2007 From: philikon at codespeak.net (philikon at codespeak.net) Date: Mon, 2 Apr 2007 01:23:52 +0200 (CEST) Subject: [z3-checkins] r41788 - z3/NudgeNudge/trunk Message-ID: <20070401232352.A720710080@code0.codespeak.net> Author: philikon Date: Mon Apr 2 01:23:50 2007 New Revision: 41788 Modified: z3/NudgeNudge/trunk/buildout.cfg Log: More cosmetics: change order in which parts are executed Modified: z3/NudgeNudge/trunk/buildout.cfg ============================================================================== --- z3/NudgeNudge/trunk/buildout.cfg (original) +++ z3/NudgeNudge/trunk/buildout.cfg Mon Apr 2 01:23:50 2007 @@ -1,5 +1,5 @@ [buildout] -parts = data server instance test +parts = data instance server test develop = . deliverance zope.paste [zope3] From kobold at codespeak.net Mon Apr 2 08:22:56 2007 From: kobold at codespeak.net (kobold at codespeak.net) Date: Mon, 2 Apr 2007 08:22:56 +0200 (CEST) Subject: [z3-checkins] r41791 - z3/sqlos/trunk/src/sqlos Message-ID: <20070402062256.0AF4A10077@code0.codespeak.net> Author: kobold Date: Mon Apr 2 08:22:53 2007 New Revision: 41791 Modified: z3/sqlos/trunk/src/sqlos/_transaction.py Log: Fixed a bug. Modified: z3/sqlos/trunk/src/sqlos/_transaction.py ============================================================================== --- z3/sqlos/trunk/src/sqlos/_transaction.py (original) +++ z3/sqlos/trunk/src/sqlos/_transaction.py Mon Apr 2 08:22:53 2007 @@ -44,9 +44,23 @@ This function should not cause SQL to be sent as it is not defined whether the SQL connection will commit before or after this is executed. + + This method should work even if the object doesn't have an id attribute: it + sometimes happens that SQLObject instances don't have one. + + >>> class Dummy: + ... def expire(self): + ... print "Expires" + + >>> obj = Dummy() + >>> expireSQLObject(obj) + Expires + """ - for connection in connCache.values(): - connection.cache.expire(obj.id, obj.__class__) + # Sometimes, the object doesn't have the id: in this case, ignore it. + if hasattr(obj, 'id'): + for connection in connCache.values(): + connection.cache.expire(obj.id, obj.__class__) # Expire object values. The transaction has either been aborted or # committed. From kobold at codespeak.net Mon Apr 2 14:11:35 2007 From: kobold at codespeak.net (kobold at codespeak.net) Date: Mon, 2 Apr 2007 14:11:35 +0200 (CEST) Subject: [z3-checkins] r41800 - z3/sqlos/trunk/src/sqlos Message-ID: <20070402121135.4FB3D10077@code0.codespeak.net> Author: kobold Date: Mon Apr 2 14:11:30 2007 New Revision: 41800 Modified: z3/sqlos/trunk/src/sqlos/_transaction.py Log: Fix. Modified: z3/sqlos/trunk/src/sqlos/_transaction.py ============================================================================== --- z3/sqlos/trunk/src/sqlos/_transaction.py (original) +++ z3/sqlos/trunk/src/sqlos/_transaction.py Mon Apr 2 14:11:30 2007 @@ -54,7 +54,6 @@ >>> obj = Dummy() >>> expireSQLObject(obj) - Expires """ # Sometimes, the object doesn't have the id: in this case, ignore it. @@ -62,9 +61,9 @@ for connection in connCache.values(): connection.cache.expire(obj.id, obj.__class__) - # Expire object values. The transaction has either been aborted or - # committed. - obj.expire() + # Expire object values. The transaction has either been aborted or + # committed. + obj.expire() class SQLObjectTransactionManager: From ianb at codespeak.net Thu Apr 5 23:25:30 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Thu, 5 Apr 2007 23:25:30 +0200 (CEST) Subject: [z3-checkins] r41917 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070405212530.6EFB810077@code0.codespeak.net> Author: ianb Date: Thu Apr 5 23:25:28 2007 New Revision: 41917 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/wsgiapp.py Log: remove commented line Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/wsgiapp.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/wsgiapp.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/wsgiapp.py Thu Apr 5 23:25:28 2007 @@ -1,6 +1,5 @@ from paste.cascade import Cascade from paste.urlparser import StaticURLParser -#from paste.deploy.config import ConfigMiddleware from paste.deploy.converters import asbool from paste.recursive import RecursiveMiddleware from paste.registry import RegistryManager From ianb at codespeak.net Thu Apr 5 23:25:58 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Thu, 5 Apr 2007 23:25:58 +0200 (CEST) Subject: [z3-checkins] r41918 - z3/deliverance/DeliveranceVHoster/trunk Message-ID: <20070405212558.73D9E10078@code0.codespeak.net> Author: ianb Date: Thu Apr 5 23:25:57 2007 New Revision: 41918 Modified: z3/deliverance/DeliveranceVHoster/trunk/setup.py Log: Add PasteScript dependency, simplify Paste dependency Modified: z3/deliverance/DeliveranceVHoster/trunk/setup.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/setup.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/setup.py Thu Apr 5 23:25:57 2007 @@ -8,7 +8,8 @@ #author_email="", #url="", install_requires=[ - 'Paste==dev,>1.1.1', # There's a bug fix we need + 'Paste>1.1.1', + 'PasteScript', 'Deliverance', 'WSGIFilter', 'HTTPEncode', From ianb at codespeak.net Thu Apr 5 23:26:18 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Thu, 5 Apr 2007 23:26:18 +0200 (CEST) Subject: [z3-checkins] r41919 - z3/deliverance/DeliveranceVHoster/trunk/tests/test-static/blah Message-ID: <20070405212618.1E69010077@code0.codespeak.net> Author: ianb Date: Thu Apr 5 23:26:17 2007 New Revision: 41919 Added: z3/deliverance/DeliveranceVHoster/trunk/tests/test-static/blah/index.html (contents, props changed) Log: Add index file to make testing directory easier Added: z3/deliverance/DeliveranceVHoster/trunk/tests/test-static/blah/index.html ============================================================================== --- (empty file) +++ z3/deliverance/DeliveranceVHoster/trunk/tests/test-static/blah/index.html Thu Apr 5 23:26:17 2007 @@ -0,0 +1 @@ +hi From ianb at codespeak.net Thu Apr 5 23:37:09 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Thu, 5 Apr 2007 23:37:09 +0200 (CEST) Subject: [z3-checkins] r41921 - in z3/deliverance/DeliveranceVHoster/trunk: docs dvhoster tests Message-ID: <20070405213709.B36D910077@code0.codespeak.net> Author: ianb Date: Thu Apr 5 23:37:08 2007 New Revision: 41921 Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dataprovider.py z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py Log: Fixed some tests that were unnecessarily failing. Add two new hooks -- find_remote_uri and should_theme_uri, which can do custom remote URI resolution, or disable theming entirely. Sort remote_uris by length (longest path first) so that there can never be dead (unreachable) paths. Allow a headers key in that dictionary, to set extra headers on the request. Removed a bunch of debugging print statements Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py Thu Apr 5 23:37:08 2007 @@ -36,4 +36,23 @@ ('X-Openplans-Project', project), ] - +def find_remote_uri(remote_uri, remote_uri_info, environ, + app_conf): + from paste.request import path_info_pop + if remote_uri is not None: + return remote_uri + path_info = environ.get('PATH_INFO', '') + if path_info.startswith('/tasks'): + remote_uri = 'http://localhost:5000' + # Move /tasks to SCRIPT_NAME + path_info_pop(environ) + else: + remote_uri = 'http://localhost:8080' + return remote_uri + +def should_theme_uri(remote_uri, environ, app_conf): + try: + no_theme = int(environ.get('HTTP_X_NO_THEME', '0')) + except ValueError: + no_theme = 0 + return not no_theme Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dataprovider.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dataprovider.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dataprovider.py Thu Apr 5 23:37:08 2007 @@ -231,6 +231,7 @@ required_keys = () optional_keys = () trail_slash_keys = () + sort_key = None def validate_python(self, value, state=None): # Should be like [{'path': path, 'remote_uri': uri, 'comment': str}] @@ -263,12 +264,16 @@ continue if not d[key].endswith('/'): d[key] += '/' + if self.sort_key: + value = sorted(value, key=self.sort_key) return value class RemoteURIValidator(ListDictValidator): - required_keys = ('path', 'remote_uri') - optional_keys = ('comment', ) + required_keys = ('path', ) + optional_keys = ('comment', 'headers', 'remote_uri') trail_slash_keys = ('path', 'remote_uri') + def sort_key(self, item): + return -len(item['path']) class RewriteValidator(ListDictValidator): required_keys = ('rewrite', ) Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Thu Apr 5 23:37:08 2007 @@ -33,6 +33,14 @@ if init_domain: init_domain = load_func(init_domain, 'init_domain') self.init_domain = init_domain + find_remote_uri = app_conf.get('find_remote_uri') + if find_remote_uri: + find_remote_uri = load_func(find_remote_uri, 'find_remote_uri') + self.find_remote_uri = find_remote_uri + should_theme_uri = app_conf.get('should_theme_uri') + if should_theme_uri: + should_theme_uri = load_func(should_theme_uri, 'should_theme_uri') + self.should_theme_uri = should_theme_uri self.app_conf = app_conf self.provider = DataProvider(data_dir) self.rewrite_links = asbool(app_conf.get('rewrite_links', True)) @@ -59,7 +67,6 @@ environ, with_query_string=False, path_info='') path_info = norm_path(environ.get('PATH_INFO', '')) - print 'path_info', repr(path_info), repr(environ['PATH_INFO']) if path_info.startswith('/.deliverance'): path_info_pop(environ) subapp = ProviderApp(domain_info) @@ -129,21 +136,22 @@ for header_name, header_value in domain_info.additional_request_headers: header_name = 'HTTP_%s' % header_name.upper().replace('-', '_') environ[header_name] = header_value - + for remote_uri_info in remote_uris: - path = remote_uri_info['path'] + path = str(remote_uri_info['path']) if not path.endswith('/'): path += '/' if path_info + '/' == path: - print 'redirect', [remote_uri_info, path, path_info] exc = httpexceptions.HTTPMovedPermanently( headers=[('location', construct_url(environ, path_info=path_info+'/'))]) return exc(environ, start_response) if path_info.startswith(path): # Found a match - remote_uri = remote_uri_info['remote_uri'] + remote_uri = remote_uri_info.get('remote_uri') environ['SCRIPT_NAME'] += path[:-1] environ['PATH_INFO'] = path_info[len(path)-1:] + if self.find_remote_uri: + remote_uri = self.find_remote_uri(remote_uri, remote_uri_info, environ, self.app_conf) break if not remote_uri: exc = httpexceptions.HTTPNotFound( @@ -152,8 +160,10 @@ % (path_info, ', '.join([repr(r['path']) for r in remote_uris]))) return exc(environ, start_response) - print 'Proxy to %r with %r (SCRIPT_NAME=%r)' % ( - remote_uri, construct_url(environ), environ['SCRIPT_NAME']) + should_theme_uri = True + if self.should_theme_uri: + should_theme_uri = self.should_theme_uri( + remote_uri, environ, self.app_conf) app = proxyapp.ForcedProxy( remote=remote_uri, force_host=True) @@ -163,11 +173,12 @@ rule_uri = construct_url( environ, with_query_string=False, path_info='/_rules/rule.xml') - app = DeliveranceMiddleware( - app, - theme_uri=domain_info.theme_uri, - rule_uri=rule_uri, - renderer=Renderer) + if should_theme_uri: + app = DeliveranceMiddleware( + app, + theme_uri=domain_info.theme_uri, + rule_uri=rule_uri, + renderer=Renderer) return app(environ, start_response) def find_file(self, path): Modified: z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Thu Apr 5 23:37:08 2007 @@ -8,6 +8,7 @@ for attr in ['get_app', 'connect']: setattr(httplib.HTTPConnection, attr, getattr(wsgi_intercept.WSGI_HTTPConnection, attr).im_func) +from wsgifilter.proxyapp import DebugHeaders data_filename = os.path.join(os.path.dirname(__file__), 'test-data') wsgi_app = make_app({}, data_dir=data_filename) @@ -30,13 +31,6 @@ ''' -static_app = StaticURLParser(os.path.join(os.path.dirname(__file__), - 'test-static')) -def make_static_app(): - return static_app - -wsgi_intercept.add_wsgi_intercept('wsgify.org', 80, make_static_app) - def test_everything(): yield (reset_env,) @@ -74,8 +68,10 @@ data = ''' [{"path": "/bar", "remote_uri": "http://wsgify.org/blah", "comment": "x"}, - {"path": "/", "remote_uri": "http://wsgify.org/"}] + {"path": "/", "remote_uri": "http://wsgify.org/"} + ] ''' + #{"path": "/testme", "headers": {"X-Test-Me": "testme"} put('/.deliverance/remote_uris', data) # It gets normalized, so it doesn't actually stay the same: @@ -88,6 +84,8 @@ #res = res.follow() #print res #assert res.status == 200 + ## So instead we just get a page we know works: + res = app.get('/bar/index.html', status=200) data = ''' [{"path": "/test1.html", "rewrite": "/test1"}, @@ -146,3 +144,14 @@ res.mustcontain('localhost') assert app.get('/.deliverance/domain').body == 'localhost2' print 'site renamed to localhost2' + + +def setup_module(module): + static_app = DebugHeaders(StaticURLParser(os.path.join(os.path.dirname(__file__), + 'test-static'))) + + wsgi_intercept.add_wsgi_intercept('wsgify.org', 80, lambda : static_app) + +def teardown_module(module): + wsgi_intercept.remove_wsgi_intercept('wsgify.org', 80) + Modified: z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py Thu Apr 5 23:37:08 2007 @@ -1,19 +1,29 @@ import os import shutil -import shutil from paste.fixture import TestApp from dvhoster.wsgiapp import make_app import httplib import simplejson +import urllib +import wsgi_intercept +for attr in ['get_app', 'connect']: + setattr(httplib.HTTPConnection, attr, + getattr(wsgi_intercept.WSGI_HTTPConnection, attr).im_func) example = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'docs', 'example_init_domain.py') data_filename = os.path.join(os.path.dirname(__file__), 'test-data') +if os.path.exists(data_filename): + shutil.rmtree(data_filename) +os.mkdir(data_filename) + wsgi_app = make_app({}, init_domain=example, + find_remote_uri=example, + should_theme_uri=example, zope_location='http://localhost:8080', - default_theme_uri='http://yahoo.com', + default_theme_uri='http://openplans.org', data_dir=data_filename) app = TestApp(wsgi_app) @@ -30,3 +40,39 @@ print got print expected assert got == expected + +def test_find_remote_uri(): + # First make double-sure wsgi_intercept is working: + f = urllib.urlopen('http://localhost:8080/foo') + c = f.read() + f.close() + assert "SERVER_PORT: '8080'" in c + # Now to test find_remote_uri; first we have to get rid of the + # remote_uri value that init_domain set: + extra = {'HTTP_HOST': 'foo2.openplans.org'} + app.post('/.deliverance/remote_uris', '[{"path": ""}]', extra_environ={'HTTP_HOST': 'foo2.openplans.org', 'REQUEST_METHOD': 'PUT', 'CONTENT_TYPE': 'application/json'}, status=204) + res = app.get('/tasks', extra_environ=extra) + res.mustcontain("SERVER_PORT: '5000'") + res = app.get('/blah', extra_environ=extra) + res.mustcontain("SERVER_PORT: '8080'") + +def test_should_theme_uri(): + # We need HTML so it can be themed: + setup_module(None, text=False) + res = app.get('/', extra_environ={'HTTP_HOST': 'foo3.openplans.org'}) + # A sign it was themed: + res.mustcontain('Plone') + res = app.get('/', extra_environ={'HTTP_X_NO_THEME': '1', 'HTTP_HOST': 'foo3.openplans.org'}) + assert 'Plone' not in res + +def setup_module(module, text=True): + from paste.script import testapp + app = testapp.TestApplication(text=text) + for port in [5000, 8080]: + wsgi_intercept.add_wsgi_intercept( + 'localhost', port, lambda : app) + +def teardown_module(module): + for port in [5000, 8080]: + wsgi_intercept.remove_wsgi_intercept( + 'localhost', port) From ianb at codespeak.net Fri Apr 6 00:20:13 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 00:20:13 +0200 (CEST) Subject: [z3-checkins] r41922 - z3/deliverance/DeliveranceVHoster/trunk/docs Message-ID: <20070405222013.B4CDD10077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 00:20:11 2007 New Revision: 41922 Added: z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py - copied, changed from r41921, z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py Log: Added a more specific openplans example Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py Fri Apr 6 00:20:11 2007 @@ -1,7 +1,6 @@ # the init_domain setting points to a file or module (a file like this # one). It allows you to define a function that will be run on any -# newly created domain. Here you can put in default settings. This -# example is what we use for openplans.org +# newly created domain. Here you can put in default settings. import re Copied: z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py (from r41921, z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py) ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/docs/example_init_domain.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py Fri Apr 6 00:20:11 2007 @@ -1,9 +1,13 @@ -# the init_domain setting points to a file or module (a file like this -# one). It allows you to define a function that will be run on any -# newly created domain. Here you can put in default settings. This -# example is what we use for openplans.org +# the init_domain, find_remote_uri, and should_theme_uri settings +# point to a file or module (a file like this one). It allows you to +# define a function that will be run on any newly created domain, +# define how requests are mapped to backend domains, and conditionally +# theme requests. Here you can put in default settings. This example +# is what we use for openplans.org import re +from paste.deploy.converters import asbool +from paste.request import path_info_pop rule_data = """\ @@ -13,46 +17,61 @@ """ - def init_domain(domain_info, app_conf): + r""" + Initializes an openplans project, based on a domain regular + expression (default ``^(.*)\.openplans\.org$`` but you can + override with the ``domain_regex`` config setting). + + Also requires a ``default_theme_uri`` setting. + """ domain = domain_info.domain - match = re.search(r'^(.*)\.openplans\.org$', domain, re.I) + regex = app_conf.get('domain_regex', r'^(.*)\.openplans\.org$') + match = re.search(domain_regex, domain, re.I) if not match: # Don't try to set up domains we don't recognize return project = match.group(1) - remote_uri = ( - '%s/VirtualHostBase/http/%s:80/openplans/projects/%s/VirtualHostRoot' - % (app_conf['zope_location'], - domain, - project)) - domain_info.remote_uris = [ + remote_uris = [ {'path': '', - 'remote_uri': remote_uri}] + 'headers': {'X-Openplans-Project': project}}, + ] + domain_info.remote_uris = remote_uris domain_info.theme_uri = app_conf['default_theme_uri'] domain_info.set_rule_file( 'rule.xml', rule_data) - domain_info.additional_request_headers = [ - ('X-Openplans-Project', project), - ] - + def find_remote_uri(remote_uri, remote_uri_info, environ, app_conf): - from paste.request import path_info_pop + """ + Maps request to a remote_uri (when none has been explicitly set). + + You must configure a task_tracker_uri and zope_uri (typically + ``http://localhost:X``). + """ if remote_uri is not None: + # It's already explicitly set return remote_uri + zope_uri = app_conf['zope_uri'] + task_tracker_uri = app_conf['task_tracker_uri'] + project = environ['HTTP_X_OPENPLANS_PROJECT'] path_info = environ.get('PATH_INFO', '') if path_info.startswith('/tasks'): - remote_uri = 'http://localhost:5000' + remote_uri = task_tracker_uri # Move /tasks to SCRIPT_NAME path_info_pop(environ) else: - remote_uri = 'http://localhost:8080' + remote_uri = '%s/VirtualHostBase/http/%s/openplans/projects/%s/VirtualHostRoot' % ( + zope_uri, + environ['HTTP_HOST'], + project) return remote_uri def should_theme_uri(remote_uri, environ, app_conf): - try: - no_theme = int(environ.get('HTTP_X_NO_THEME', '0')) - except ValueError: - no_theme = 0 - return not no_theme + """ + This will avoid theming Zope URLs, but only if you have set the + configuration ``no_filter_zope = true`` + """ + if not asbool(app_conf.get('no_filter_zope')): + return True + return not remote_uri.startswith(app_conf['zope_uri']) From ianb at codespeak.net Fri Apr 6 00:25:41 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 00:25:41 +0200 (CEST) Subject: [z3-checkins] r41923 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070405222541.D8CF110077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 00:25:41 2007 New Revision: 41923 Added: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py - copied unchanged from r41921, z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dataprovider.py Removed: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dataprovider.py Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Log: Renamed dataprovider to more normal name 'model' Deleted: /z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dataprovider.py ============================================================================== --- /z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dataprovider.py Fri Apr 6 00:25:41 2007 +++ (empty file) @@ -1,327 +0,0 @@ -import re -import os -from ohm import persist -from ohm import server -from ohm import descriptors -from ohm import lildav -from ohm.validators import LineConverter -from formencode import validators -from formencode.foreach import ForEach -from formencode.compound import All - -domain_re = re.compile(r'^localhost|[a-z][a-z0-9.\-]+\.[a-z]+$', re.I) - -default_rule_xml = '''\ - - - - - - -''' - -marker = '' -abstract_rule_xml = '''\ - -MARKER - - - - - -'''.replace('MARKER', marker) - -default_standardrules = '''\ - - - - - - - - -''' - -class DataProvider(object): - - def __init__(self, dir): - self.dir = dir - if not os.path.exists(self.dir): - print 'Creating data directory %s' % self.dir - os.makedirs(self.dir) - - def alias_fn(self, alias): - return os.path.join(self.dir, alias+'-alias.txt') - - def domain(self, domain_name, aliases=()): - """ - Return the domain object for the given domain_name. The - domain name given may not be the canonical domain. - """ - domain_name = self.normalize(domain_name) - alias_fn = self.alias_fn(domain_name) - if os.path.exists(alias_fn): - f = open(alias_fn, 'rb') - alias = f.read() - f.close() - aliases = list(aliases) - if alias in aliases: - raise ValueError( - "Infinite alias loop (found: %s -> %s)" - % (aliases, alias)) - aliases.append(alias) - return self.domain(alias, aliases=aliases) - dir = os.path.join(self.dir, domain_name) - if not os.path.exists(dir): - os.mkdir(dir) - return DomainDataProvider(self, domain_name, dir) - - def normalize(self, domain_name): - """ - Normalize a domain name (lower case, no funny characters). - """ - domain_name = domain_name.strip().lower() - if not domain_re.search(domain_name): - raise ValueError( - 'Bad domain name: %r' % domain_name) - return domain_name - - def add_alias(self, alias, domain): - """ - Add an alias domain ``alias`` which points to ``domain``. - There must not be an existing alias. - """ - alias = self.normalize(alias) - domain = self.normalize(domain) - fn = self.alias_fn(alias) - assert not os.path.exists(fn), ( - "Alias file already exists (%r)" % fn) - f = open(fn, 'wb') - f.write(domain) - f.close() - - def remove_alias(self, alias, domain=None): - """ - Remove the alias for the domain ``alias``. ``domain`` (if - given) is what the alias should currently be pointing to (we - won't remove the alias then if it points elsewhere). - """ - alias = self.normalize(alias) - if domain: - domain = self.normalize(domain) - fn = self.alias_fn(alias) - if not os.path.exists(fn): - raise ValueError( - "Alias does not exist (not file %r)" % fn) - if domain: - f = open(fn, 'rb') - existing = f.read().strip() - f.close() - if existing != domain: - raise ValueError( - "Tried to remove alias from %s->%s, but %s actually " - "points to %s" % (alias, domain, alias, existing)) - os.unlink(fn) - - def rename_domain(self, old_domain_name, new_domain_name): - """ - Rename the old domain to the new domain, returning the new domain - object. - """ - old_domain_name = self.normalize(old_domain_name) - new_domain_name = self.normalize(new_domain_name) - old_dir = os.path.join(self.dir, old_domain_name) - new_dir = os.path.join(self.dir, new_domain_name) - assert os.path.exists(old_dir) - assert not os.path.exists(new_dir) - os.rename(old_dir, new_dir) - return self.domain(new_domain_name) - -class DomainDataProvider(object): - - def __init__(self, provider, domain_name, base_dir): - self._domain_name = domain_name - self.base_dir = base_dir - self.provider = provider - - def __repr__(self): - return '<%s %s for %s in %r>' % ( - self.__class__.__name__, - hex(id(self)), - self._domain_name, - self.base_dir) - - def initialize(self): - for dir in [self.base_dir, self.rule_dir, - self.static_dir]: - if not os.path.exists(dir): - os.mkdir(dir) - for filename, content in [ - ('rule.xml', default_rule_xml), - ('standardrules.xml', default_standardrules)]: - filename = os.path.join(self.rule_dir, filename) - if not os.path.exists(filename): - f = open(filename, 'w') - f.write(content) - f.close() - - @property - def initialized(self): - return hasattr(self, 'remote_uris') - - @property - def rule_dir(self): - return os.path.join(self.base_dir, 'rules') - - @property - def static_dir(self): - return os.path.join(self.base_dir, 'static') - - remote_uris = descriptors.json_converter( - persist.file_property('remote_uris.txt')) - theme_uri = persist.file_property('theme_uri.txt') - redirects = descriptors.json_converter( - persist.file_property('redirects.txt', default='[]')) - - additional_request_headers = descriptors.json_converter( - persist.file_property('additional_request_headers.txt', default='[]')) - - def aliases__rename(self, old_aliases, new_aliases): - dropped = list(old_aliases) - added = list(new_aliases) - for new_alias in new_aliases: - if new_alias in dropped: - dropped.remove(new_alias) - added.remove(new_alias) - for alias in dropped: - self.provider.remove_alias(alias, self.domain) - for alias in added: - self.provider.add_alias(alias, self.domain) - - aliases = descriptors.watcher( - descriptors.line_converter(persist.file_property('aliases.txt', default='')), - after_watcher=aliases__rename) - - def set_rule_file(self, filename, content): - filename = os.path.join(self.rule_dir, filename) - f = open(filename, 'wb') - f.write(content) - f.close() - - def domain__get(self): - return self._domain_name - - def domain__set(self, new_domain): - aliases = self.aliases - old_domain = self.domain - new_obj = self.provider.rename_domain(self.domain, new_domain) - # Clone new object as self: - self.__dict__.update(new_obj.__dict__) - for alias in aliases: - self.provider.remove_alias(alias, domain=old_domain) - self.provider.add_alias(alias, new_domain) - - domain = property(domain__get, domain__set) - -class DomainValidator(validators.Regex): - regex = r'^[a-z0-9\-]+|[a-z0-9][a-z0-9\-\.\_]*\.[a-z]+$' - -class ListDictValidator(validators.FancyValidator): - required_keys = () - optional_keys = () - trail_slash_keys = () - sort_key = None - - def validate_python(self, value, state=None): - # Should be like [{'path': path, 'remote_uri': uri, 'comment': str}] - # comment is optional - try: - assert isinstance(value, list), 'Must be list' - for d in value: - keys = d.keys() - for key in self.required_keys: - assert key in d, "Must have key %r in %r" % d - assert isinstance(d[key], basestring), ( - '%s must be string (not %r)' % (key, d[key])) - keys.remove(key) - for key in self.optional_keys: - if key not in d: - continue - assert isinstance(d[key], basestring), ( - '%s must be string (not %r)' % (key, d[key])) - keys.remove(key) - assert not keys, ( - "Key(s) not allowed: %s in %r" - % (', '.join(map(repr, keys)), d)) - except AssertionError, e: - raise validators.Invalid(str(e), value, state) - - def _to_python(self, value, state=None): - for d in value: - for key in self.trail_slash_keys: - if key not in d: - continue - if not d[key].endswith('/'): - d[key] += '/' - if self.sort_key: - value = sorted(value, key=self.sort_key) - return value - -class RemoteURIValidator(ListDictValidator): - required_keys = ('path', ) - optional_keys = ('comment', 'headers', 'remote_uri') - trail_slash_keys = ('path', 'remote_uri') - def sort_key(self, item): - return -len(item['path']) - -class RewriteValidator(ListDictValidator): - required_keys = ('rewrite', ) - # @@: Really we should require one of path or prefix - optional_keys = ('path', 'prefix', 'comment') - trail_slash_keys = ('prefix', 'rewrite') - -class HeaderValidator(validators.FancyValidator): - header_name_re = re.compile(r'^[a-zA-Z][a-zA-Z0-9-]+$') - header_value_re = re.compile(r'^[^\n\r\000-\037\200-\377]*$') - - def validate_python(self, value, state=None): - assert isinstance(value, list), ( - "Value must be a list (not %r)" % value) - for item in list: - assert isinstance(value, (list, tuple)), ( - "Item must be a list or tuple (not %r)" % value) - assert len(item) == 2, ( - "Item must be a two-tuple (not %r)" % value) - header, value = item - assert header_name_re.search(header), ( - "Header name invalid: %r" % header) - assert header_value_re.search(value), ( - "Header value invalid: %r" % value) - -class ProviderApp(server.ApplicationWrapper): - - theme_uri = server.Setter( - validator=validators.URL()) - domain = server.Setter( - validator=DomainValidator()) - aliases = server.Setter( - validator=All(ForEach(DomainValidator()), LineConverter())) - remote_uris = server.JSONSetter( - validator=RemoteURIValidator()) - redirects = server.JSONSetter( - validator=RewriteValidator()) - additional_request_headers = server.JSONSetter( - validator=HeaderValidator()) - - @server.appfactory() - def rules(self): - if not os.path.exists(self.rule_dir): - os.makedirs(self.rule_dir) - return lildav.LilDAV(self.rule_dir) - - @server.appfactory() - def static(self): - if not os.path.exists(self.static_dir): - os.makedirs(self.static_dir) - return lildav.LilDAV(self.static_dir) Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 00:25:41 2007 @@ -10,7 +10,7 @@ from wsgifilter import proxyapp from wsgifilter import relocateresponse from deliverance.wsgimiddleware import DeliveranceMiddleware -from dvhoster.dataprovider import DataProvider, ProviderApp +from dvhoster.model import DataProvider, ProviderApp from dvhoster import current_environ from dvhoster.debuginterp import Renderer from dvhoster.util import load_func From ianb at codespeak.net Fri Apr 6 00:32:22 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 00:32:22 +0200 (CEST) Subject: [z3-checkins] r41924 - z3/deliverance/DeliveranceVHoster/trunk/docs Message-ID: <20070405223222.825F210077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 00:32:21 2007 New Revision: 41924 Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py Log: typo Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py Fri Apr 6 00:32:21 2007 @@ -26,7 +26,7 @@ Also requires a ``default_theme_uri`` setting. """ domain = domain_info.domain - regex = app_conf.get('domain_regex', r'^(.*)\.openplans\.org$') + domain_regex = app_conf.get('domain_regex', r'^(.*)\.openplans\.org$') match = re.search(domain_regex, domain, re.I) if not match: # Don't try to set up domains we don't recognize From ianb at codespeak.net Fri Apr 6 00:33:35 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 00:33:35 +0200 (CEST) Subject: [z3-checkins] r41925 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070405223335.6704310077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 00:33:34 2007 New Revision: 41925 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py Log: Rename some classes in dvhost.model to make more sense. Add some docstrings too Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 00:33:34 2007 @@ -10,7 +10,7 @@ from wsgifilter import proxyapp from wsgifilter import relocateresponse from deliverance.wsgimiddleware import DeliveranceMiddleware -from dvhoster.model import DataProvider, ProviderApp +from dvhoster.model import DomainInfoSet, DomainInfoApp from dvhoster import current_environ from dvhoster.debuginterp import Renderer from dvhoster.util import load_func @@ -42,7 +42,7 @@ should_theme_uri = load_func(should_theme_uri, 'should_theme_uri') self.should_theme_uri = should_theme_uri self.app_conf = app_conf - self.provider = DataProvider(data_dir) + self.provider = DomainInfoSet(data_dir) self.rewrite_links = asbool(app_conf.get('rewrite_links', True)) if app_conf.get('clean_environ_headers_regex'): self.clean_environ_headers_regex = re.compile(app_conf['clean_environ_headers_regex']) @@ -69,7 +69,7 @@ path_info = norm_path(environ.get('PATH_INFO', '')) if path_info.startswith('/.deliverance'): path_info_pop(environ) - subapp = ProviderApp(domain_info) + subapp = DomainInfoApp(domain_info) return subapp(environ, start_response) if domain_info.domain != domain: Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py Fri Apr 6 00:33:34 2007 @@ -43,7 +43,16 @@ ''' -class DataProvider(object): +class DomainInfoSet(object): + """ + Represents a set of DomainInfo objects. + + Concretely, all domain info is in one directory, and this + represents that directory. It creates new DomainInfo objects and + does some management around them. + + This also handles aliases -- domains that point to other domains. + """ def __init__(self, dir): self.dir = dir @@ -52,12 +61,31 @@ os.makedirs(self.dir) def alias_fn(self, alias): + """ + Filename where an alias is kept. + + Each alias is just the domain name plus ``-alias.txt``, and + contains the name of another domain name. + """ return os.path.join(self.dir, alias+'-alias.txt') + def normalize(self, domain_name): + """ + Normalize a domain name (lower case, no funny characters). + """ + domain_name = domain_name.strip().lower() + if not domain_re.search(domain_name): + raise ValueError( + 'Bad domain name: %r' % domain_name) + return domain_name + def domain(self, domain_name, aliases=()): """ - Return the domain object for the given domain_name. The - domain name given may not be the canonical domain. + Return the domain object for the given domain_name. + + The domain name given may not be the canonical domain (that + is, you might get back and object with a different value for + ``.domain``). """ domain_name = self.normalize(domain_name) alias_fn = self.alias_fn(domain_name) @@ -75,17 +103,7 @@ dir = os.path.join(self.dir, domain_name) if not os.path.exists(dir): os.mkdir(dir) - return DomainDataProvider(self, domain_name, dir) - - def normalize(self, domain_name): - """ - Normalize a domain name (lower case, no funny characters). - """ - domain_name = domain_name.strip().lower() - if not domain_re.search(domain_name): - raise ValueError( - 'Bad domain name: %r' % domain_name) - return domain_name + return DomainInfo(self, domain_name, dir) def add_alias(self, alias, domain): """ @@ -138,12 +156,15 @@ os.rename(old_dir, new_dir) return self.domain(new_domain_name) -class DomainDataProvider(object): +class DomainInfo(object): + """ + Represents the information about a single domain. + """ - def __init__(self, provider, domain_name, base_dir): + def __init__(self, domain_set, domain_name, base_dir): self._domain_name = domain_name self.base_dir = base_dir - self.provider = provider + self.domain_set = domain_set def __repr__(self): return '<%s %s for %s in %r>' % ( @@ -153,6 +174,9 @@ self.base_dir) def initialize(self): + """ + Create the directory and basic files for this domain. + """ for dir in [self.base_dir, self.rule_dir, self.static_dir]: if not os.path.exists(dir): @@ -195,9 +219,9 @@ dropped.remove(new_alias) added.remove(new_alias) for alias in dropped: - self.provider.remove_alias(alias, self.domain) + self.domain_set.remove_alias(alias, self.domain) for alias in added: - self.provider.add_alias(alias, self.domain) + self.domain_set.add_alias(alias, self.domain) aliases = descriptors.watcher( descriptors.line_converter(persist.file_property('aliases.txt', default='')), @@ -215,12 +239,12 @@ def domain__set(self, new_domain): aliases = self.aliases old_domain = self.domain - new_obj = self.provider.rename_domain(self.domain, new_domain) + new_obj = self.domain_set.rename_domain(self.domain, new_domain) # Clone new object as self: self.__dict__.update(new_obj.__dict__) for alias in aliases: - self.provider.remove_alias(alias, domain=old_domain) - self.provider.add_alias(alias, new_domain) + self.domain_set.remove_alias(alias, domain=old_domain) + self.domain_set.add_alias(alias, new_domain) domain = property(domain__get, domain__set) @@ -299,7 +323,11 @@ assert header_value_re.search(value), ( "Header value invalid: %r" % value) -class ProviderApp(server.ApplicationWrapper): +class DomainInfoApp(server.ApplicationWrapper): + """ + Provides the ``/.deliverance/`` web/REST API for a single + DomainInfo object. + """ theme_uri = server.Setter( validator=validators.URL()) From ianb at codespeak.net Fri Apr 6 00:37:53 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 00:37:53 +0200 (CEST) Subject: [z3-checkins] r41926 - z3/deliverance/DeliveranceVHoster/trunk/docs Message-ID: <20070405223753.BA96A10077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 00:37:53 2007 New Revision: 41926 Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py Log: Print out warning/error if domain_regex doesn't match Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/openplans_hooks.py Fri Apr 6 00:37:53 2007 @@ -30,6 +30,7 @@ match = re.search(domain_regex, domain, re.I) if not match: # Don't try to set up domains we don't recognize + print 'The domain %r cannot be recognized by the domain_regex %r' % (domain, domain_regex) return project = match.group(1) remote_uris = [ From ianb at codespeak.net Fri Apr 6 00:42:58 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 00:42:58 +0200 (CEST) Subject: [z3-checkins] r41927 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070405224258.3974110077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 00:42:57 2007 New Revision: 41927 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py Log: Be more lax in domain name checking (e.g., allow 'flow') Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py Fri Apr 6 00:42:57 2007 @@ -9,7 +9,7 @@ from formencode.foreach import ForEach from formencode.compound import All -domain_re = re.compile(r'^localhost|[a-z][a-z0-9.\-]+\.[a-z]+$', re.I) +domain_re = re.compile(r'^[a-z][a-z0-9-]+|[a-z][a-z0-9.\-]+\.[a-z]+$', re.I) default_rule_xml = '''\ From ianb at codespeak.net Fri Apr 6 00:59:12 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 00:59:12 +0200 (CEST) Subject: [z3-checkins] r41928 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070405225912.5BD0610077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 00:59:11 2007 New Revision: 41928 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/wsgiapp.py Log: Add the HTTPException middleware, so that pieces can raise exceptions (e.g., redirect) Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/wsgiapp.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/wsgiapp.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/wsgiapp.py Fri Apr 6 00:59:11 2007 @@ -5,12 +5,13 @@ from paste.registry import RegistryManager from wsgifilter import proxyapp from paste.exceptions.errormiddleware import ErrorMiddleware - +from paste.httpexceptions import HTTPExceptionHandler from dvhoster.dispatcher import DeliveranceDispatcher def make_app(global_conf, **app_conf): """Create a WSGI application and return it""" app = DeliveranceDispatcher(app_conf) + app = HTTPExceptionHandler(app) app = RecursiveMiddleware(app) app = RegistryManager(app) debug = app_conf['debug'] = asbool(app_conf.get('debug', global_conf.get('debug'))) From ianb at codespeak.net Fri Apr 6 00:59:53 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 00:59:53 +0200 (CEST) Subject: [z3-checkins] r41929 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070405225953.8F36910077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 00:59:52 2007 New Revision: 41929 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Log: Refactoring of the giant __call__ method; separating out the fetching of domain_info Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 00:59:52 2007 @@ -53,15 +53,7 @@ if 'paste.registry' in environ: environ['paste.registry'].register(current_environ, environ) self.clean_environ_headers(environ) - domain = environ['HTTP_HOST'] - if ':' in domain: - domain = domain.split(':', 1)[0] - domain = self.provider.normalize(domain) - domain_info = self.provider.domain(domain) - if not domain_info.initialized: - domain_info.initialize() - if self.init_domain: - self.init_domain(domain_info, self.app_conf) + domain_info = self.get_domain_info(environ) environ['dvhoster.domain_info'] = domain_info environ['dvhoster.base_url'] = construct_url( environ, with_query_string=False, @@ -71,19 +63,6 @@ path_info_pop(environ) subapp = DomainInfoApp(domain_info) return subapp(environ, start_response) - - if domain_info.domain != domain: - # We got an alias - assert domain in domain_info.aliases, ( - "Domain %r not found in aliases %r" - % (domain, domain_info.aliases)) - new_environ = environ.copy() - new_environ['HTTP_HOST'] = domain_info.domain - new_url = construct_url(new_environ) - exc = httpexceptions.HTTPMovedPermanently( - headers=[('Location', new_url)], - comment='Redirecting to canonical domain') - return exc(environ, start_response) if path_info.startswith('/_rules'): path_info_pop(environ) @@ -195,3 +174,29 @@ if self.clean_environ_headers_regex.search(key): # @@: Should log this del environ[key] + + def get_domain_info(self, environ): + domain = environ['HTTP_HOST'] + if ':' in domain: + domain = domain.split(':', 1)[0] + domain = self.provider.normalize(domain) + domain_info = self.provider.domain(domain) + if not domain_info.initialized: + domain_info.initialize() + if self.init_domain: + self.init_domain(domain_info, self.app_conf) + + if domain_info.domain != domain: + # We got an alias + assert domain in domain_info.aliases, ( + "Domain %r not found in aliases %r" + % (domain, domain_info.aliases)) + new_environ = environ.copy() + new_environ['HTTP_HOST'] = domain_info.domain + new_url = construct_url(new_environ) + exc = httpexceptions.HTTPMovedPermanently( + headers=[('Location', new_url)], + comment='Redirecting to canonical domain') + raise exc + + return domain_info From ianb at codespeak.net Fri Apr 6 05:35:48 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 05:35:48 +0200 (CEST) Subject: [z3-checkins] r41930 - z3/deliverance/DeliveranceVHoster/trunk/tests Message-ID: <20070406033548.22E4310077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 05:35:46 2007 New Revision: 41930 Modified: z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Log: Remove DebugHeaders printing Modified: z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Fri Apr 6 05:35:46 2007 @@ -147,11 +147,11 @@ def setup_module(module): - static_app = DebugHeaders(StaticURLParser(os.path.join(os.path.dirname(__file__), - 'test-static'))) + static_app = StaticURLParser(os.path.join(os.path.dirname(__file__), + 'test-static')) wsgi_intercept.add_wsgi_intercept('wsgify.org', 80, lambda : static_app) def teardown_module(module): wsgi_intercept.remove_wsgi_intercept('wsgify.org', 80) - + From ianb at codespeak.net Fri Apr 6 05:36:03 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 05:36:03 +0200 (CEST) Subject: [z3-checkins] r41931 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070406033603.8289610078@code0.codespeak.net> Author: ianb Date: Fri Apr 6 05:36:02 2007 New Revision: 41931 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Log: Refactoring that changes no functionality Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 05:36:02 2007 @@ -27,6 +27,12 @@ class DeliveranceDispatcher(object): + """ + Main WSGI application that handles dispatching to the API handler, + redirects, static files, and finding the proper remote URI to + proxy to. + """ + def __init__(self, app_conf): data_dir = app_conf['data_dir'] init_domain = app_conf.get('init_domain') @@ -50,6 +56,9 @@ self.clean_environ_headers_regex = None def __call__(self, environ, start_response): + """ + WSGI interface + """ if 'paste.registry' in environ: environ['paste.registry'].register(current_environ, environ) self.clean_environ_headers(environ) @@ -58,7 +67,7 @@ environ['dvhoster.base_url'] = construct_url( environ, with_query_string=False, path_info='') - path_info = norm_path(environ.get('PATH_INFO', '')) + environ['PATH_INFO'] = path_info = norm_path(environ.get('PATH_INFO', '')) if path_info.startswith('/.deliverance'): path_info_pop(environ) subapp = DomainInfoApp(domain_info) @@ -70,68 +79,26 @@ subapp = StaticURLParser( domain_info.rule_dir) return subapp(environ, start_response) - - static_path = os.path.join(domain_info.static_dir, - path_info.lstrip('/')) - static_fn = self.find_file(static_path) - if static_fn is not None: - # Explicit override of a file - app = FileApp(static_fn) - return app(environ, start_response) - try: - remote_uris = domain_info.remote_uris - except AttributeError: + + static_app = self.find_static_file(domain_info, path_info) + if static_app: + return static_app(environ, start_response) + + if not domain_info.initialized: exc = httpexceptions.HTTPNotFound( "This domain has not yet been configured (no remote " "URI has been configured for %s)" % domain_info.domain) return exc(environ, start_response) - remote_uri = None - current_uri = construct_url(environ) - for redirect_info in domain_info.redirects: - if 'path' in redirect_info: - if redirect_info['path'] == path_info: - new = redirect_info['rewrite'] - new = urlparse.urljoin(current_uri, redirect_info['rewrite']) - exc = httpexceptions.HTTPMovedPermanently( - headers=[('Location', new)]) - return exc(environ, start_response) - elif 'prefix' in redirect_info: - assert redirect_info['prefix'].endswith('/') - new_uri = None - if path_info == redirect_info['prefix'][:-1]: - new_uri = redirect_info['rewrite'] + '/' - elif path_info.startswith(redirect_info['prefix']): - new_uri = redirect_info['rewrite'] - if not new_uri.endswith('/'): - new_uri += '/' - new_uri += path_info[len(redirect_info['prefix']):] - if new_uri is not None: - new_uri = urlparse.urljoin(current_uri, new_uri) - exc = httpexceptions.HTTPMovedPermanently( - headers=[('Location', new_uri)]) - return exc(environ, start_response) - for header_name, header_value in domain_info.additional_request_headers: - header_name = 'HTTP_%s' % header_name.upper().replace('-', '_') - environ[header_name] = header_value + redir_app = self.check_redirects( + path_info, domain_info, environ) + if redir_app: + return redir_app(environ, start_response) - for remote_uri_info in remote_uris: - path = str(remote_uri_info['path']) - if not path.endswith('/'): - path += '/' - if path_info + '/' == path: - exc = httpexceptions.HTTPMovedPermanently( - headers=[('location', construct_url(environ, path_info=path_info+'/'))]) - return exc(environ, start_response) - if path_info.startswith(path): - # Found a match - remote_uri = remote_uri_info.get('remote_uri') - environ['SCRIPT_NAME'] += path[:-1] - environ['PATH_INFO'] = path_info[len(path)-1:] - if self.find_remote_uri: - remote_uri = self.find_remote_uri(remote_uri, remote_uri_info, environ, self.app_conf) - break + self.set_additional_headers(domain_info, environ) + + remote_uri = self.match_remote_uri(path_info, domain_info, environ) if not remote_uri: exc = httpexceptions.HTTPNotFound( "No URL is mapped to %r (somewhat oddly); only %s prefixes " @@ -139,6 +106,7 @@ % (path_info, ', '.join([repr(r['path']) for r in remote_uris]))) return exc(environ, start_response) + should_theme_uri = True if self.should_theme_uri: should_theme_uri = self.should_theme_uri( @@ -146,12 +114,15 @@ app = proxyapp.ForcedProxy( remote=remote_uri, force_host=True) + if self.rewrite_links: app = relocateresponse.RelocateMiddleware( app, old_href=remote_uri) + rule_uri = construct_url( environ, with_query_string=False, path_info='/_rules/rule.xml') + if should_theme_uri: app = DeliveranceMiddleware( app, @@ -160,14 +131,25 @@ renderer=Renderer) return app(environ, start_response) - def find_file(self, path): - if os.path.isdir(path): - path = os.path.join(path, 'index.html') - if os.path.exists(path): - return path + def find_static_file(self, domain_info, path_info): + """ + If the request matches a static file, returns a WSGI + application to handle that file. Else returns None. + """ + static_path = os.path.join(domain_info.static_dir, + path_info.lstrip('/')) + if os.path.isdir(static_path): + static_path = os.path.join( + static_path, 'index.html') + if os.path.exists(static_path): + return FileApp(static_path) return None def clean_environ_headers(self, environ): + """ + Remove any request headers that overlap with internal request + headers (as configured with ``clean_environ_headers_regex``). + """ if not self.clean_environ_headers_regex: return for key in environ.keys(): @@ -176,6 +158,12 @@ del environ[key] def get_domain_info(self, environ): + """ + Get the DomainInfo object associated with the request. + + May raise a redirect if the requested host is not the + canonical host. + """ domain = environ['HTTP_HOST'] if ':' in domain: domain = domain.split(':', 1)[0] @@ -200,3 +188,75 @@ raise exc return domain_info + + def check_redirects(self, path_info, domain_info, environ): + """ + Checks if the request matches any configured redirection. If + so, return a WSGI application that does the redirect; else + return None. + """ + current_uri = construct_url(environ) + for redirect_info in domain_info.redirects: + if 'path' in redirect_info: + if redirect_info['path'] == path_info: + new = redirect_info['rewrite'] + new = urlparse.urljoin(current_uri, redirect_info['rewrite']) + exc = httpexceptions.HTTPMovedPermanently( + headers=[('Location', new)]) + return exc + elif 'prefix' in redirect_info: + assert redirect_info['prefix'].endswith('/') + new_uri = None + if path_info == redirect_info['prefix'][:-1]: + new_uri = redirect_info['rewrite'] + '/' + elif path_info.startswith(redirect_info['prefix']): + new_uri = redirect_info['rewrite'] + if not new_uri.endswith('/'): + new_uri += '/' + new_uri += path_info[len(redirect_info['prefix']):] + if new_uri is not None: + new_uri = urlparse.urljoin(current_uri, new_uri) + exc = httpexceptions.HTTPMovedPermanently( + headers=[('Location', new_uri)]) + return exc + return None + + def set_additional_headers(self, domain_info, environ): + """ + Set any additional headers that the domain has been configured + to send. + """ + for header_name, header_value in domain_info.additional_request_headers: + header_name = 'HTTP_%s' % header_name.upper().replace('-', '_') + environ[header_name] = header_value + + def match_remote_uri(self, path_info, domain_info, environ): + """ + Look at the ``domain_info.remote_uris`` and what remote_uri + this request matches, returning that remote_uri (or None if + nothing is found). + + Also calls ``find_remote_uri`` if configured. + """ + remote_uri = None + + for remote_uri_info in domain_info.remote_uris: + path = str(remote_uri_info['path']) + if not path.endswith('/'): + path += '/' + if path_info + '/' == path: + exc = httpexceptions.HTTPMovedPermanently( + headers=[('location', construct_url(environ, path_info=path_info+'/'))]) + raise exc + if path_info.startswith(path): + # Found a match + remote_uri = remote_uri_info.get('remote_uri') + environ['SCRIPT_NAME'] += path[:-1] + environ['PATH_INFO'] = path_info[len(path)-1:] + if self.find_remote_uri: + remote_uri = self.find_remote_uri(remote_uri, remote_uri_info, environ, self.app_conf) + return remote_uri + if self.find_remote_uri: + # Last change for find_remote_uri to do something + remote_uri = self.find_remote_uri(remote_uri, None, environ, self.app_conf) + return remote_uri From ianb at codespeak.net Fri Apr 6 05:45:30 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 05:45:30 +0200 (CEST) Subject: [z3-checkins] r41932 - in z3/deliverance/DeliveranceVHoster/trunk: dvhoster tests Message-ID: <20070406034530.CAF4A10077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 05:45:30 2007 New Revision: 41932 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Log: Added test for remote_uris header setting (plus implementation) Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 05:45:30 2007 @@ -255,6 +255,12 @@ environ['PATH_INFO'] = path_info[len(path)-1:] if self.find_remote_uri: remote_uri = self.find_remote_uri(remote_uri, remote_uri_info, environ, self.app_conf) + if remote_uri_info.get('headers'): + for header_name, header_value in remote_uri_info['headers'].items(): + header_name = header_name.upper().replace('-', '_') + if not header_name.startswith('HTTP_'): + header_name = 'HTTP_' + header_name + environ[str(header_name)] = str(header_value) return remote_uri if self.find_remote_uri: # Last change for find_remote_uri to do something Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py Fri Apr 6 05:45:30 2007 @@ -255,6 +255,7 @@ required_keys = () optional_keys = () trail_slash_keys = () + validate_keys = {} sort_key = None def validate_python(self, value, state=None): @@ -266,14 +267,12 @@ keys = d.keys() for key in self.required_keys: assert key in d, "Must have key %r in %r" % d - assert isinstance(d[key], basestring), ( - '%s must be string (not %r)' % (key, d[key])) + self.validate_key(key, d[key], state) keys.remove(key) for key in self.optional_keys: if key not in d: continue - assert isinstance(d[key], basestring), ( - '%s must be string (not %r)' % (key, d[key])) + self.validate_key(key, d[key], state) keys.remove(key) assert not keys, ( "Key(s) not allowed: %s in %r" @@ -281,6 +280,14 @@ except AssertionError, e: raise validators.Invalid(str(e), value, state) + def validate_key(self, key, value, state): + if key in self.validate_keys: + if self.validate_keys[key]: + self.validate_keys[key].validate_python(value, state) + else: + assert isinstance(value, basestring), ( + '%s must be string (not %r)' % (key, value)) + def _to_python(self, value, state=None): for d in value: for key in self.trail_slash_keys: @@ -296,6 +303,8 @@ required_keys = ('path', ) optional_keys = ('comment', 'headers', 'remote_uri') trail_slash_keys = ('path', 'remote_uri') + validate_keys = dict( + headers=None) def sort_key(self, item): return -len(item['path']) Modified: z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Fri Apr 6 05:45:30 2007 @@ -68,13 +68,13 @@ data = ''' [{"path": "/bar", "remote_uri": "http://wsgify.org/blah", "comment": "x"}, - {"path": "/", "remote_uri": "http://wsgify.org/"} + {"path": "/", "remote_uri": "http://wsgify.org/"}, + {"path": "/testme", "headers": {"X-Test-Me": "testme"}, "remote_uri": "http://wsgify.org:9999"} ] ''' - #{"path": "/testme", "headers": {"X-Test-Me": "testme"} put('/.deliverance/remote_uris', data) - # It gets normalized, so it doesn't actually stay the same: + # It gets normalized, so it doesn't actually stay quite the same: #assert app.get('/.deliverance/remote_uris').body == data res = app.get('/bar', status=301) @@ -87,6 +87,9 @@ ## So instead we just get a page we know works: res = app.get('/bar/index.html', status=200) + res = app.get('/testme/', status=200) + res.mustcontain("HTTP_X_TEST_ME: 'testme'") + data = ''' [{"path": "/test1.html", "rewrite": "/test1"}, {"prefix": "/test2", "rewrite": "/test3", "comment": "rename"}, @@ -147,10 +150,12 @@ def setup_module(module): + from paste.script import testapp static_app = StaticURLParser(os.path.join(os.path.dirname(__file__), 'test-static')) wsgi_intercept.add_wsgi_intercept('wsgify.org', 80, lambda : static_app) + wsgi_intercept.add_wsgi_intercept('wsgify.org', 9999, lambda : testapp.TestApplication(text=True)) def teardown_module(module): wsgi_intercept.remove_wsgi_intercept('wsgify.org', 80) From ianb at codespeak.net Fri Apr 6 05:51:02 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 05:51:02 +0200 (CEST) Subject: [z3-checkins] r41933 - z3/deliverance/DeliveranceVHoster/trunk/docs Message-ID: <20070406035102.3094A10077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 05:51:01 2007 New Revision: 41933 Added: z3/deliverance/DeliveranceVHoster/trunk/docs/index.txt (contents, props changed) Log: index of docs Added: z3/deliverance/DeliveranceVHoster/trunk/docs/index.txt ============================================================================== --- (empty file) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/index.txt Fri Apr 6 05:51:01 2007 @@ -0,0 +1,19 @@ +DeliveranceVHoster +================== + +Introduction +------------ + +This is a product to do a virtual-hosting frontend with `Deliverance +`_ theming, and multiple +configurable backends (e.g., a Zope HTTP server, a WSGI HTTP server, +an Apache/PHP HTTP server). + +It is configurable through a REST API. + +Documentation +------------- + +* `REST API and Configuration `_ +* `Hooks for programmatic domain setup `_ +* `Configuration of the server `_ From ianb at codespeak.net Fri Apr 6 07:51:22 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 07:51:22 +0200 (CEST) Subject: [z3-checkins] r41934 - z3/deliverance/DeliveranceVHoster/trunk/docs Message-ID: <20070406055122.0657810077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 07:51:21 2007 New Revision: 41934 Added: z3/deliverance/DeliveranceVHoster/trunk/docs/hooks.txt (contents, props changed) z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt (contents, props changed) Log: Added some docs Added: z3/deliverance/DeliveranceVHoster/trunk/docs/hooks.txt ============================================================================== --- (empty file) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/hooks.txt Fri Apr 6 07:51:21 2007 @@ -0,0 +1,78 @@ +Hooks +===== + +There are three hooks that you can configure (in the ``.ini`` file) to +programmatically handle parts of DeliveranceVHoster's handling of +requests. + +Each is configured by pointing to a Python file (ending in ``.py``), +giving a module name, or pointing directly to an object. If you point +to a Python file or module then the system will look for an object +(usually a function) with the name of the setting (e.g., +``init_domain``). You can point all the settings to the same file or +module, and have different functions in that file. + +All the hook functions get access to the configuration (referred to as +``app_conf``), a dictionary of all the settings in the +DeliveranceVHoster section of the ``.ini`` file. You can put +arbitrary settings in that file, to pass them through to the hooks. + +``init_domain`` +--------------- + +When a new virtual host is visited, a ``dvhoster.model.DomainInfo`` +object is created for that virtual host. After it is created, this +function is called, and can setup any initial settings. To see what +attributes are available see the ``dvhoster.model`` file; generally +they are attributes with the same names as used in the `REST API +`_. + +The function should look like:: + + def init_domain(domain_info, app_conf): + domain_info.remote_uris = [] + +Or potentially something more useful. + +``find_remote_uri`` +------------------- + +This function is called after the ``remote_uris`` have been searched +and a match found (or potentially no match found). It returns the +remote_uri that should be used. It should look like:: + + def find_remote_uri(remote_uri, remote_uri_info, environ, + app_conf): + if remote_uri is not None: + # It was set explicitly in the remote_uris setting + return remote_uri + if remote_uri_info is None: + # There was no match found at all; fail: + return None + # You can make changes to environ if you want, or read it + # for more information. + # This would actually be a little silly, though: + return remote_uri_info['comment'] + +``should_theme_uri`` +-------------------- + +This function is called and should return True (the request should be +themed) or False (the request should be left alone). It should look +like:: + + def should_theme_uri(remote_uri, environ, app_conf): + if 'python' in environ.get('HTTP_USER_AGENT'): + # probably a request with urllib or something + return False + return True + +Exceptions +---------- + +In any of these methods you may raise exceptions from +`paste.httpexceptions +`_. Perhaps, +for instance, a redirection is in order (``HTTPMovedPermanently``). +Or an unauthorized request (``HTTPForbidden``). + Added: z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt ============================================================================== --- (empty file) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt Fri Apr 6 07:51:21 2007 @@ -0,0 +1,113 @@ +REST API +======== + +DeliveranceVHoster configuration is done through a REST API. This API +is made up of a series of `resources +`_ that can be read +with GET and set with PUT. + +The API is available at ``/.deliverance/`` under the domain. Each +domain presents the configuration for that domain. So when you access +``http://foo.com/.deliverance/aliases`` you are accessing the +``aliases`` parameter for ``foo.com``. There's no global +configuration presented though this API, only domain-specific +configuration. + +Resources +--------- + +Each of these resources can be read with GET and set with PUT. + +``/.deliverance/static/*`` + + Static files. You can use PUT to update or save files, and DELETE + to remove them. Use MKCOL to create directories. When a file is + put here it becomes a global override. So if you PUT a file at + ``/.deliverance/static/styles/style.css``, then a request to + ``/styles/style.css`` will return that file. Any other request to + ``/styles/*`` will be handled normally. ``index.html`` can be used + to override directory indexes. + + You can GET directories here, which will give you directory listings + in simple HTML. PROPLIST (from WebDAV) is not supported. + +``/.deliverance/rules/rules.xml`` + + The rules file. This is an XML document. A file + ``/.deliverance/rules/standardrules.xml`` is always set, which are + standard rules applicable to HTML. You can include these rules with + XInclude. + + Similar to ``/.deliverance/static`` you can create directories here + and add arbitrary files. The files appear in ``/_rules`` (e.g., + ``/_rules/rules.xml``). + +``/.deliverance/domain`` + + The main domain, like ``foo.com`` in the example. A simple string. + You can PUT a new value here to rename the entire site. If you + rename a site, the original name automatically becomes an alias. + + You can GET this value to see if you are accessing the canonical + domain or an alias; for ``/.deliverance`` requests the redirection + does not occur. + +``/.deliverance/aliases`` + + This is a list of domains that are aliases for this domain. The + list text, one domain per line. Any request to one of these aliases + will be redirected to the canonical/main domain. + +``/.deliverance/theme_uri`` + + The URI of the theme. This is a simple string, with no encoding. + +``/.deliverance/remote_uris`` + + A JSON structure, looking like:: + + [{"path": "", "remote_uri": "http://localhost:8080"}, + {"path": "/tasks", "remote_uri": "http://localhost:9090"}] + + This means that requests to ``/tasks/*`` will be forwarded to the + server at ``http://localhost:9090``, while all other requests + (anything under the null path ``""``) will be forwarded to + ``http://localhost:8080``. Requests ``/tasks`` will be redirected + to ``/tasks/``. + + Each setting can also have a key ``headers``, which is a dictionary + of headers to set on the request. These are header the application + being forwarded to can consume. Additionally a ``comment`` key can + hold any value. + + ``remote_uri`` is optional. Using `hooks `_ you can set + a remote URI programmatically without setting it in domain-specific + configuration. + + This resource must be set for a new virtual host to be usable. You + can do this with a `hook `_, or otherwise all requests + (except to this configuration API) will fail. + +``/.deliverance/addition_request_headers`` + + A JSON structure looking like:: + + [["X-Foo", "Bar"], + ["X-Foo-2", "Bar 2"]] + + Each item is a header (name and value) that will be set on all + requests to this domain. + +``/.deliverance/redirects`` + + A JSON structure, looking like:: + + [{"path": "/foo.html", "rewrite": "/foo/", + "comment": "from old site"}, + {"prefix": "/blog/index.cgi/", "rewrite": "/blog/"}] + + Each item represents a redirection, typically set to support an old + set of URLs after a transition. Each item must contain either + ``path`` (a fixed path to redirect) or ``prefix`` (a directory to + redirect). It is redirected to the value of ``rewrite``. You may + optionally include a comment. From ianb at codespeak.net Fri Apr 6 17:03:30 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 17:03:30 +0200 (CEST) Subject: [z3-checkins] r41938 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070406150330.2897510077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 17:03:29 2007 New Revision: 41938 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Log: Set remote_uris headers before calling find_remote_uri Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 17:03:29 2007 @@ -253,14 +253,14 @@ remote_uri = remote_uri_info.get('remote_uri') environ['SCRIPT_NAME'] += path[:-1] environ['PATH_INFO'] = path_info[len(path)-1:] - if self.find_remote_uri: - remote_uri = self.find_remote_uri(remote_uri, remote_uri_info, environ, self.app_conf) if remote_uri_info.get('headers'): for header_name, header_value in remote_uri_info['headers'].items(): header_name = header_name.upper().replace('-', '_') if not header_name.startswith('HTTP_'): header_name = 'HTTP_' + header_name environ[str(header_name)] = str(header_value) + if self.find_remote_uri: + remote_uri = self.find_remote_uri(remote_uri, remote_uri_info, environ, self.app_conf) return remote_uri if self.find_remote_uri: # Last change for find_remote_uri to do something From ianb at codespeak.net Fri Apr 6 17:19:47 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 17:19:47 +0200 (CEST) Subject: [z3-checkins] r41939 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070406151947.2953C10077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 17:19:46 2007 New Revision: 41939 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Log: Move rewrite_links into the should_theme_uri conditional block, just like theming Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 17:19:46 2007 @@ -115,15 +115,15 @@ remote=remote_uri, force_host=True) - if self.rewrite_links: - app = relocateresponse.RelocateMiddleware( - app, old_href=remote_uri) - - rule_uri = construct_url( - environ, with_query_string=False, - path_info='/_rules/rule.xml') - if should_theme_uri: + if self.rewrite_links: + app = relocateresponse.RelocateMiddleware( + app, old_href=remote_uri) + + rule_uri = construct_url( + environ, with_query_string=False, + path_info='/_rules/rule.xml') + app = DeliveranceMiddleware( app, theme_uri=domain_info.theme_uri, From ianb at codespeak.net Fri Apr 6 17:20:26 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 17:20:26 +0200 (CEST) Subject: [z3-checkins] r41940 - z3/deliverance/DeliveranceVHoster/trunk/docs Message-ID: <20070406152026.2A9C610077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 17:20:25 2007 New Revision: 41940 Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/hooks.txt Log: note last commit in docs Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/hooks.txt ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/docs/hooks.txt (original) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/hooks.txt Fri Apr 6 17:20:25 2007 @@ -67,6 +67,9 @@ return False return True +If you return false it will also shortcut the link-rewriting phase +(which can also be problematic). + Exceptions ---------- From ianb at codespeak.net Fri Apr 6 17:33:52 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 17:33:52 +0200 (CEST) Subject: [z3-checkins] r41941 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070406153352.C27EB10077@code0.codespeak.net> Author: ianb Date: Fri Apr 6 17:33:52 2007 New Revision: 41941 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Log: Save remote_uri in the environment; use that in the error formatter, instead of redoing the remote_uri matching (which is more complex now) Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py Fri Apr 6 17:33:52 2007 @@ -16,23 +16,12 @@ error_container.attrib['style'] = self.error_style if not current_environ.get('dvhoster.has_errors'): current_environ['dvhoster.has_errors'] = True - domain_info = current_environ['dvhoster.domain_info'] - remote_uris = domain_info.remote_uris - path_info = current_environ['PATH_INFO'] - for remote_info in remote_uris: - if (path_info.startswith(remote_info['path']) - or not path_info and remote_info['path'] == '/'): - remote = remote_info['remote_uri'] - break - else: - assert 0, ( - "Nothing in remote_uris (%r) matches PATH_INFO=%r" - % (remote_uris, path_info)) - remote += current_environ.get('PATH_INFO', '') + remote_uri = current_environ['dvhoster.remote_uri'] + remote_uri += current_environ.get('PATH_INFO', '') if current_environ.get('QUERY_STRING'): - remote += '?' + current_environ['QUERY_STRING'] + remote_uri += '?' + current_environ['QUERY_STRING'] link = etree.Element('a') - link.attrib['href'] = remote + link.attrib['href'] = remote_uri link.attrib['target'] = '_blank' link.text = 'View original content source' container = etree.Element('div') Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 17:33:52 2007 @@ -107,6 +107,8 @@ for r in remote_uris]))) return exc(environ, start_response) + environ['dvhoster.remote_uri'] = remote_uri + should_theme_uri = True if self.should_theme_uri: should_theme_uri = self.should_theme_uri( From ianb at codespeak.net Fri Apr 6 18:27:05 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 18:27:05 +0200 (CEST) Subject: [z3-checkins] r41942 - in z3/deliverance/DeliveranceVHoster/trunk: . dvhoster tests Message-ID: <20070406162705.A70D610078@code0.codespeak.net> Author: ianb Date: Fri Apr 6 18:27:04 2007 New Revision: 41942 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py z3/deliverance/DeliveranceVHoster/trunk/setup.py z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py Log: Fixed the error message generator, which had problems when recursive errors occurred. Requires update from Paste Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py Fri Apr 6 18:27:04 2007 @@ -14,12 +14,26 @@ error_container = etree.Element('div') error_container.attrib['style'] = self.error_style - if not current_environ.get('dvhoster.has_errors'): - current_environ['dvhoster.has_errors'] = True - remote_uri = current_environ['dvhoster.remote_uri'] - remote_uri += current_environ.get('PATH_INFO', '') - if current_environ.get('QUERY_STRING'): - remote_uri += '?' + current_environ['QUERY_STRING'] + environ = None + for match_environ in current_environ._object_stack(): + if 'dvhoster.remote_uri' in match_environ: + # This was the original request that we are interested in + environ = match_environ + break + else: + warnings.warn( + "No environment with dvhoster.remote_uri can be found, " + "which is unexpected") + # But lets not actually keep the error message from working + assert 0 + return error + environ = current_environ._current_obj() + if not environ.get('dvhoster.has_errors'): + environ['dvhoster.has_errors'] = True + remote_uri = environ.get('dvhoster.remote_uri', '') + remote_uri += environ.get('PATH_INFO', '') + if environ.get('QUERY_STRING'): + remote_uri += '?' + environ['QUERY_STRING'] link = etree.Element('a') link.attrib['href'] = remote_uri link.attrib['target'] = '_blank' Modified: z3/deliverance/DeliveranceVHoster/trunk/setup.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/setup.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/setup.py Fri Apr 6 18:27:04 2007 @@ -8,7 +8,7 @@ #author_email="", #url="", install_requires=[ - 'Paste>1.1.1', + 'Paste==dev,>=1.3.1dev-r6419', 'PasteScript', 'Deliverance', 'WSGIFilter', Modified: z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py Fri Apr 6 18:27:04 2007 @@ -65,6 +65,13 @@ res = app.get('/', extra_environ={'HTTP_X_NO_THEME': '1', 'HTTP_HOST': 'foo3.openplans.org'}) assert 'Plone' not in res +def test_error_message(): + # This will give us some content that can't be themed, testing the + # error messages: + setup_module(None, text=False) + res = app.get('/', extra_environ={'HTTP_HOST': 'foo4.openplans.org'}) + res.mustcontain('View original content source') + def setup_module(module, text=True): from paste.script import testapp app = testapp.TestApplication(text=text) From ianb at codespeak.net Fri Apr 6 19:37:59 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 19:37:59 +0200 (CEST) Subject: [z3-checkins] r41943 - in z3/deliverance/DeliveranceVHoster/trunk: docs dvhoster tests Message-ID: <20070406173759.C21451006F@code0.codespeak.net> Author: ianb Date: Fri Apr 6 19:37:59 2007 New Revision: 41943 Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Log: Added POST methods for remote_uris setting Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt (original) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt Fri Apr 6 19:37:59 2007 @@ -88,6 +88,12 @@ can do this with a `hook `_, or otherwise all requests (except to this configuration API) will fail. +``/.deliverance/remote_uris?{add or remove}`` + + You can post to one of these resources to add or remove items from + the list, without effecting other parts of the list. Each takes a + JSON list. With ``?add`` the list extends the current items. + ``/.deliverance/addition_request_headers`` A JSON structure looking like:: Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py Fri Apr 6 19:37:59 2007 @@ -4,7 +4,7 @@ from ohm import server from ohm import descriptors from ohm import lildav -from ohm.validators import LineConverter +from ohm.validators import LineConverter, JSONConverter from formencode import validators from formencode.foreach import ForEach from formencode.compound import All @@ -338,6 +338,34 @@ DomainInfo object. """ + def add_remote_uris(self, body): + validator = RemoteURIValidator() + body = validator.to_python(body) + self.remote_uris = validator.to_python(self.remote_uris + body) + + def remove_remote_uris(self, body): + cur = self.remote_uris + for item in body: + path = item['path'] + # Normalization that normally RemoteURIValidator does: + if not path.endswith('/'): + item['path'] = path = path + '/' + for existing in cur: + if existing['path'] == path: + # Found a match + break + else: + raise ValueError( + "There's no current remote_uri matching %r " + "(of paths: %s)" % + (path, ', '.join([repr(e['path']) for e in cur]))) + for item in body: + path = item['path'] + for existing in cur[:]: + if existing['path'] == path: + cur.remove(existing) + self.remote_uris = cur + theme_uri = server.Setter( validator=validators.URL()) domain = server.Setter( @@ -345,7 +373,9 @@ aliases = server.Setter( validator=All(ForEach(DomainValidator()), LineConverter())) remote_uris = server.JSONSetter( - validator=RemoteURIValidator()) + validator=RemoteURIValidator(), + POST={'add': (JSONConverter(), add_remote_uris), + 'remove': (JSONConverter(), remove_remote_uris)}) redirects = server.JSONSetter( validator=RewriteValidator()) additional_request_headers = server.JSONSetter( Modified: z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Fri Apr 6 19:37:59 2007 @@ -9,6 +9,7 @@ setattr(httplib.HTTPConnection, attr, getattr(wsgi_intercept.WSGI_HTTPConnection, attr).im_func) from wsgifilter.proxyapp import DebugHeaders +from simplejson import loads as json_loads data_filename = os.path.join(os.path.dirname(__file__), 'test-data') wsgi_app = make_app({}, data_dir=data_filename) @@ -90,6 +91,20 @@ res = app.get('/testme/', status=200) res.mustcontain("HTTP_X_TEST_ME: 'testme'") + # Now lets try adding and removing via POST + app.post('/.deliverance/remote_uris?add', + '[{"path": "/testpost", "remote_uri": "http://blah.com"}]', + status=204) + res = app.get('/.deliverance/remote_uris') + res.mustcontain('testpost') + app.post('/.deliverance/remote_uris?remove', + '[{"path": "/testpost"}]', + status=204) + res = app.get('/.deliverance/remote_uris') + data = json_loads(res.body) + for item in data: + assert not item['path'].startswith('/testpost') + data = ''' [{"path": "/test1.html", "rewrite": "/test1"}, {"prefix": "/test2", "rewrite": "/test3", "comment": "rename"}, From ianb at codespeak.net Fri Apr 6 20:03:48 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 20:03:48 +0200 (CEST) Subject: [z3-checkins] r41944 - in z3/deliverance/DeliveranceVHoster/trunk: docs dvhoster tests Message-ID: <20070406180348.3571810050@code0.codespeak.net> Author: ianb Date: Fri Apr 6 20:03:47 2007 New Revision: 41944 Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Log: Added POST API for redirects Modified: z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt (original) +++ z3/deliverance/DeliveranceVHoster/trunk/docs/rest-api.txt Fri Apr 6 20:03:47 2007 @@ -117,3 +117,8 @@ ``path`` (a fixed path to redirect) or ``prefix`` (a directory to redirect). It is redirected to the value of ``rewrite``. You may optionally include a comment. + +``/.deliverance/redirects?{add or remove}`` + + Like ``remote_uris`` you can POST to these locations to add and + remove redirects, without reseting all redirects. Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/model.py Fri Apr 6 20:03:47 2007 @@ -6,6 +6,7 @@ from ohm import lildav from ohm.validators import LineConverter, JSONConverter from formencode import validators +from formencode.api import Invalid from formencode.foreach import ForEach from formencode.compound import All @@ -355,10 +356,11 @@ # Found a match break else: - raise ValueError( + raise Invalid( "There's no current remote_uri matching %r " "(of paths: %s)" % - (path, ', '.join([repr(e['path']) for e in cur]))) + (path, ', '.join([repr(e['path']) for e in cur])), + body, None) for item in body: path = item['path'] for existing in cur[:]: @@ -366,6 +368,49 @@ cur.remove(existing) self.remote_uris = cur + def add_redirects(self, body): + validator = RewriteValidator() + body = validator.to_python(body) + self.redirects = validator.to_python(self.redirects + body) + + def remove_redirects(self, body): + cur = self.redirects + for item in body: + if item.get('path'): + path = item['path'] + for existing in cur: + if existing.get('path') == path: + break + else: + raise Invalid( + "There's no current redirect matching path=%r" + % path, body, None) + else: + prefix = item['prefix'] + if not prefix.endswith('/'): + item['prefix'] = prefix = prefix + '/' + for existing in cur: + if existing.get('prefix') == prefix: + break + else: + raise Invalid( + "There's no current redirect matching prefix=%r" + % prefix, body, None) + for item in body: + if item.get('path'): + path = item['path'] + for existing in cur[:]: + if existing.get('path') == path: + cur.remove(existing) + break + else: + prefix = item['prefix'] + for existing in cur[:]: + if existing.get('prefix') == prefix: + cur.remove(existing) + break + self.redirects = cur + theme_uri = server.Setter( validator=validators.URL()) domain = server.Setter( @@ -377,7 +422,9 @@ POST={'add': (JSONConverter(), add_remote_uris), 'remove': (JSONConverter(), remove_remote_uris)}) redirects = server.JSONSetter( - validator=RewriteValidator()) + validator=RewriteValidator(), + POST={'add': (JSONConverter(), add_redirects), + 'remove': (JSONConverter(), remove_redirects)}) additional_request_headers = server.JSONSetter( validator=HeaderValidator()) Modified: z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/tests/test_functional_api.py Fri Apr 6 20:03:47 2007 @@ -111,6 +111,26 @@ {"path": "/other.html", "rewrite": "http://otherexample.com/other.html"}] ''' put('/.deliverance/redirects', data) + + app.post('/.deliverance/redirects?add', + '[{"path": "/something-special", "rewrite": "http://whatever.com"}]', + status=204) + res = app.get('/.deliverance/redirects') + res.mustcontain('whatever.com', 'otherexample.com') + # Let's try a bad request: + app.post('/.deliverance/redirects?remove', + '[{"path": "/blahblah"}]', status=400) + # Then a good one: + app.post('/.deliverance/redirects?remove', + '[{"path": "/something-special"}]', + status=204) + res = app.get('/.deliverance/redirects') + res.mustcontain('otherexample.com') + data = json_loads(res.body) + for item in data: + if not item.get('path'): + continue + assert not item.get('path').startswith('/something-special') data = 'some data!' put('/.deliverance/static/data.html', data) From ianb at codespeak.net Fri Apr 6 21:22:09 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 21:22:09 +0200 (CEST) Subject: [z3-checkins] r41945 - in z3/deliverance/DeliveranceVHoster/trunk: . dvhoster Message-ID: <20070406192209.307A310053@code0.codespeak.net> Author: ianb Date: Fri Apr 6 21:22:07 2007 New Revision: 41945 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/cli.py z3/deliverance/DeliveranceVHoster/trunk/setup.py Log: make cli a proper script: Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/cli.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/cli.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/cli.py Fri Apr 6 21:22:07 2007 @@ -125,13 +125,18 @@ else: print "[X] unknown command" +cli_usage = """\ +usage: %prog domain [options] + +Gives you an interactive console to run commands against the server""" + def main(argv=None): if argv is None: argv = sys.argv - usage = "usage: %prog domain [options]" - parser = optparse.OptionParser(usage=usage) - parser.add_option('-F','--force-host', + parser = optparse.OptionParser(usage=cli_usage) + parser.add_option('-F','--force-host', + metavar="HOST", help="specify the administration server to contact", dest='force_host', default=None) Modified: z3/deliverance/DeliveranceVHoster/trunk/setup.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/setup.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/setup.py Fri Apr 6 21:22:07 2007 @@ -26,7 +26,12 @@ entry_points=""" [paste.app_factory] main=dvhoster:make_app + [paste.app_install] main=paste.script.appinstall:Installer + + [console_scripts] + dvhoster-api = dvhoster.cli:main + """, ) From ianb at codespeak.net Fri Apr 6 22:16:37 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Fri, 6 Apr 2007 22:16:37 +0200 (CEST) Subject: [z3-checkins] r41946 - z3/deliverance/DeliveranceVHoster/trunk/dvhoster Message-ID: <20070406201637.901FB10060@code0.codespeak.net> Author: ianb Date: Fri Apr 6 22:16:33 2007 New Revision: 41946 Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Log: Make rewrite_links false by default Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Fri Apr 6 22:16:33 2007 @@ -49,7 +49,7 @@ self.should_theme_uri = should_theme_uri self.app_conf = app_conf self.provider = DomainInfoSet(data_dir) - self.rewrite_links = asbool(app_conf.get('rewrite_links', True)) + self.rewrite_links = asbool(app_conf.get('rewrite_links', False)) if app_conf.get('clean_environ_headers_regex'): self.clean_environ_headers_regex = re.compile(app_conf['clean_environ_headers_regex']) else: From ltucker at codespeak.net Fri Apr 6 23:16:12 2007 From: ltucker at codespeak.net (ltucker at codespeak.net) Date: Fri, 6 Apr 2007 23:16:12 +0200 (CEST) Subject: [z3-checkins] r41947 - z3/deliverance/trunk/deliverance Message-ID: <20070406211612.3244110053@code0.codespeak.net> Author: ltucker Date: Fri Apr 6 23:16:11 2007 New Revision: 41947 Modified: z3/deliverance/trunk/deliverance/resource_fetcher.py z3/deliverance/trunk/deliverance/wsgimiddleware.py Log: oops. I lied, use whole inbound environment in all fetches by default for now Modified: z3/deliverance/trunk/deliverance/resource_fetcher.py ============================================================================== --- z3/deliverance/trunk/deliverance/resource_fetcher.py (original) +++ z3/deliverance/trunk/deliverance/resource_fetcher.py Fri Apr 6 23:16:11 2007 @@ -127,13 +127,13 @@ return body class ExternalResourceFetcher(object): - def __init__(self, uri, headers_only=False): + def __init__(self, in_environ, uri, headers_only=False): self.uri = uri url_chunks = urlparse.urlsplit(uri) loc = urlparse.urlsplit(uri) - self.environ = {} + self.environ = in_environ.copy() if headers_only: self.environ['REQUEST_METHOD'] = 'HEAD' Modified: z3/deliverance/trunk/deliverance/wsgimiddleware.py ============================================================================== --- z3/deliverance/trunk/deliverance/wsgimiddleware.py (original) +++ z3/deliverance/trunk/deliverance/wsgimiddleware.py Fri Apr 6 23:16:11 2007 @@ -368,7 +368,7 @@ return InternalResourceFetcher(environ, uri[len(internalBaseURL):], self.app) else: - return ExternalResourceFetcher(uri) + return ExternalResourceFetcher(environ, uri) def get_resource_uris(self, rules): From ianb at codespeak.net Sat Apr 7 00:08:14 2007 From: ianb at codespeak.net (ianb at codespeak.net) Date: Sat, 7 Apr 2007 00:08:14 +0200 (CEST) Subject: [z3-checkins] r41948 - in z3/deliverance/DeliveranceVHoster/trunk: dvhoster tests Message-ID: <20070406220814.758E910053@code0.codespeak.net> Author: ianb Date: Sat Apr 7 00:08:13 2007 New Revision: 41948 Added: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/logcreate.py (contents, props changed) Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py z3/deliverance/DeliveranceVHoster/trunk/tests/test_init_func.py Log: Added logging of error messages Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/debuginterp.py Sat Apr 7 00:08:13 2007 @@ -2,6 +2,7 @@ from lxml import etree from paste.request import construct_url from dvhoster import current_environ +import re class Renderer(PyRenderer): @@ -11,7 +12,6 @@ error = super(PyRenderer, self).format_error(message, rule, elts) if error is None: return None - error_container = etree.Element('div') error_container.attrib['style'] = self.error_style environ = None @@ -25,15 +25,14 @@ "No environment with dvhoster.remote_uri can be found, " "which is unexpected") # But lets not actually keep the error message from working - assert 0 return error - environ = current_environ._current_obj() - if not environ.get('dvhoster.has_errors'): - environ['dvhoster.has_errors'] = True - remote_uri = environ.get('dvhoster.remote_uri', '') - remote_uri += environ.get('PATH_INFO', '') - if environ.get('QUERY_STRING'): - remote_uri += '?' + environ['QUERY_STRING'] + top_environ = current_environ._current_obj() + remote_uri = environ.get('dvhoster.remote_uri', '') + remote_uri += environ.get('PATH_INFO', '') + if environ.get('QUERY_STRING'): + remote_uri += '?' + environ['QUERY_STRING'] + if not top_environ.get('dvhoster.has_errors'): + top_environ['dvhoster.has_errors'] = True link = etree.Element('a') link.attrib['href'] = remote_uri link.attrib['target'] = '_blank' @@ -41,8 +40,36 @@ container = etree.Element('div') container.append(link) error_container.append(container) + self.log_error(environ, error, remote_uri) error_container.append(error) return error_container - - + def log_error(self, environ, error, remote_uri): + logger = environ['dvhoster.logger'] + cur_url = environ['dvhoster.original_url'] + error = el_as_string(error) + error = '\n'.join([' '+l for l in error.splitlines()]) + logger.warn('Error in matching rule in URL %s content at %s:\n%s' + % (cur_url, remote_uri, error)) + +_space_re = re.compile(r'[ ][ ]+') + +def el_as_string(el): + s = [] + _el_to_string(el, s) + s = ''.join(s) + s = s.replace('\t', ' ') + s = _space_re.sub(' ', s) + return s.strip() + +def _el_to_string(el, s): + if el.text: + s.append(el.text) + if el.tag == 'br': + s.append('\n') + for sub_el in el.getchildren(): + _el_to_string(sub_el, s) + if sub_el.tail: + s.append(sub_el.tail) + if el.tag == 'a' and el.attrib.get('href'): + s.append(' (%s) ' % el.attrib['href']) Modified: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py ============================================================================== --- z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py (original) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/dispatcher.py Sat Apr 7 00:08:13 2007 @@ -14,6 +14,7 @@ from dvhoster import current_environ from dvhoster.debuginterp import Renderer from dvhoster.util import load_func +from dvhoster.logcreate import make_logger def norm_path(urlpath): if not urlpath: @@ -54,6 +55,8 @@ self.clean_environ_headers_regex = re.compile(app_conf['clean_environ_headers_regex']) else: self.clean_environ_headers_regex = None + logger = make_logger(app_conf.get('logger'), 'dvhoster') + self.logger = logger def __call__(self, environ, start_response): """ @@ -67,6 +70,8 @@ environ['dvhoster.base_url'] = construct_url( environ, with_query_string=False, path_info='') + environ['dvhoster.original_url'] = construct_url(environ) + environ['dvhoster.logger'] = self.logger environ['PATH_INFO'] = path_info = norm_path(environ.get('PATH_INFO', '')) if path_info.startswith('/.deliverance'): path_info_pop(environ) Added: z3/deliverance/DeliveranceVHoster/trunk/dvhoster/logcreate.py ============================================================================== --- (empty file) +++ z3/deliverance/DeliveranceVHoster/trunk/dvhoster/logcreat