[shpy-commit] r2884 - in shpy/trunk/dist: . shpy/net shpy/ui_pygame
hpk@codespeak.net
hpk@codespeak.net
Fri, 23 Jan 2004 15:27:20 +0100 (MET)
Author: hpk
Date: Fri Jan 23 15:27:20 2004
New Revision: 2884
Removed:
shpy/trunk/dist/shpy/net/structure.py
Modified:
shpy/trunk/dist/shpy/net/gateway.py
shpy/trunk/dist/shpy/net/inputoutput.py
shpy/trunk/dist/shpy/net/register.py
shpy/trunk/dist/shpy/ui_pygame/decorate.py
shpy/trunk/dist/shpy/ui_pygame/ui_pygame.py
shpy/trunk/dist/startserver.py
Log:
- reorganized Gateway-classes
- got rid of pickle everywhere
- now the startserver.py executes a single line of code only
and does not unpickle anything.
- execution server is now a Popen2-process by default (uff!)
(this is actually difficult to debug ...)
Modified: shpy/trunk/dist/shpy/net/gateway.py
==============================================================================
--- shpy/trunk/dist/shpy/net/gateway.py (original)
+++ shpy/trunk/dist/shpy/net/gateway.py Fri Jan 23 15:27:20 2004
@@ -1,6 +1,5 @@
import threading, struct, Queue, select, StringIO, socket
-
-from unittest2.tool import dyncode
+import thread, time
class Gateway:
def __init__(self, io, ns = None):
@@ -39,6 +38,47 @@
def exec_remote(self, source):
self.outgoing.put(source)
+ def import_remote(self, modsource, modname, reload=False):
+ l = []
+ if not reload:
+ l.append("""
+ try:
+ import sys
+ %(modname)s = sys.modules[%(modname)r]
+ except KeyError:
+ """)
+ l.append("""
+ import new, sys
+ %(modname)s = new.module(%(modname)r)
+ exec %(modsource)r in vars(%(modname)s)
+ sys.modules[%(modname)r] = %(modname)s
+ """)
+ self.exec_remote("".join(l) % locals())
+
+ def sendchanges(self, *structures):
+ lines = [representstructure(s) for s in structures]
+ arg = [getstructureid(s) for s in structures]
+ lines.append('''gateway.propagate(%r)''' % (arg,))
+ self.exec_remote('\n'.join(lines))
+
+ def propagate(self, structids):
+ import shared
+ for c in shared.getclientlist():
+ if c is not self:
+ try:
+ c.sendchanges(*[getstructure(structid) for structid in structids])
+ except:
+ import traceback
+ traceback.print_exc()
+
+ def exec_remote_synchronous(self, source, timeout=10):
+ self.synchronizationevent = threading.Event()
+ source += '''\ngateway.exec_remote("gateway.synchronizationevent.set()")'''
+ self.exec_remote(source)
+ self.synchronizationevent.wait(timeout)
+ if not self.synchronizationevent.isSet():
+ raise IOError, "communication timeout"
+
class SockQueueReader(threading.Thread):
def __init__(self, queue):
@@ -63,14 +103,23 @@
self.queue = queue
def run(self):
+ try:
+ import autopath
+ except ImportError:
+ pass
+ try:
+ from unittest2.tool.dyncode import makecode
+ except ImportError:
+ print "please give me *some* way to access dyncode"
+ raise
while 1:
source = self.queue.get()
if source is None:
break
try:
print "executing source", source[:50]
- co = dyncode.makecode(source)
- exec co in self.gateway.ns
+ co = makecode(source)
+ exec co in globals(), self.gateway.ns
except:
import traceback, sys
l = traceback.format_exception(*sys.exc_info())
@@ -94,3 +143,91 @@
finally:
print 'SockQueueSender is leaving'
self.gateway.running = 0
+
+
+# ____________________________________________________________
+# XXX make a nicer namespace organization
+
+class Structure:
+
+ def __init__(self, **attributes):
+ self.__dict__.update(attributes)
+
+
+ATOMIC = (int, float, str, unicode, type(None))
+
+def represent(obj, ref):
+ if isinstance(obj, ATOMIC):
+ return repr(obj)
+ elif isinstance(obj, dict):
+ items = ['%s: %s' % (represent(key,ref), represent(value,ref))
+ for key, value in obj.items()]
+ return '{%s}' % (', '.join(items))
+ elif isinstance(obj, tuple):
+ items = [represent(item,ref)+',' for item in obj]
+ return '(%s)' % (''.join(items))
+ elif isinstance(obj, list):
+ items = [represent(item,ref) for item in obj]
+ return '[%s]' % (','.join(items))
+ elif obj.__class__.__name__ == 'Structure': # ahem
+ ref[obj] = True
+ return 'getstructure(%r)' % getstructureid(obj)
+ else:
+ raise TypeError, 'cannot represent %r' % (obj,)
+
+
+structure2id = {}
+id2structure = {}
+directionflag = ">"
+maplock = thread.allocate_lock()
+hostname = socket.getfqdn()
+
+def getstructureid(structure):
+ maplock.acquire()
+ try:
+ try:
+ return structure2id[structure]
+ except KeyError:
+ structid = '%s%d@%s@%s' % (directionflag, len(structure2id), hostname, time.time())
+ structure2id[structure] = structid
+ id2structure[structid] = structure
+ return structid
+ finally:
+ maplock.release()
+
+def getstructure(structid):
+ print "getstructure(%r)" % structid
+ maplock.acquire()
+ try:
+ try:
+ return id2structure[structid]
+ except KeyError:
+ structure = Structure()
+ structure2id[structure] = structid
+ id2structure[structid] = structure
+ return structure
+ finally:
+ maplock.release()
+
+def representstructure(structure, ref=None):
+ data = {}
+ for key, value in structure.__dict__.items():
+ if not key.startswith('_'):
+ data[key] = value
+ if ref is None:
+ ref = {}
+ return 'getstructure(%r).__dict__ = %s' % (
+ getstructureid(structure), represent(data, ref))
+
+def representeverything(root):
+ lines = []
+ pending = [root]
+ seen = {root: True}
+ for structure in pending:
+ ref = {}
+ lines.append(representstructure(structure, ref))
+ for referenced in ref:
+ if referenced not in seen:
+ pending.append(referenced)
+ seen[referenced] = True
+ return lines
Modified: shpy/trunk/dist/shpy/net/inputoutput.py
==============================================================================
--- shpy/trunk/dist/shpy/net/inputoutput.py (original)
+++ shpy/trunk/dist/shpy/net/inputoutput.py Fri Jan 23 15:27:20 2004
@@ -1,16 +1,33 @@
-import socket
+import socket, os
def connect(addr):
- if isinstance(addr, str):
+ """ return an IO object for the given addr which can be either
+ popen=fullcommandline or
+ tcp=host:port ("tcp=" is optional)
+ """
+ if '=' not in addr:
+ protocol = 'tcp'
+ else:
+ protocol, addr = addr.split('=', 1)
+
+ if protocol == 'tcp':
host, port = addr.split(':')
port = int(port)
addr = (host, port)
- assert len(addr) == 2
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- sock.connect(addr)
- return SocketIO(sock)
-
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.connect(addr)
+ return SocketIO(sock)
+
+ if protocol == 'popen':
+ outfile, infile = os.popen2(addr)
+ return Popen2IO(outfile, infile)
+
+ raise ValueError, "protocol %r unknown" % (protocol,)
+
+
class SocketIO:
+ server_stmt = "io = SocketIO(clientsock)"
+
def __init__(self, sock):
self.sock = sock
try:
@@ -36,4 +53,33 @@
def close(self):
self.sock.shutdown(2)
+class Popen2IO:
+ server_stmt = """
+import sys
+io = Popen2IO(sys.stdout, sys.stdin)
+sys.stdout = sys.stderr = open('/tmp/debug.log', 'a', 0)
+print '='*60
+"""
+
+ def __init__(self, outfile, infile):
+ self.outfile = outfile
+ self.infile = infile
+ def read(self, bytes):
+ #import sys
+ #print >> sys.stderr, "reading..."
+ s = self.infile.read(bytes)
+ #print >> sys.stderr, "read: %r" % s
+ if len(s) < bytes:
+ raise EOFError
+ return s
+
+ def write(self, data):
+ #import sys
+ #print >> sys.stderr, "writing: %r" % data
+ self.outfile.write(data)
+ self.outfile.flush()
+
+ def close(self):
+ self.outfile.close()
+ self.infile.close()
Modified: shpy/trunk/dist/shpy/net/register.py
==============================================================================
--- shpy/trunk/dist/shpy/net/register.py (original)
+++ shpy/trunk/dist/shpy/net/register.py Fri Jan 23 15:27:20 2004
@@ -1,83 +1,42 @@
import autopath, os
-import inspect, socket, pickle, thread
-from shpy.net import gateway, structure, inputoutput
-
+import inspect, socket, thread
+from shpy.net import gateway, inputoutput
+from unittest2.tool import dyncode
+dyncode.installhacks()
class InstallableGateway(gateway.Gateway):
def __init__(self, addr, ns = None):
+ io = inputoutput.connect(addr)
source = [inspect.getsource(inputoutput),
inspect.getsource(gateway),
+ 'directionflag = "<"',
]
- source.append('thread = Gateway(SocketIO(clientsock))')
+ source.append(io.server_stmt)
+ source.append('gateway = Gateway(io)')
source = "\n".join(source)
- sockio = inputoutput.connect(addr)
- sockio.write(pickle.dumps(source, bin=True))
- gateway.Gateway.__init__(self, sockio, ns)
-
- def import_remote(self, filename, modname, reload=False):
- filename = os.path.join(autopath.thisdir, filename)
- modsource = open(filename, 'r').read()
- l = []
- if not reload:
- l.append("""
- try:
- import sys
- %(modname)s = sys.modules[%(modname)r]
- except KeyError:
- """)
- l.append("""
- import new, sys
- %(modname)s = new.module(%(modname)r)
- exec %(modsource)r in vars(%(modname)s)
- sys.modules[%(modname)r] = %(modname)s
- """)
- self.exec_remote("".join(l) % locals())
-
+ io.write('%r\n' % source)
+ gateway.Gateway.__init__(self, io, ns)
class ServerGateway(InstallableGateway):
+ propagate_callback = None
+
def registerclient(self, reload=False):
- self.ns['getstructure'] = structure.getstructure
- self.import_remote('shared.py', 'shared', reload)
+ filename = os.path.join(autopath.thisdir, 'shared.py')
+ modsource = open(filename, 'r').read()
+ self.import_remote(modsource, 'shared', reload)
self.exec_remote("shared.registerclient(gateway)")
# import the root
- self.waiting_for_root = thread.allocate_lock()
- self.waiting_for_root.acquire()
- self.exec_remote(inspect.getsource(structure))
- self.exec_remote('''
+ self.exec_remote_synchronous('''
lines = representeverything(shared.root)
- lines.append('root = getstructure(%r)' % getstructureid(shared.root))
- gateway.exec_remote('gateway.setstructure(%r)' % (lines,))
+ lines.append('gateway.root = getstructure(%r)' % getstructureid(shared.root))
+ gateway.exec_remote('\\n'.join(lines))
''')
- self.waiting_for_root.acquire()
- del self.waiting_for_root
return self.root
- def setstructure(self, lines):
- l = {}
- exec '\n'.join(lines) in vars(structure), l
- if 'root' in l:
- self.root = l['root']
- self.waiting_for_root.release()
-
- def notifychanges(self, *structures):
- lines = [structure.representstructure(s) for s in structures]
- lines.append('''if 1:
- for c in shared.getclientlist():
- if c is not gateway:
- try:
- rerepresent = c.ns['representstructure']
- relines = []
- for structid in %r:
- obj = getstructure(structid)
- relines.append(rerepresent(obj))
- relines.append("terminal.postrepaintevent()")
- c.exec_remote('\\n'.join(relines))
- except:
- import traceback
- traceback.print_exc()
- ''' % [structure.getstructureid(s) for s in structures])
- self.exec_remote('\n'.join(lines))
+ def propagate(self, structids):
+ if self.propagate_callback is not None:
+ self.propagate_callback()
class ExecGateway(InstallableGateway):
Deleted: /shpy/trunk/dist/shpy/net/structure.py
==============================================================================
--- /shpy/trunk/dist/shpy/net/structure.py Fri Jan 23 15:27:20 2004
+++ (empty file)
@@ -1,85 +0,0 @@
-import socket, thread, time
-
-class Structure:
-
- def __init__(self, **attributes):
- self.__dict__.update(attributes)
-
-
-
-ATOMIC = (int, float, str, unicode, type(None))
-
-def represent(obj, ref):
- if isinstance(obj, ATOMIC):
- return repr(obj)
- elif isinstance(obj, dict):
- items = ['%s: %s' % (represent(key,ref), represent(value,ref))
- for key, value in obj.items()]
- return '{%s}' % (', '.join(items))
- elif isinstance(obj, tuple):
- items = [represent(item,ref)+',' for item in obj]
- return '(%s)' % (''.join(items))
- elif isinstance(obj, list):
- items = [represent(item,ref) for item in obj]
- return '[%s]' % (','.join(items))
- elif obj.__class__.__name__ == 'Structure': # ahem
- ref[obj] = True
- return 'getstructure(%r)' % getstructureid(obj)
- else:
- raise TypeError, 'cannot represent %r' % (obj,)
-
-
-structure2id = {}
-id2structure = {}
-maplock = thread.allocate_lock()
-hostname = socket.getfqdn()
-
-def getstructureid(structure):
- maplock.acquire()
- try:
- try:
- return structure2id[structure]
- except KeyError:
- structid = '%d@%s@%s' % (len(structure2id), hostname, time.time())
- structure2id[structure] = structid
- id2structure[structid] = structure
- return structid
- finally:
- maplock.release()
-
-def getstructure(structid):
- print "getstructure(%r)" % structid
- maplock.acquire()
- try:
- try:
- return id2structure[structid]
- except KeyError:
- structure = Structure()
- structure2id[structure] = structid
- id2structure[structid] = structure
- return structure
- finally:
- maplock.release()
-
-def representstructure(structure, ref=None):
- data = {}
- for key, value in structure.__dict__.items():
- if not key.startswith('_'):
- data[key] = value
- if ref is None:
- ref = {}
- return 'getstructure(%r).__dict__ = %s' % (
- getstructureid(structure), represent(data, ref))
-
-def representeverything(root):
- lines = []
- pending = [root]
- seen = {root: True}
- for structure in pending:
- ref = {}
- lines.append(representstructure(structure, ref))
- for referenced in ref:
- if referenced not in seen:
- pending.append(referenced)
- seen[referenced] = True
- return lines
Modified: shpy/trunk/dist/shpy/ui_pygame/decorate.py
==============================================================================
--- shpy/trunk/dist/shpy/ui_pygame/decorate.py (original)
+++ shpy/trunk/dist/shpy/ui_pygame/decorate.py Fri Jan 23 15:27:20 2004
@@ -1,6 +1,6 @@
from __future__ import generators
import inspect, new
-from shpy.net.structure import Structure, ATOMIC
+from shpy.net.gateway import Structure, ATOMIC
class structureproperty(object):
Modified: shpy/trunk/dist/shpy/ui_pygame/ui_pygame.py
==============================================================================
--- shpy/trunk/dist/shpy/ui_pygame/ui_pygame.py (original)
+++ shpy/trunk/dist/shpy/ui_pygame/ui_pygame.py Fri Jan 23 15:27:20 2004
@@ -4,7 +4,6 @@
import pygame
from pygame.locals import *
import shpy.net.register
-from shpy.net.structure import representstructure, getstructureid
from shpy import info
from shpy.ui_pygame import decorate
@@ -124,6 +123,7 @@
def __init__(self, sharedserver, execserver):
ns = {'terminal': self}
self.servergateway = shpy.net.register.ServerGateway(sharedserver, ns)
+ self.servergateway.propagate_callback = self.postrepaintevent
self.execgateway = shpy.net.register.ExecGateway(execserver)
pygame.init()
@@ -177,9 +177,11 @@
self.scroll_a_bit_towards_cursor()
def drawcursors(self):
- assert self.cursor in self.root.users
+ cursors = list(self.root.users)
+ cursors.remove(self.cursor)
+ cursors.append(self.cursor)
cursor_in_view = False
- for cursor in self.root.users:
+ for cursor in cursors:
x = cursor.x
line = cursor.line
try:
@@ -295,7 +297,7 @@
if self.changed:
keys = self.changed.keys()
self.changed.clear()
- self.servergateway.notifychanges(*keys)
+ self.servergateway.sendchanges(*keys)
self.invalid = True
if self.invalid:
self.invalid = False
@@ -343,10 +345,9 @@
if len(sys.argv) > 2:
execserver = sys.argv[2]
else:
- execserver = 'localhost:8888'
+ execserver = 'popen=python -u -c "exec input(); import time; time.sleep(999)"'
t = Terminal(sharedserver, execserver)
try:
t.run()
finally:
t.close()
-
Modified: shpy/trunk/dist/startserver.py
==============================================================================
--- shpy/trunk/dist/startserver.py (original)
+++ shpy/trunk/dist/startserver.py Fri Jan 23 15:27:20 2004
@@ -7,16 +7,12 @@
# this part of the program only executes on the server side
#
-# XXX
-# a hack for convenience of displaying tracebacks
-from unittest2.tool import dyncode # just convenience
-import linecache
-linecache.getline = dyncode.getline
-# end hack
+from unittest2.tool import dyncode
+dyncode.installhacks()
-progname = 'unpickling_exec_server-1.0'
+progname = 'readline_exec_server-1.1'
# starting the agent server
-import sys, pickle
+import sys
def getlisteningsocket(hostport):
from socket import socket, AF_INET, SOCK_STREAM
@@ -32,13 +28,11 @@
clientsock,address = sock.accept()
print progname, 'got new connection from %s %s' % address
clientfile = clientsock.makefile('r+b',0)
- unpickler = pickle.Unpickler(clientfile)
- pickler = pickle.Pickler(clientfile)
+ source = clientfile.readline()
+ clientfile.close()
g = {'clientsock' : clientsock,
- 'address' : address,
- 'unpickler' : unpickler,
- 'pickler' : pickler}
- co = dyncode.makecode(unpickler.load())
+ 'address' : address}
+ co = dyncode.makecode(eval(source))
exec co in g
except KeyboardInterrupt:
sys.exit(1)