From jlg at codespeak.net  Wed Aug  1 16:23:20 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Wed,  1 Aug 2007 16:23:20 +0200 (CEST)
Subject: [pypy-svn] r45447 - in pypy/dist/pypy/lang/scheme: . test
Message-ID: <20070801142320.E0372814E@code0.codespeak.net>

Author: jlg
Date: Wed Aug  1 16:23:19 2007
New Revision: 45447

Modified:
   pypy/dist/pypy/lang/scheme/execution.py
   pypy/dist/pypy/lang/scheme/object.py
   pypy/dist/pypy/lang/scheme/ssparser.py
   pypy/dist/pypy/lang/scheme/test/test_macro.py
   pypy/dist/pypy/lang/scheme/test/test_parser.py
Log:
bug in macro expansions removed, SyntacticClosure is no more, instead we have PairClosure and SymbolClosure; parsing for ellipsis; macros with flat ellipsis works

Modified: pypy/dist/pypy/lang/scheme/execution.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/execution.py	(original)
+++ pypy/dist/pypy/lang/scheme/execution.py	Wed Aug  1 16:23:19 2007
@@ -83,13 +83,11 @@
         self.closure = closure
 
     def _dispatch(self, symb):
-        if isinstance(symb, W_Symbol):
-            return (self, symb.name)
+        if isinstance(symb, SymbolClosure):
+            return (symb.closure, symb.name)
 
-        elif isinstance(symb, SyntacticClosure):
-            symbol = symb.sexpr
-            if isinstance(symbol, W_Symbol):
-                return (symb.closure, symbol.name)
+        elif isinstance(symb, W_Symbol):
+            return (self, symb.name)
 
         raise SchemeSyntaxError
 

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Wed Aug  1 16:23:19 2007
@@ -77,6 +77,8 @@
     def eq_symbol(self, w_symb):
         return w_symb is self
 
+w_ellipsis = W_Symbol("...")
+
 def symbol(name):
     #use this to create new symbols, it stores all symbols
     #in W_Symbol.obarray dict
@@ -205,6 +207,9 @@
         #end proper list with dotted
         return car + " . " + cdr.to_string()
 
+    def __repr__(self):
+        return "<W_Pair " + self.to_string() + ">"
+
     def eval_tr(self, ctx):
         oper = self.car.eval(ctx)
         if not isinstance(oper, W_Callable):
@@ -316,7 +321,7 @@
         return "#<procedure %s>" % (self.pname,)
 
     def procedure_tr(self, ctx, lst):
-        """must be tail-recursive aware, uses eval_body"""
+        #must be tail-recursive aware, uses eval_body
         #ctx is a caller context, which is joyfully ignored
 
         local_ctx = self.closure.copy()
@@ -619,7 +624,7 @@
 
 class MacroIf(W_Macro):
     def call_tr(self, ctx, lst):
-        """if needs to be tail-recursive aware"""
+        #if needs to be tail-recursive aware
         if not isinstance(lst, W_Pair):
             raise SchemeSyntaxError
         w_condition = lst.car
@@ -667,7 +672,7 @@
 
 class LetStar(W_Macro):
     def call_tr(self, ctx, lst):
-        """let* uses eval_body, so it is tail-recursive aware"""
+        #let* uses eval_body, so it is tail-recursive aware
         if not isinstance(lst, W_Pair):
             raise SchemeSyntaxError
         local_ctx = ctx.copy()
@@ -856,6 +861,11 @@
         #closes template in syntactic enviroment at the point of definition
         return W_Transformer(syntax_lst, ctx)
 
+class Ellipsis(Exception):
+    def __init__(self, expr, level):
+        self.expr = expr
+        self.level = level
+
 class SyntaxRule(object):
     def __init__(self, pattern, template, literals):
         self.pattern = pattern
@@ -886,6 +896,13 @@
                 if w_form is not w_literal:
                     return (False, {})
 
+            w_pattcdr = w_patt.cdr
+            if isinstance(w_pattcdr, W_Pair) and w_pattcdr.car is w_ellipsis:
+                #w_pattcar should be matched 0-inf times in ellipsis
+                print w_patt, w_expr
+                match_dict[w_pattcar.to_string()] = Ellipsis(w_expr, 1)
+                return (True, match_dict)
+
             if isinstance(w_pattcar, W_Pair):
                 if not isinstance(w_exprcar, W_Pair):
                     return (False, {})
@@ -897,28 +914,60 @@
                 match_dict.update(match_nested)
 
             match_dict[w_pattcar.to_string()] = w_exprcar
+
             w_patt = w_patt.cdr
             w_expr = w_expr.cdr
 
-        if w_expr is w_nil and w_patt is w_nil:
-            return (True, match_dict)
+        if (w_expr is w_nil) and (w_patt is w_nil):
+                return (True, match_dict)
+
+        if w_patt is w_nil:
+            return (False, {})
+
+        #w_patt is symbol or primitive //as cdr of dotted list
+        match_dict[w_patt.to_string()] = w_expr
+        return (True, match_dict)
 
         return (False, {})
 
-class SyntacticClosure(W_Root):
-    def __init__(self, ctx, sexpr):
-        assert not isinstance(sexpr, SyntacticClosure)
-        assert isinstance(sexpr, W_Root)
-        self.sexpr = sexpr
+class SymbolClosure(W_Symbol):
+    def __init__(self, ctx, symbol):
+        assert isinstance(symbol, W_Symbol)
+        assert not isinstance(symbol, SymbolClosure)
+        self.symbol = symbol
+        self.name = symbol.name
         self.closure = ctx
 
     def eval_tr(self, ctx):
         #this symbol is in Syntactic Closure 
-        return self.sexpr.eval_tr(self.closure)
+        return self.symbol.eval_tr(self.closure)
+
+    def to_string(self):
+        #return "#<closure: " + self.sexpr.to_string() + ">"
+        return self.symbol.to_string()
+
+    def __repr__(self):
+        return "<sc:W_Symbol " + self.to_string() + ">"
+
+class PairClosure(W_Pair):
+    def __init__(self, ctx, pair):
+        assert isinstance(pair, W_Pair)
+        assert not isinstance(pair, PairClosure)
+        self.pair = pair
+        self.car = pair.car
+        self.cdr = pair.cdr
+        self.closure = ctx
+
+    def eval_tr(self, ctx):
+        #this pair is in Syntactic Closure 
+        return self.pair.eval_tr(self.closure)
 
     def to_string(self):
         #return "#<closure: " + self.sexpr.to_string() + ">"
-        return self.sexpr.to_string()
+        return self.pair.to_string()
+
+    def __repr__(self):
+        return "<sc:W_Pair " + self.to_string() + ">"
 
 class W_Transformer(W_Procedure):
     def __init__(self, syntax_lst, ctx, pname=""):
@@ -950,16 +999,31 @@
                 # enviroment at the point of use
 
                 #not always needed, because w_sub can have no W_Symbol inside
-                if isinstance(w_sub, W_Symbol) or isinstance(w_sub, W_Pair):
-                    return SyntacticClosure(ctx, w_sub)
+                if isinstance(w_sub, W_Symbol) and \
+                        not isinstance(w_sub, SymbolClosure):
+                    return SymbolClosure(ctx, w_sub)
+
+                if isinstance(w_sub, W_Pair) and \
+                        not isinstance(w_sub, PairClosure):
+                    return PairClosure(ctx, w_sub)
+
+                if isinstance(w_sub, Ellipsis):
+                    raise w_sub
 
                 return w_sub
 
             return sexpr
 
         elif isinstance(sexpr, W_Pair):
-            w_pair = W_Pair(self.substitute(ctx, sexpr.car, match_dict),
-                    self.substitute(ctx, sexpr.cdr, match_dict))
+            try:
+                w_pair = W_Pair(self.substitute(ctx, sexpr.car, match_dict),
+                        self.substitute(ctx, sexpr.cdr, match_dict))
+            except Ellipsis, e:
+                scdr = sexpr.cdr
+                if isinstance(scdr, W_Pair) and scdr.car is w_ellipsis:
+                    return e.expr
+                else:
+                    raise SchemeSyntaxError
 
             w_paircar = w_pair.car
             if isinstance(w_paircar, W_Symbol):
@@ -972,8 +1036,7 @@
                 except UnboundVariable:
                     pass
 
-            elif isinstance(w_paircar, SyntacticClosure) and \
-                    isinstance(w_paircar.sexpr, W_Symbol):
+            elif isinstance(w_paircar, SymbolClosure):
                 try:
                     #ops, which context?
                     w_macro = ctx.get(w_paircar.sexpr.to_string())
@@ -1043,7 +1106,6 @@
         w_formal = lst.car
         while isinstance(w_formal, W_Pair):
             w_def = w_formal.get_car_as_pair()
-            #evaluate the values in caller ctx
             w_transformer = w_def.get_cdr_as_pair().car.eval(ctx)
             if not isinstance(w_transformer, W_Transformer):
                 raise SchemeSyntaxError

Modified: pypy/dist/pypy/lang/scheme/ssparser.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/ssparser.py	(original)
+++ pypy/dist/pypy/lang/scheme/ssparser.py	Wed Aug  1 16:23:19 2007
@@ -2,7 +2,8 @@
 from pypy.rlib.parsing.pypackrat import PackratParser
 from pypy.rlib.parsing.makepackrat import BacktrackException, Status
 from pypy.lang.scheme.object import W_Pair, W_Integer, W_String, symbol, \
-        w_nil, W_Boolean, W_Real, quote, qq, unquote, unquote_splicing
+        w_nil, W_Boolean, W_Real, quote, qq, unquote, unquote_splicing, \
+        w_ellipsis
 
 def str_unquote(s):
     str_lst = []
@@ -29,13 +30,18 @@
         IGNORE*
         return {symbol(c)};
 
+    ELLIPSIS:
+        c = '...'
+        IGNORE*
+        return {w_ellipsis};
+
     FIXNUM:
         c = `\-?(0|([1-9][0-9]*))`
         IGNORE*
         return {W_Integer(int(c))};
 
     FLOAT:
-        c = `\-?[0-9]*\.[0-9]*`
+        c = `\-?([0-9]*\.[0-9]+|[0-9]+\.[0-9]*)`
         IGNORE*
         return {W_Real(float(c))};
 
@@ -83,6 +89,7 @@
       | qq
       | unquote_splicing
       | unquote
+      | ELLIPSIS
       | FLOAT
       | FIXNUM
       | BOOLEAN

Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Wed Aug  1 16:23:19 2007
@@ -236,6 +236,19 @@
     assert eval_(ctx, "(my-or #f 42)").to_number() == 42
     assert eval_(ctx, "(my-or #f #f 82)").to_number() == 82
 
+def test_macro_expand():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax foo (syntax-rules ()
+                                          ((foo) #t)
+                                          ((foo arg) arg)))""")
+    eval_(ctx, """(define-syntax bar (syntax-rules ()
+                                          ((bar) (foo))
+                                          ((bar arg) (foo arg))))""")
+
+    w_expr = parse("(bar 42)")[0]
+    #should expand directly (recursively) to 42
+    assert ctx.get("bar").expand(ctx, w_expr).to_string() == "42"
+
 def test_let_syntax():
     ctx = ExecutionContext()
     w_result = \
@@ -260,3 +273,32 @@
 
     assert eval_(ctx, "a").to_number() == 0
 
+def test_reverse():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax reverse-order
+                                 (syntax-rules () 
+                                   ((_ e) (reverse-order e ())) 
+                                   ((_ (e . rest) r)
+                                    (reverse-order rest (e . r))) 
+                                   ((_ () r) r)))""")
+
+    w_result = eval_(ctx, "(reverse-order (2 3 -))")
+    assert w_result.to_number() == 1
+
+def test_ellipsis_symbol():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax or (syntax-rules ()
+                                      ((or) #f)
+                                      ((or e) e)
+                                      ((or e1 e2 ...)
+                                       (let ((temp e1))
+                                         (if temp
+                                             temp
+                                             (or e2 ...))))))""")
+
+    assert eval_(ctx, "(or 12)").to_number() == 12
+    assert eval_(ctx, "(or 12 42)").to_number() == 12
+    assert eval_(ctx, "(or #f 42)").to_number() == 42
+    assert eval_(ctx, "(or #f #f 82)").to_number() == 82
+    assert eval_(ctx, "(or #f #f #f 162)").to_number() == 162
+

Modified: pypy/dist/pypy/lang/scheme/test/test_parser.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_parser.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_parser.py	Wed Aug  1 16:23:19 2007
@@ -2,6 +2,7 @@
 from pypy.lang.scheme.ssparser import parse
 from pypy.lang.scheme.object import W_Boolean, W_Real, W_Integer, W_String
 from pypy.lang.scheme.object import W_Pair, W_Nil, W_Symbol, W_Symbol
+from pypy.rlib.parsing.makepackrat import BacktrackException
 
 def parse_sexpr(expr):
     return parse(expr)[0]
@@ -56,6 +57,15 @@
     assert isinstance(w_float, W_Real)
     assert unwrap(w_float) == -123456.1234
 
+    w_float = parse_sexpr('.1234')
+    assert isinstance(w_float, W_Real)
+    assert unwrap(w_float) == 0.1234
+    w_float = parse_sexpr('12.')
+    assert isinstance(w_float, W_Real)
+    assert unwrap(w_float) == 12.0
+
+    py.test.raises(BacktrackException, parse_sexpr, '.')
+
 def test_sexpr():
     w_list = parse_sexpr('( 1 )')
     assert isinstance(w_list, W_Pair)
@@ -155,3 +165,8 @@
     assert unwrap(t) == ['unquote-splicing', ['list',
                                 ['unquote-splicing', 'b'], 3]]
 
+def test_ellipsis():
+    w_float = parse_sexpr('...')
+    assert isinstance(w_float, W_Symbol)
+    assert unwrap(w_float) == "..."
+

From jlg at codespeak.net  Wed Aug  1 16:47:41 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Wed,  1 Aug 2007 16:47:41 +0200 (CEST)
Subject: [pypy-svn] r45448 - pypy/dist/pypy/lang/scheme
Message-ID: <20070801144741.3B5F18157@code0.codespeak.net>

Author: jlg
Date: Wed Aug  1 16:47:40 2007
New Revision: 45448

Modified:
   pypy/dist/pypy/lang/scheme/object.py
Log:
scheme translation repaired

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Wed Aug  1 16:47:40 2007
@@ -861,11 +861,15 @@
         #closes template in syntactic enviroment at the point of definition
         return W_Transformer(syntax_lst, ctx)
 
-class Ellipsis(Exception):
+class Ellipsis(W_Root):
     def __init__(self, expr, level):
         self.expr = expr
         self.level = level
 
+class EllipsisException(SchemeException):
+    def __init__(self, ellipsis):
+        self.expr = ellipsis.expr
+
 class SyntaxRule(object):
     def __init__(self, pattern, template, literals):
         self.pattern = pattern
@@ -1008,7 +1012,7 @@
                     return PairClosure(ctx, w_sub)
 
                 if isinstance(w_sub, Ellipsis):
-                    raise w_sub
+                    raise EllipsisException(w_sub)
 
                 return w_sub
 
@@ -1018,7 +1022,7 @@
             try:
                 w_pair = W_Pair(self.substitute(ctx, sexpr.car, match_dict),
                         self.substitute(ctx, sexpr.cdr, match_dict))
-            except Ellipsis, e:
+            except EllipsisException, e:
                 scdr = sexpr.cdr
                 if isinstance(scdr, W_Pair) and scdr.car is w_ellipsis:
                     return e.expr
@@ -1027,6 +1031,8 @@
 
             w_paircar = w_pair.car
             if isinstance(w_paircar, W_Symbol):
+                #XXX what if we have here SymbolClosure?
+                # can happen when recursive macro
                 try:
                     w_macro = ctx.get(w_paircar.name)
 
@@ -1036,18 +1042,6 @@
                 except UnboundVariable:
                     pass
 
-            elif isinstance(w_paircar, SymbolClosure):
-                try:
-                    #ops, which context?
-                    w_macro = ctx.get(w_paircar.sexpr.to_string())
-
-                    # recursive macro expansion
-                    if isinstance(w_macro, W_DerivedMacro):
-                        return w_macro.expand(ctx, w_pair)
-
-                except UnboundVariable:
-                    pass
-
             return w_pair
 
         return sexpr

From arigo at codespeak.net  Wed Aug  1 18:37:31 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Wed,  1 Aug 2007 18:37:31 +0200 (CEST)
Subject: [pypy-svn] r45449 - in pypy/dist/pypy/translator/sandbox: . test
Message-ID: <20070801163731.46F3A817F@code0.codespeak.net>

Author: arigo
Date: Wed Aug  1 18:37:29 2007
New Revision: 45449

Added:
   pypy/dist/pypy/translator/sandbox/sandlib.py   (contents, props changed)
   pypy/dist/pypy/translator/sandbox/test/test_sandlib.py   (contents, props changed)
Modified:
   pypy/dist/pypy/translator/sandbox/sandboxmsg.py
Log:
Start a library meant for the parent process that runs a subprocess
that was translated with --sandbox.


Modified: pypy/dist/pypy/translator/sandbox/sandboxmsg.py
==============================================================================
--- pypy/dist/pypy/translator/sandbox/sandboxmsg.py	(original)
+++ pypy/dist/pypy/translator/sandbox/sandboxmsg.py	Wed Aug  1 18:37:29 2007
@@ -155,11 +155,58 @@
             raise ValueError
         return self.value[i:self.pos]
 
+    def decode(self, argtypes):
+        "NOT_RPYTHON"  # optimized decoder
+        v = self.value
+        i = self.pos
+        for t in argtypes:
+            if v[i] != t:
+                raise ValueError
+            end = i + 5
+            if t == "s":
+                length, = struct.unpack("!i", v[i+1:i+5])
+                end += length
+                yield v[i+5:end]
+            elif t == "i":
+                result, = struct.unpack("!i", v[i+1:end])
+                yield result
+            elif t == "I":
+                result, = struct.unpack("!I", v[i+1:end])
+                yield result
+            else:
+                raise ValueError
+            i = end
+        if i != len(v):
+            raise ValueError("more values to decode")
+
+def encode_message(types, values):
+    "NOT_RPYTHON"  # optimized encoder for messages
+    chars = ["!"]
+    entries = []
+    if len(types) != len(values):
+        raise ValueError("mismatch in the number of values to encode")
+    for t, val in zip(types, values):
+        chars.append("c")
+        entries.append(t)
+        if t == "s":
+            if not isinstance(val, str):
+                raise TypeError
+            chars.append("i%ds" % len(val))
+            entries.append(len(val))
+            entries.append(val)
+        elif t in "iI":
+            chars.append(t)
+            entries.append(val)
+        else:
+            raise ValueError
+    data = struct.pack(''.join(chars), *entries)
+    return struct.pack("!i", len(data) + 4) + data
+
 def timeout_read(f, size, timeout=None):
     if size < 0:
         raise ValueError("negative size")
     if timeout is None:
-        return f.read(size)
+        result = f.read(size)
     else:
         # XXX not Win32-compliant!
         assert not sys.platform.startswith('win'), "XXX fix me"
@@ -173,9 +220,11 @@
                     len(result), timeout, size))
             buf = os.read(fd, size - len(result))
             if not buf:
-                raise EOFError
+                break
             result += buf
-        return result
+    if len(result) < size:
+        raise EOFError
+    return result
 
 class Timeout(Exception):
     pass

Added: pypy/dist/pypy/translator/sandbox/sandlib.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/translator/sandbox/sandlib.py	Wed Aug  1 18:37:29 2007
@@ -0,0 +1,110 @@
+"""
+A Python library to execute and communicate with a subprocess that
+was translated from RPython code with --sandbox.  This library is
+for the outer process, which can run CPython or PyPy.
+"""
+
+from py.compat import subprocess
+from pypy.translator.sandbox.sandboxmsg import Message, encode_message
+from pypy.translator.sandbox.sandboxmsg import read_message
+
+class SandboxedProc(object):
+    """Base class to control a sandboxed subprocess.
+    Inherit from this class and implement all the do_xxx() methods
+    for the external functions xxx that you want to support.
+    """
+    def __init__(self, args):
+        """'args' should a sequence of argument for the subprocess,
+        starting with the full path of the executable.
+        """
+        self.popen = subprocess.Popen(args,
+                                      stdin=subprocess.PIPE,
+                                      stdout=subprocess.PIPE)
+
+    def poll(self):
+        return self.popen.poll()
+
+    def wait(self):
+        return self.popen.wait()
+
+    def handle_forever(self):
+        while True:
+            try:
+                msg = read_message(self.popen.stdout)
+            except EOFError, e:
+                break
+            answer = self.handle_message(msg)
+            self.popen.stdin.write(answer)
+        returncode = self.popen.wait()
+        if returncode != 0:
+            raise OSError("the sandboxed subprocess exited with code %d" % (
+                returncode,))
+
+    def handle_message(self, msg):
+        fn = msg.nextstring()
+        try:
+            argtypes, restypes = self.TYPES[fn]
+        except KeyError:
+            raise IOError("trying to invoke unknown external function %r" % (
+                fn,))
+        handler = getattr(self, 'do_' + fn)
+        answers = handler(*msg.decode(argtypes))
+        if len(restypes) == 0:
+            assert answers is None
+            answers = (0,)
+        elif len(restypes) == 1:
+            answers = (0, answers)
+        else:
+            answers = (0,) + answers
+        return encode_message("i" + restypes, answers)
+
+    TYPES = {
+        "open": ("sii", "i"),
+        "read": ("iI", "s"),
+        "write": ("is", "I"),
+        "close": ("i", "i"),
+        }
+
+
+class SimpleIOSandboxedProc(SandboxedProc):
+    """Control a sandboxed subprocess which is only allowed to read from
+    its stdin and write to its stdout and stderr.
+    """
+    _input = None
+    _output = None
+    _error = None
+
+    def communicate(self, input=None):
+        """Send data to stdin. Read data from stdout and stderr,
+        until end-of-file is reached. Wait for process to terminate.
+        """
+        import cStringIO
+        if input:
+            if isinstance(input, str):
+                input = cStringIO.StringIO(input)
+            self._input = input
+        self._output = cStringIO.StringIO()
+        self._error = cStringIO.StringIO()
+        self.handle_forever()
+        output = self._output.getvalue()
+        self._output = None
+        error = self._error.getvalue()
+        self._error = None
+        return (output, error)
+
+    def do_read(self, fd, size):
+        if fd == 0:
+            if self._input is None:
+                return ""
+            else:
+                return self._input.read(size)
+        raise OSError("trying to read from fd %d" % (fd,))
+
+    def do_write(self, fd, data):
+        if fd == 1:
+            self._output.write(data)
+            return len(data)
+        if fd == 2:
+            self._error.write(data)
+            return len(data)
+        raise OSError("trying to write to fd %d" % (fd,))

Added: pypy/dist/pypy/translator/sandbox/test/test_sandlib.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/translator/sandbox/test/test_sandlib.py	Wed Aug  1 18:37:29 2007
@@ -0,0 +1,79 @@
+import os, StringIO
+from pypy.tool.sourcetools import func_with_new_name
+from pypy.translator.sandbox.sandlib import SandboxedProc
+from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc
+from pypy.translator.interactive import Translation
+
+
+class MySandboxedProc(SandboxedProc):
+
+    def __init__(self, args, expected):
+        SandboxedProc.__init__(self, args)
+        self.expected = expected
+        self.seen = 0
+
+    def _make_method(name):
+        def do_xxx(self, *input):
+            print "decoded from subprocess: %s%r" % (name, input)
+            expectedmsg, expectedinput, output = self.expected[self.seen]
+            assert name == expectedmsg
+            assert input == expectedinput
+            self.seen += 1
+            return output
+        return func_with_new_name(do_xxx, 'do_%s' % name)
+
+    do_open  = _make_method("open")
+    do_read  = _make_method("read")
+    do_write = _make_method("write")
+    do_close = _make_method("close")
+
+
+def test_lib():
+    def entry_point(argv):
+        fd = os.open("/tmp/foobar", os.O_RDONLY, 0777)
+        assert fd == 77
+        res = os.read(fd, 123)
+        assert res == "he\x00llo"
+        count = os.write(fd, "world\x00!\x00")
+        assert count == 42
+        for arg in argv:
+            count = os.write(fd, arg)
+            assert count == 61
+        os.close(fd)
+        return 0
+    t = Translation(entry_point, backend='c', standalone=True, sandbox=True)
+    exe = t.compile()
+
+    proc = MySandboxedProc([exe, 'x1', 'y2'], expected = [
+        ("open", ("/tmp/foobar", os.O_RDONLY, 0777), 77),
+        ("read", (77, 123), "he\x00llo"),
+        ("write", (77, "world\x00!\x00"), 42),
+        ("write", (77, exe), 61),
+        ("write", (77, "x1"), 61),
+        ("write", (77, "y2"), 61),
+        ("close", (77,), 0),
+        ])
+    proc.handle_forever()
+    assert proc.seen == len(proc.expected)
+
+def test_simpleio():
+    def entry_point(argv):
+        print "Please enter a number:"
+        buf = ""
+        while True:
+            t = os.read(0, 1)    # 1 character from stdin
+            if not t:
+                raise EOFError
+            if t == '\n':
+                break
+            buf += t
+        num = int(buf)
+        print "The double is:", num * 2
+        return 0
+    t = Translation(entry_point, backend='c', standalone=True, sandbox=True)
+    exe = t.compile()
+
+    proc = SimpleIOSandboxedProc([exe, 'x1', 'y2'])
+    output, error = proc.communicate("21\n")
+    assert output == "Please enter a number:\nThe double is: 42\n"
+    assert error == ""

From jlg at codespeak.net  Thu Aug  2 11:46:03 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Thu,  2 Aug 2007 11:46:03 +0200 (CEST)
Subject: [pypy-svn] r45450 - in pypy/dist/pypy/lang/scheme: . test
Message-ID: <20070802094603.29C1B80EC@code0.codespeak.net>

Author: jlg
Date: Thu Aug  2 11:46:01 2007
New Revision: 45450

Modified:
   pypy/dist/pypy/lang/scheme/object.py
   pypy/dist/pypy/lang/scheme/test/test_macro.py
Log:
ellipsis in macro template can refer to list

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Thu Aug  2 11:46:01 2007
@@ -862,9 +862,8 @@
         return W_Transformer(syntax_lst, ctx)
 
 class Ellipsis(W_Root):
-    def __init__(self, expr, level):
+    def __init__(self, expr):
         self.expr = expr
-        self.level = level
 
 class EllipsisException(SchemeException):
     def __init__(self, ellipsis):
@@ -904,7 +903,7 @@
             if isinstance(w_pattcdr, W_Pair) and w_pattcdr.car is w_ellipsis:
                 #w_pattcar should be matched 0-inf times in ellipsis
                 print w_patt, w_expr
-                match_dict[w_pattcar.to_string()] = Ellipsis(w_expr, 1)
+                match_dict[w_pattcar.to_string()] = Ellipsis(w_expr)
                 return (True, match_dict)
 
             if isinstance(w_pattcar, W_Pair):
@@ -1024,10 +1023,18 @@
                         self.substitute(ctx, sexpr.cdr, match_dict))
             except EllipsisException, e:
                 scdr = sexpr.cdr
+                print ">", sexpr, e.expr
                 if isinstance(scdr, W_Pair) and scdr.car is w_ellipsis:
                     return e.expr
                 else:
-                    raise SchemeSyntaxError
+                    plst = []
+                    w_pair = e.expr
+                    while isinstance(w_pair, W_Pair):
+                        plst.append(W_Pair(w_pair.car, scdr))
+                        w_pair = w_pair.cdr
+
+                    ellipsis = Ellipsis(plst2lst(plst))
+                    raise EllipsisException(ellipsis)
 
             w_paircar = w_pair.car
             if isinstance(w_paircar, W_Symbol):

Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Thu Aug  2 11:46:01 2007
@@ -302,3 +302,40 @@
     assert eval_(ctx, "(or #f #f 82)").to_number() == 82
     assert eval_(ctx, "(or #f #f #f 162)").to_number() == 162
 
+def test_ellipsis_list_template():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax letzero
+                                 (syntax-rules ()
+                                    ((_ (sym ...) body ...)
+                                     (let ((sym 0) ...) body ...))))""")
+
+    assert eval_(ctx, "(letzero (x) x)").to_number() == 0
+    assert eval_(ctx, "(letzero (x) (set! x 1) x)").to_number() == 1
+
+    assert eval_(ctx, "(letzero (x y z) (+ x y z))").to_number() == 0
+    assert eval_(ctx, """(letzero (x y z) (set! x 1)
+                                          (set! y 1)
+                                          (set! z 1)
+                                          (+ x y z))""").to_number() == 3
+
+def test_ellipsis_list_pattern():
+    py.test.skip("in progress")
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax rlet
+                                 (syntax-rules ()
+                                    ((_ ((val sym) ...) body ...)
+                                     (let ((sym val) ...) body ...))))""")
+
+    assert eval_(ctx, "(rlet ((0 x)) x)").to_number() == 0
+    assert eval_(ctx, "(rlet ((0 x)) (set! x 1) x)").to_number() == 1
+
+    assert eval_(ctx, """(rlet ((0 x) (0 y) (0 z))
+                               (+ x y z))""").to_number() == 0
+    assert eval_(ctx, """(rlet ((0 x) (0 y) (0 z))
+                               (set! x 1)
+                               (set! y 1)
+                               (set! z 1)
+                               (+ x y z))""").to_number() == 3
+
+    assert False
+

From jlg at codespeak.net  Thu Aug  2 12:28:07 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Thu,  2 Aug 2007 12:28:07 +0200 (CEST)
Subject: [pypy-svn] r45451 - in pypy/dist/pypy/lang/scheme: . test
Message-ID: <20070802102807.9C12580E6@code0.codespeak.net>

Author: jlg
Date: Thu Aug  2 12:28:00 2007
New Revision: 45451

Modified:
   pypy/dist/pypy/lang/scheme/object.py
   pypy/dist/pypy/lang/scheme/test/test_macro.py
Log:
not nested ellipsis in template in glory

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Thu Aug  2 12:28:00 2007
@@ -866,8 +866,9 @@
         self.expr = expr
 
 class EllipsisException(SchemeException):
-    def __init__(self, ellipsis):
+    def __init__(self, ellipsis, name):
         self.expr = ellipsis.expr
+        self.name = name
 
 class SyntaxRule(object):
     def __init__(self, pattern, template, literals):
@@ -1011,7 +1012,7 @@
                     return PairClosure(ctx, w_sub)
 
                 if isinstance(w_sub, Ellipsis):
-                    raise EllipsisException(w_sub)
+                    raise EllipsisException(w_sub, sexpr.name)
 
                 return w_sub
 
@@ -1023,18 +1024,23 @@
                         self.substitute(ctx, sexpr.cdr, match_dict))
             except EllipsisException, e:
                 scdr = sexpr.cdr
-                print ">", sexpr, e.expr
+                print ">", sexpr, e.name, e.expr
                 if isinstance(scdr, W_Pair) and scdr.car is w_ellipsis:
-                    return e.expr
-                else:
+                    print ">>", sexpr, e.name, e.expr
                     plst = []
                     w_pair = e.expr
                     while isinstance(w_pair, W_Pair):
-                        plst.append(W_Pair(w_pair.car, scdr))
+                        #plst.append(W_Pair(w_pair.car, scdr))
+                        zzz = self.substitute(ctx, sexpr.car,
+                                {e.name: w_pair.car})
+                        plst.append(zzz)
                         w_pair = w_pair.cdr
 
-                    ellipsis = Ellipsis(plst2lst(plst))
-                    raise EllipsisException(ellipsis)
+                    ellipsis = plst2lst(plst)
+                    print ellipsis
+                    return ellipsis
+                else:
+                    raise e #EllipsisException(ellipsis, e.name)
 
             w_paircar = w_pair.car
             if isinstance(w_paircar, W_Symbol):

Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Thu Aug  2 12:28:00 2007
@@ -318,6 +318,22 @@
                                           (set! z 1)
                                           (+ x y z))""").to_number() == 3
 
+def test_ellipsis_expr_template():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax zero-if-true
+                                 (syntax-rules ()
+                                    ((_ sym ...)
+                                     (begin
+                                       (if sym (set! sym 0)) ...))))""")
+
+    eval_(ctx, "(define x #t)")
+    eval_(ctx, "(define y #f)")
+    eval_(ctx, "(define z #t)")
+    eval_(ctx, "(zero-if-true x y z)")
+    assert eval_(ctx, "x").to_number() == 0
+    assert eval_(ctx, "y").to_boolean() is False
+    assert eval_(ctx, "z").to_number() == 0
+
 def test_ellipsis_list_pattern():
     py.test.skip("in progress")
     ctx = ExecutionContext()

From arigo at codespeak.net  Thu Aug  2 13:38:34 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 13:38:34 +0200 (CEST)
Subject: [pypy-svn] r45452 - pypy/dist/pypy/translator/goal
Message-ID: <20070802113834.89A0780E8@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 13:38:33 2007
New Revision: 45452

Modified:
   pypy/dist/pypy/translator/goal/targetjsstandalone.py
Log:
Fix the JS target to use the current interface.


Modified: pypy/dist/pypy/translator/goal/targetjsstandalone.py
==============================================================================
--- pypy/dist/pypy/translator/goal/targetjsstandalone.py	(original)
+++ pypy/dist/pypy/translator/goal/targetjsstandalone.py	Thu Aug  2 13:38:33 2007
@@ -3,7 +3,6 @@
 """
 
 import sys
-from pypy.rlib.streamio import open_file_as_stream
 from pypy.lang.js.interpreter import *
 from pypy.lang.js.jsobj import ExecutionReturned
 
@@ -13,8 +12,8 @@
 
 def entry_point(argv):
     if len(argv) == 2:
-        f = open_file_as_stream(argv[1])
-        interp.run(load_source(f.readall()))
+        t = load_file(argv[1])
+        interp.run(t)
         return 0
     elif argv[0] == 'foo':
         raise ExecutionReturned(None)

From arigo at codespeak.net  Thu Aug  2 13:46:40 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 13:46:40 +0200 (CEST)
Subject: [pypy-svn] r45453 - pypy/dist/pypy/rpython
Message-ID: <20070802114640.00E2580C4@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 13:46:40 2007
New Revision: 45453

Modified:
   pypy/dist/pypy/rpython/extfunc.py
Log:
A more explicit error message for bad annotations reaching external
function calls.



Modified: pypy/dist/pypy/rpython/extfunc.py
==============================================================================
--- pypy/dist/pypy/rpython/extfunc.py	(original)
+++ pypy/dist/pypy/rpython/extfunc.py	Thu Aug  2 13:46:40 2007
@@ -82,9 +82,19 @@
         if self.signature_args is not None:
             assert len(args_s) == len(self.signature_args),\
                    "Argument number mismatch"
-            for arg, expected in zip(args_s, self.signature_args):
-                arg = unionof(arg, expected)
-                assert expected.contains(arg)
+            for i, expected in enumerate(self.signature_args):
+                arg = unionof(args_s[i], expected)
+                if not expected.contains(arg):
+                    name = getattr(self, 'name', None)
+                    if not name:
+                        try:
+                            name = self.instance.__name__
+                        except AttributeError:
+                            name = '?'
+                    raise Exception("In call to external function %r:\n"
+                                    "arg %d must be %s,\n"
+                                    "          got %s" % (
+                        name, i+1, expected, args_s[i]))
         return self.signature_result
 
     def specialize_call(self, hop):

From arigo at codespeak.net  Thu Aug  2 13:50:11 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 13:50:11 +0200 (CEST)
Subject: [pypy-svn] r45454 - pypy/dist/pypy/lang/prolog/builtin
Message-ID: <20070802115011.E5EB580E8@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 13:50:11 2007
New Revision: 45454

Modified:
   pypy/dist/pypy/lang/prolog/builtin/formatting.py
Log:
Annotation fix - the implicitly returned None eventually reaches os.write(),
where is explodes.


Modified: pypy/dist/pypy/lang/prolog/builtin/formatting.py
==============================================================================
--- pypy/dist/pypy/lang/prolog/builtin/formatting.py	(original)
+++ pypy/dist/pypy/lang/prolog/builtin/formatting.py	Thu Aug  2 13:50:11 2007
@@ -56,6 +56,8 @@
             return self.format_term(term)
         elif isinstance(term, Var):
             return self.format_var(term)
+        else:
+            return '?'
 
     def format_atom(self, s):
         from pypy.rlib.parsing.deterministic import LexerError

From jlg at codespeak.net  Thu Aug  2 16:07:00 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Thu,  2 Aug 2007 16:07:00 +0200 (CEST)
Subject: [pypy-svn] r45455 - in pypy/dist/pypy/lang/scheme: . test
Message-ID: <20070802140700.2BD2B80F4@code0.codespeak.net>

Author: jlg
Date: Thu Aug  2 16:06:58 2007
New Revision: 45455

Modified:
   pypy/dist/pypy/lang/scheme/object.py
   pypy/dist/pypy/lang/scheme/test/test_macro.py
Log:
SyntaxRule match refactoring; test_macro.py changed; more ellipsis working

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Thu Aug  2 16:06:58 2007
@@ -862,13 +862,18 @@
         return W_Transformer(syntax_lst, ctx)
 
 class Ellipsis(W_Root):
-    def __init__(self, expr):
-        self.expr = expr
+    def __init__(self, mdict_lst):
+        self.mdict_lst = mdict_lst
 
 class EllipsisException(SchemeException):
-    def __init__(self, ellipsis, name):
-        self.expr = ellipsis.expr
-        self.name = name
+    def __init__(self, ellipsis):
+        self.mdict_lst = ellipsis.mdict_lst
+
+class EllipsisPattern(SchemeException):
+    pass
+
+class MatchError(SchemeException):
+    pass
 
 class SyntaxRule(object):
     def __init__(self, pattern, template, literals):
@@ -881,58 +886,63 @@
 
     def match(self, ctx, w_expr, pattern=None):
         if pattern is None:
-            w_patt = self.pattern
-        else:
-            w_patt = pattern
+            return self.matchr(ctx, self.pattern, w_expr)
 
-        match_dict = {}
-        while isinstance(w_patt, W_Pair) and isinstance(w_expr, W_Pair):
+        return self.matchr(ctx, pattern, w_expr)
+
+    def matchr(self, ctx, w_patt, w_expr):
+        if isinstance(w_patt, W_Pair):
             w_pattcar = w_patt.car
-            w_exprcar = w_expr.car
+            if isinstance(w_expr, W_Pair):
+                mdict_car = self.matchr(ctx, w_pattcar, w_expr.car)
 
-            w_literal = self.literals.get(w_pattcar.to_string(), None)
-            if w_literal is not None:
                 try:
-                    w_form = ctx.get(w_exprcar.to_string())
-                except UnboundVariable:
-                    w_form = w_exprcar
-
-                if w_form is not w_literal:
-                    return (False, {})
+                #we catch EllipsisPattern here because in car
+                # we dont know how to deal with it
+                    mdict_cdr = self.matchr(ctx, w_patt.cdr, w_expr.cdr)
+                except EllipsisPattern:
+                    print "ellipsis matched", w_patt, w_expr
 
-            w_pattcdr = w_patt.cdr
-            if isinstance(w_pattcdr, W_Pair) and w_pattcdr.car is w_ellipsis:
-                #w_pattcar should be matched 0-inf times in ellipsis
-                print w_patt, w_expr
-                match_dict[w_pattcar.to_string()] = Ellipsis(w_expr)
-                return (True, match_dict)
-
-            if isinstance(w_pattcar, W_Pair):
-                if not isinstance(w_exprcar, W_Pair):
-                    return (False, {})
+                    mdict_lst = []
+                    w_pair = w_expr
+                    while isinstance(w_pair, W_Pair):
+                        mdict = self.matchr(ctx, w_pattcar, w_pair.car)
+                        mdict_lst.append(mdict)
+                        w_pair = w_pair.cdr
 
-                (matched, match_nested) = self.match(ctx, w_exprcar, w_pattcar)
-                if not matched:
-                    return (False, {})
+                    mdict_cdr = {}
+                    ellipsis = Ellipsis(mdict_lst)
+                    for name in mdict_lst[0].keys():
+                        mdict_cdr[name] = ellipsis
+
+                mdict_car.update(mdict_cdr)
+                return mdict_car
 
-                match_dict.update(match_nested)
+            if w_pattcar is w_ellipsis and w_expr is w_nil:
+                raise EllipsisPattern
 
-            match_dict[w_pattcar.to_string()] = w_exprcar
+        if w_patt is w_ellipsis:
+            raise EllipsisPattern
 
-            w_patt = w_patt.cdr
-            w_expr = w_expr.cdr
+        if isinstance(w_patt, W_Symbol):
+            w_literal = self.literals.get(w_patt.name, None)
+            if w_literal is not None:
+                try:
+                    w_form = ctx.get(w_expr.to_string())
+                except UnboundVariable:
+                    w_form = w_expr
 
-        if (w_expr is w_nil) and (w_patt is w_nil):
-                return (True, match_dict)
+                if w_form is not w_literal:
+                    raise MatchError
 
-        if w_patt is w_nil:
-            return (False, {})
+            return {w_patt.name: w_expr}
 
-        #w_patt is symbol or primitive //as cdr of dotted list
-        match_dict[w_patt.to_string()] = w_expr
-        return (True, match_dict)
+        if w_patt is w_nil and w_expr is w_nil:
+            return {}
 
-        return (False, {})
+        #w_patt is w_nil, but w_expr is not
+        # or w_patt is W_Pair but w_expr is not
+        raise MatchError
 
 class SymbolClosure(W_Symbol):
     def __init__(self, ctx, symbol):
@@ -981,16 +991,18 @@
 
     def match(self, ctx, w_expr):
         for rule in self.syntax_lst:
-            (matched, match_dict) = rule.match(ctx, w_expr)
-            if matched:
+            try:
+                match_dict = rule.match(ctx, w_expr)
                 return (rule.template, match_dict)
+            except MatchError:
+                pass
 
-        return (None, {})
+        raise MatchError
 
     def expand(self, ctx, w_expr):
-        (template, match_dict) = self.match(ctx, w_expr)
-
-        if template is None :
+        try:
+            (template, match_dict) = self.match(ctx, w_expr)
+        except MatchError:
             raise SchemeSyntaxError
 
         return self.substitute(ctx, template, match_dict)
@@ -1012,7 +1024,7 @@
                     return PairClosure(ctx, w_sub)
 
                 if isinstance(w_sub, Ellipsis):
-                    raise EllipsisException(w_sub, sexpr.name)
+                    raise EllipsisException(w_sub)
 
                 return w_sub
 
@@ -1024,23 +1036,20 @@
                         self.substitute(ctx, sexpr.cdr, match_dict))
             except EllipsisException, e:
                 scdr = sexpr.cdr
-                print ">", sexpr, e.name, e.expr
+                print ">", sexpr, e.mdict_lst
                 if isinstance(scdr, W_Pair) and scdr.car is w_ellipsis:
-                    print ">>", sexpr, e.name, e.expr
+                    print ">>", sexpr, e.mdict_lst
+
                     plst = []
-                    w_pair = e.expr
-                    while isinstance(w_pair, W_Pair):
-                        #plst.append(W_Pair(w_pair.car, scdr))
-                        zzz = self.substitute(ctx, sexpr.car,
-                                {e.name: w_pair.car})
+                    for mdict in e.mdict_lst:
+                        zzz = self.substitute(ctx, sexpr.car, mdict)
                         plst.append(zzz)
-                        w_pair = w_pair.cdr
 
                     ellipsis = plst2lst(plst)
                     print ellipsis
                     return ellipsis
                 else:
-                    raise e #EllipsisException(ellipsis, e.name)
+                    raise e
 
             w_paircar = w_pair.car
             if isinstance(w_paircar, W_Symbol):

Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Thu Aug  2 16:06:58 2007
@@ -9,41 +9,42 @@
 def eval_noctx(expr):
     return parse(expr)[0].eval(ExecutionContext())
 
+def parse_(expr):
+    return parse(expr)[0]
+
 def test_syntax_rules_match():
     ctx = ExecutionContext()
     py.test.raises(SchemeSyntaxError, eval_noctx, "(syntax-rules 1)")
     py.test.raises(SchemeSyntaxError, eval_noctx, "(syntax-rules () 1)")
 
     w_transformer = eval_noctx("(syntax-rules ())")
-    w_expr = parse("(foo)")[0]
-    assert not w_transformer.match(ctx, w_expr)[0]
+    w_expr = parse_("(foo)")
+    py.test.raises(MatchError, w_transformer.match, ctx, w_expr)
 
     w_transformer = eval_noctx("(syntax-rules () ((foo) #t))")
-    w_expr = parse("(bar)")[0]
-    assert w_transformer.match(ctx, w_expr)[0]
-    w_expr = parse("(foo bar)")[0]
-    assert not w_transformer.match(ctx, w_expr)[0]
+    w_expr = parse_("(bar)")
+    assert w_transformer.match(ctx, w_expr)[0].to_boolean()
+    w_expr = parse_("(foo bar)")
+    py.test.raises(MatchError, w_transformer.match, ctx, w_expr)
 
     w_transformer = eval_noctx("""(syntax-rules () ((_) #t)
                                                    ((_ foo) foo))""")
-    w_expr = parse("(foo)")[0]
+    w_expr = parse_("(foo)")
     assert w_transformer.match(ctx, w_expr)[0].to_boolean()
-    w_expr = parse("(foo bar)")[0]
+    w_expr = parse_("(foo bar)")
     (template, match_dict) = w_transformer.match(ctx, w_expr)
     assert template.to_string() == "foo"
     assert match_dict["foo"].to_string() == "bar"
 
-    w_expr = parse("(foo bar boo)")[0]
-    (template, match_dict) = w_transformer.match(ctx, w_expr)
-    assert not template
-    assert match_dict == {}
+    w_expr = parse_("(foo bar boo)")
+    py.test.raises(MatchError, w_transformer.match, ctx, w_expr)
 
     w_transformer = eval_noctx("(syntax-rules () ((foo (bar)) bar))")
-    w_expr = parse("(_ fuzz)")[0]
-    assert not w_transformer.match(ctx, w_expr)[0]
-    w_expr = parse("(_ (fuzz))")[0]
+    w_expr = parse_("(_ fuzz)")
+    py.test.raises(MatchError, w_transformer.match, ctx, w_expr)
+    w_expr = parse_("(_ (fuzz))")
     (template, match_dict) = w_transformer.match(ctx, w_expr)
-    assert template
+    assert template.to_string() == "bar"
     assert match_dict["bar"].to_string() == "fuzz"
 
 def test_syntax_rules_literals():
@@ -53,14 +54,14 @@
     # w_transformer created in ctx
     w_transformer = eval_(ctx, "(syntax-rules (=>) ((foo => bar) #t))")
 
-    w_expr = parse("(foo bar boo)")[0]
-    assert not w_transformer.match(ctx, w_expr)[0]
+    w_expr = parse_("(foo bar boo)")
+    py.test.raises(MatchError, w_transformer.match, ctx, w_expr)
 
     # exact match
-    w_expr = parse("(foo => boo)")[0]
+    w_expr = parse_("(foo => boo)")
 
     # within the same context
-    assert w_transformer.match(ctx, w_expr)[0]
+    assert w_transformer.match(ctx, w_expr)[0].to_boolean()
 
     w_42 = W_Number(42)
 
@@ -68,19 +69,19 @@
     closure = ctx.copy()
     closure.put("=>", w_42)
     w_transformer = eval_(ctx, "(syntax-rules (=>) ((foo => bar) #t))")
-    assert not w_transformer.match(closure, w_expr)[0]
+    py.test.raises(MatchError, w_transformer.match, closure, w_expr)
 
     # different lexical scope, not the same bindings for => in ctx and closure
     ctx.put("=>", W_Number(12))
     assert ctx.get("=>") is not closure.get("=>")
     w_transformer = eval_(ctx, "(syntax-rules (=>) ((foo => bar) #t))")
-    assert not w_transformer.match(closure, w_expr)[0]
+    py.test.raises(MatchError, w_transformer.match, closure, w_expr)
 
     # the same binding for => in ctx and closure
     ctx.put("=>", w_42)
     assert ctx.get("=>") is closure.get("=>")
     w_transformer = eval_(ctx, "(syntax-rules (=>) ((foo => bar) #t))")
-    assert w_transformer.match(closure, w_expr)[0]
+    assert w_transformer.match(closure, w_expr)[0].to_boolean()
 
 def test_syntax_rules_expand_simple():
     ctx = ExecutionContext()
@@ -88,12 +89,12 @@
     w_transformer = eval_(ctx, """(syntax-rules () ((_) #t)
                                                    ((_ foo) foo))""")
 
-    w_expr = parse("(foo)")[0]
+    w_expr = parse_("(foo)")
     w_expanded = w_transformer.expand(ctx, w_expr)
     assert isinstance(w_expanded, W_Boolean)
     assert w_expanded.to_boolean() == True
 
-    w_expr = parse("(foo bar)")[0]
+    w_expr = parse_("(foo bar)")
     w_expanded = w_transformer.expand(ctx, w_expr)
     assert w_expanded.to_string() == "bar"
 
@@ -101,7 +102,7 @@
                                         ((let1 var val body)
                                          (let ((var val)) body)))""")
 
-    w_expr = parse("(let1 var 12 (+ 1 var))")[0]
+    w_expr = parse_("(let1 var 12 (+ 1 var))")
     w_expanded = w_transformer.expand(ctx, w_expr)
     assert isinstance(w_expanded, W_Pair)
     assert w_expanded.to_string() == "(let ((var 12)) (+ 1 var))"
@@ -110,7 +111,7 @@
                                         ((let1 (var val) body)
                                          (let ((var val)) body)))""")
 
-    w_expr = parse("(let1 (var 12) (+ 1 var))")[0]
+    w_expr = parse_("(let1 (var 12) (+ 1 var))")
     w_expanded = w_transformer.expand(ctx, w_expr)
     assert isinstance(w_expanded, W_Pair)
     assert w_expanded.to_string() == "(let ((var 12)) (+ 1 var))"
@@ -122,14 +123,14 @@
                                        ((_ var)
                                         (let ((temp 1)) (+ var temp))))""")
 
-    w_expr = parse("(_ 12)")[0]
+    w_expr = parse_("(_ 12)")
     w_expanded = w_transformer.expand(ctx, w_expr)
     assert w_expanded.to_string() == "(let ((temp 1)) (+ 12 temp))"
     assert w_transformer.expand_eval(ctx, w_expr).to_number() == 13
 
     #transparency
     eval_(ctx, "(define temp 12)")
-    w_expr = parse("(_ temp)")[0]
+    w_expr = parse_("(_ temp)")
     w_expanded = w_transformer.expand(ctx, w_expr)
     assert w_expanded.to_string() == "(let ((temp 1)) (+ temp temp))"
     assert w_transformer.expand_eval(ctx, w_expr).to_number() == 13
@@ -157,11 +158,11 @@
                                                 (loop (- counter 1)))))))
                                         (loop count))))""")
 
-    w_expr = parse("(dotimes 5 (set! counter (+ counter 1)))")[0]
+    w_expr = parse_("(dotimes 5 (set! counter (+ counter 1)))")
     py.test.raises(UnboundVariable, w_transformer.expand_eval, ctx, w_expr)
 
     eval_(ctx, "(define counter 0)")
-    w_expr = parse("(dotimes 5 (set! counter (+ counter 1)))")[0]
+    w_expr = parse_("(dotimes 5 (set! counter (+ counter 1)))")
     w_transformer.expand_eval(ctx, w_expr)
     assert ctx.get("counter").to_number() == 5
 
@@ -172,7 +173,7 @@
                                      ((shadow used-arg body)
                                       (let ((used-arg 5)) body)))""")
 
-    w_expr = parse("(shadow test test)")[0]
+    w_expr = parse_("(shadow test test)")
     assert w_transformer.expand_eval(ctx, w_expr).to_number() == 5
 
     eval_(ctx, "(define test 7)")
@@ -182,7 +183,7 @@
                                      ((shadow used-arg body)
                                       (letrec ((used-arg 5)) body)))""")
 
-    w_expr = parse("(shadow test test)")[0]
+    w_expr = parse_("(shadow test test)")
     assert w_transformer.expand_eval(ctx, w_expr).to_number() == 5
 
     eval_(ctx, "(define test 7)")
@@ -226,10 +227,10 @@
     assert eval_(ctx, "(my-or 12)").to_number() == 12
 
     #should expand recursively and after that eval
-    w_expr = parse("(my-or 12 42)")[0]
+    w_expr = parse_("(my-or 12 42)")
     assert ctx.get("my-or").expand(ctx, w_expr).to_string() == \
             "(if 12 12 42)"
-    w_expr = parse("(my-or 12 42 82)")[0]
+    w_expr = parse_("(my-or 12 42 82)")
     assert ctx.get("my-or").expand(ctx, w_expr).to_string() == \
             "(if 12 12 (if 42 42 82))"
     assert eval_(ctx, "(my-or 12 42)").to_number() == 12
@@ -245,7 +246,7 @@
                                           ((bar) (foo))
                                           ((bar arg) (foo arg))))""")
 
-    w_expr = parse("(bar 42)")[0]
+    w_expr = parse_("(bar 42)")
     #should expand directly (recursively) to 42
     assert ctx.get("bar").expand(ctx, w_expr).to_string() == "42"
 
@@ -298,7 +299,7 @@
 
     assert eval_(ctx, "(or 12)").to_number() == 12
     assert eval_(ctx, "(or 12 42)").to_number() == 12
-    assert eval_(ctx, "(or #f 42)").to_number() == 42
+    
     assert eval_(ctx, "(or #f #f 82)").to_number() == 82
     assert eval_(ctx, "(or #f #f #f 162)").to_number() == 162
 
@@ -335,7 +336,6 @@
     assert eval_(ctx, "z").to_number() == 0
 
 def test_ellipsis_list_pattern():
-    py.test.skip("in progress")
     ctx = ExecutionContext()
     eval_(ctx, """(define-syntax rlet
                                  (syntax-rules ()
@@ -353,5 +353,3 @@
                                (set! z 1)
                                (+ x y z))""").to_number() == 3
 
-    assert False
-

From jlg at codespeak.net  Thu Aug  2 17:30:48 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Thu,  2 Aug 2007 17:30:48 +0200 (CEST)
Subject: [pypy-svn] r45459 - in pypy/dist/pypy/lang/scheme: . test
Message-ID: <20070802153048.9D9AB80D6@code0.codespeak.net>

Author: jlg
Date: Thu Aug  2 17:30:47 2007
New Revision: 45459

Modified:
   pypy/dist/pypy/lang/scheme/object.py
   pypy/dist/pypy/lang/scheme/test/test_macro.py
Log:
more on macros and ellipses; W_Transformer.substitute changes

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Thu Aug  2 17:30:47 2007
@@ -866,8 +866,8 @@
         self.mdict_lst = mdict_lst
 
 class EllipsisException(SchemeException):
-    def __init__(self, ellipsis):
-        self.mdict_lst = ellipsis.mdict_lst
+    def __init__(self, length):
+        self.length = length
 
 class EllipsisPattern(SchemeException):
     pass
@@ -897,8 +897,8 @@
                 mdict_car = self.matchr(ctx, w_pattcar, w_expr.car)
 
                 try:
-                #we catch EllipsisPattern here because in car
-                # we dont know how to deal with it
+                    #we catch EllipsisPattern here because in car
+                    # we dont know how to deal with it
                     mdict_cdr = self.matchr(ctx, w_patt.cdr, w_expr.cdr)
                 except EllipsisPattern:
                     print "ellipsis matched", w_patt, w_expr
@@ -1007,13 +1007,22 @@
 
         return self.substitute(ctx, template, match_dict)
 
-    def substitute(self, ctx, sexpr, match_dict):
+    def substitute(self, ctx, sexpr, match_dict, ellipsis_cnt=-1):
         if isinstance(sexpr, W_Symbol):
             w_sub = match_dict.get(sexpr.name, None)
             if w_sub is not None:
                 # Hygenic macros close their input forms in the syntactic
                 # enviroment at the point of use
 
+                if isinstance(w_sub, Ellipsis):
+                    if ellipsis_cnt < 0:
+                        raise EllipsisException(len(w_sub.mdict_lst))
+                    else:
+                        mdict = w_sub.mdict_lst[ellipsis_cnt]
+                        w_sub = mdict[sexpr.name]
+                        #for nested ellipsis we should probably raise
+                        # here if w_sub is still Ellipsis
+
                 #not always needed, because w_sub can have no W_Symbol inside
                 if isinstance(w_sub, W_Symbol) and \
                         not isinstance(w_sub, SymbolClosure):
@@ -1023,30 +1032,24 @@
                         not isinstance(w_sub, PairClosure):
                     return PairClosure(ctx, w_sub)
 
-                if isinstance(w_sub, Ellipsis):
-                    raise EllipsisException(w_sub)
-
                 return w_sub
 
             return sexpr
 
         elif isinstance(sexpr, W_Pair):
             try:
-                w_pair = W_Pair(self.substitute(ctx, sexpr.car, match_dict),
-                        self.substitute(ctx, sexpr.cdr, match_dict))
+                w_pair = W_Pair(
+                    self.substitute(ctx, sexpr.car, match_dict, ellipsis_cnt),
+                    self.substitute(ctx, sexpr.cdr, match_dict, ellipsis_cnt))
             except EllipsisException, e:
                 scdr = sexpr.cdr
-                print ">", sexpr, e.mdict_lst
                 if isinstance(scdr, W_Pair) and scdr.car is w_ellipsis:
-                    print ">>", sexpr, e.mdict_lst
-
                     plst = []
-                    for mdict in e.mdict_lst:
-                        zzz = self.substitute(ctx, sexpr.car, mdict)
+                    for i in range(e.length):
+                        zzz = self.substitute(ctx, sexpr.car, match_dict, i)
                         plst.append(zzz)
 
                     ellipsis = plst2lst(plst)
-                    print ellipsis
                     return ellipsis
                 else:
                     raise e

Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Thu Aug  2 17:30:47 2007
@@ -353,3 +353,50 @@
                                (set! z 1)
                                (+ x y z))""").to_number() == 3
 
+def test_ellipsis_mixed():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax set-if-true
+                                 (syntax-rules ()
+                                    ((_ (sym val) ...)
+                                     (begin
+                                       (if sym (set! sym val)) ...))))""")
+
+    eval_(ctx, "(define x #t)")
+    eval_(ctx, "(define y #f)")
+    eval_(ctx, "(define z #t)")
+    eval_(ctx, "(set-if-true (x 1) (y 2) (z 3))")
+    assert eval_(ctx, "x").to_number() == 1
+    assert eval_(ctx, "y").to_boolean() is False
+    assert eval_(ctx, "z").to_number() == 3
+
+def test_ellipsis_wo_ellipsis():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax let-default
+                                 (syntax-rules ()
+                                    ((_ (sym ...) val body ...)
+                                     (let ((sym val) ...) body ...))))""")
+
+    assert eval_(ctx, "(let-default (x y z) 1 (+ x y z))").to_number() == 3
+
+def test_different_ellipsis():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax let2
+                                 (syntax-rules ()
+                                    ((_ (sym ...) (val ...) body ...)
+                                     (let ((sym val) ...) body ...))))""")
+
+    assert eval_(ctx, "(let2 (x y z) (1 2 3) (+ x y z))").to_number() == 6
+
+def test_nested_ellipsis():
+    py.test.skip("in progress")
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax quote-append
+                                 (syntax-rules ()
+                                    ((_ (obj ...) ...)
+                                     (quote (obj ... ...)))))""")
+
+    assert eval_(ctx, """(quote-append (x y)
+                                       (1 2 3 4)
+                                       (+))""").to_string() == \
+            "(x y 1 2 3 4 +)"
+

From arigo at codespeak.net  Thu Aug  2 19:15:40 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 19:15:40 +0200 (CEST)
Subject: [pypy-svn] r45460 - pypy/dist/pypy/rpython
Message-ID: <20070802171540.8F6BA80A4@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 19:15:39 2007
New Revision: 45460

Modified:
   pypy/dist/pypy/rpython/normalizecalls.py
Log:
A hard-to-test fix in assign_inheritence_ids().  This tries to ensure
that newly seen classes all go to the end of the peers list as long as
they don't extend previous hierarchies (no common base class).


Modified: pypy/dist/pypy/rpython/normalizecalls.py
==============================================================================
--- pypy/dist/pypy/rpython/normalizecalls.py	(original)
+++ pypy/dist/pypy/rpython/normalizecalls.py	Thu Aug  2 19:15:39 2007
@@ -273,16 +273,6 @@
 
 # ____________________________________________________________
 
-class Max(object):
-    def __cmp__(self, other):
-        if self is other:
-            return 0
-        else:
-            return 1
-
-MAX = Max()    # a maximum object
-
-
 class TotalOrderSymbolic(ComputedIntSymbolic):
 
     def __init__(self, orderwitness, peers):
@@ -301,14 +291,34 @@
         if self.value is None:
             self.peers.sort()
             for i, peer in enumerate(self.peers):
-                assert peer.value is None
+                assert peer.value is None or peer.value == i
                 peer.value = i
             assert self.value is not None
         return self.value
 
+    def dump(self, annotator):   # for debugging
+        self.peers.sort()
+        mapping = {}
+        for classdef in annotator.bookkeeper.classdefs:
+            if hasattr(classdef, '_unique_cdef_id'):
+                mapping[classdef._unique_cdef_id] = classdef
+        for peer in self.peers:
+            if peer is self:
+                print '==>',
+            else:
+                print '   ',
+            print 'value %4s --' % (peer.value,), peer.orderwitness,
+            if peer.orderwitness[-1] in mapping:
+                print mapping[peer.orderwitness[-1]]
+            else:
+                print
+
 def assign_inheritance_ids(annotator):
     # we sort the classes by lexicographic order of reversed(mro),
-    # which gives a nice depth-first order.
+    # which gives a nice depth-first order.  The classes are turned
+    # into numbers in order to (1) help determinism, (2) ensure that
+    # new hierarchies of classes with no common base classes can be
+    # added later and get higher numbers.
     bk = annotator.bookkeeper
     try:
         lst = bk._inheritance_id_symbolics
@@ -316,11 +326,22 @@
         lst = bk._inheritance_id_symbolics = []
     for classdef in annotator.bookkeeper.classdefs:
         if not hasattr(classdef, 'minid'):
-            witness = list(classdef.getmro())
+            witness = [get_unique_cdef_id(cdef) for cdef in classdef.getmro()]
             witness.reverse()
             classdef.minid = TotalOrderSymbolic(witness, lst)
             classdef.maxid = TotalOrderSymbolic(witness + [MAX], lst)
 
+MAX = 1E100
+_cdef_id_counter = 0
+def get_unique_cdef_id(cdef):
+    global _cdef_id_counter
+    try:
+        return cdef._unique_cdef_id
+    except AttributeError:
+        cdef._unique_cdef_id = _cdef_id_counter
+        _cdef_id_counter += 1
+        return cdef._unique_cdef_id
+
 # ____________________________________________________________
 
 def perform_normalizations(rtyper):

From arigo at codespeak.net  Thu Aug  2 19:16:23 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 19:16:23 +0200 (CEST)
Subject: [pypy-svn] r45461 - pypy/dist/pypy/translator/sandbox
Message-ID: <20070802171623.C50F680E3@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 19:16:23 2007
New Revision: 45461

Modified:
   pypy/dist/pypy/translator/sandbox/rsandbox.py
Log:
Cannot use EOFError here.  If the original program didn't use it already,
then it's a new Exception subclass and normalizecalls is unhappy.


Modified: pypy/dist/pypy/translator/sandbox/rsandbox.py
==============================================================================
--- pypy/dist/pypy/translator/sandbox/rsandbox.py	(original)
+++ pypy/dist/pypy/translator/sandbox/rsandbox.py	Thu Aug  2 19:16:23 2007
@@ -38,10 +38,8 @@
     while length > 0:
         size = rffi.cast(rffi.SIZE_T, length)
         count = rffi.cast(lltype.Signed, ll_write_not_sandboxed(fd, buf, size))
-        if count < 0:
+        if count <= 0:
             raise IOError
-        if count == 0:
-            raise EOFError
         length -= count
         buf = lltype.direct_ptradd(lltype.direct_arrayitems(buf), count)
         buf = rffi.cast(rffi.CCHARP, buf)
@@ -53,10 +51,8 @@
     while got < length:
         size1 = rffi.cast(rffi.SIZE_T, length - got)
         count = rffi.cast(lltype.Signed, ll_read_not_sandboxed(fd, p, size1))
-        if count < 0:
+        if count <= 0:
             raise IOError
-        if count == 0:
-            raise EOFError
         got += count
         p = lltype.direct_ptradd(lltype.direct_arrayitems(p), count)
         p = rffi.cast(rffi.CCHARP, p)

From arigo at codespeak.net  Thu Aug  2 19:17:24 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 19:17:24 +0200 (CEST)
Subject: [pypy-svn] r45462 - in pypy/dist/pypy: rpython
	translator/sandbox/test
Message-ID: <20070802171724.CE56380EB@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 19:17:24 2007
New Revision: 45462

Modified:
   pypy/dist/pypy/rpython/extfunc.py
   pypy/dist/pypy/rpython/extfuncregistry.py
   pypy/dist/pypy/translator/sandbox/test/test_sandbox.py
Log:
Mark the math.*() functions as sandbox-safe, i.e. they are kept unmodified
in executables even if they are built with the --sandbox option.


Modified: pypy/dist/pypy/rpython/extfunc.py
==============================================================================
--- pypy/dist/pypy/rpython/extfunc.py	(original)
+++ pypy/dist/pypy/rpython/extfunc.py	Thu Aug  2 19:17:24 2007
@@ -76,6 +76,8 @@
                            annotation(self.instance.result, self.bookkeeper))
 
 class ExtFuncEntry(ExtRegistryEntry):
+    safe_not_sandboxed = False
+
     def compute_result_annotation(self, *args_s):
         if hasattr(self, 'ann_hook'):
             self.ann_hook()
@@ -117,7 +119,8 @@
                 impl, self.signature_args, hop.s_result)
         else:
             obj = rtyper.type_system.getexternalcallable(args_ll, ll_result,
-                                 name, _external_name=self.name, _callable=fakeimpl)
+                                 name, _external_name=self.name, _callable=fakeimpl,
+                                 _safe_not_sandboxed=self.safe_not_sandboxed)
         vlist = [hop.inputconst(typeOf(obj), obj)] + hop.inputargs(*args_r)
         hop.exception_is_here()
         return hop.genop('direct_call', vlist, r_result)
@@ -125,7 +128,8 @@
 def _register_external(function, args, result=None, export_name=None,
                        llimpl=None, ooimpl=None,
                        llfakeimpl=None, oofakeimpl=None,
-                       annotation_hook=None):
+                       annotation_hook=None,
+                       sandboxsafe=False):
     """
     function: the RPython function that will be rendered as an external function (e.g.: math.floor)
     args: a list containing the annotation of the arguments
@@ -134,10 +138,12 @@
     llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function
     llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter
     annotationhook: optional; a callable that is called during annotation, useful for genc hacks
+    sandboxsafe: use True if the function performs no I/O (safe for --sandbox)
     """
 
     class FunEntry(ExtFuncEntry):
         _about_ = function
+        safe_not_sandboxed = sandboxsafe
         if args is None:
             signature_args = None
         else:

Modified: pypy/dist/pypy/rpython/extfuncregistry.py
==============================================================================
--- pypy/dist/pypy/rpython/extfuncregistry.py	(original)
+++ pypy/dist/pypy/rpython/extfuncregistry.py	Thu Aug  2 19:17:24 2007
@@ -24,7 +24,8 @@
     'floor', 'log', 'log10', 'sin', 'sinh', 'sqrt', 'tan', 'tanh'
     ]
 for name in simple_math_functions:
-    _register_external(getattr(math, name), [float], float, "ll_math.ll_math_%s" % name)
+    _register_external(getattr(math, name), [float], float, "ll_math.ll_math_%s" % name,
+                       sandboxsafe=True)
 
 def frexp_hook():
     from pypy.rpython.extfunctable import record_call
@@ -54,7 +55,8 @@
     oofake = getattr(oo_math, 'll_math_%s' % name, None)
     _register_external(func, args, res, 'll_math.ll_math_%s' % name,
                        llfakeimpl=llfake, oofakeimpl=oofake,
-                       annotation_hook = hook)
+                       annotation_hook = hook,
+                       sandboxsafe=True)
 
 
 # ___________________________

Modified: pypy/dist/pypy/translator/sandbox/test/test_sandbox.py
==============================================================================
--- pypy/dist/pypy/translator/sandbox/test/test_sandbox.py	(original)
+++ pypy/dist/pypy/translator/sandbox/test/test_sandbox.py	Thu Aug  2 19:17:24 2007
@@ -130,3 +130,26 @@
     tail = f.read()
     f.close()
     assert tail == ""
+
+class TestPrintedResults:
+
+    def run(self, entry_point, args, expected):
+        t = Translation(entry_point, backend='c', standalone=True,
+                        sandbox=True)
+        exe = t.compile()
+        from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc
+        proc = SimpleIOSandboxedProc([exe] + args)
+        output, error = proc.communicate()
+        assert error == ''
+        assert output == expected
+
+    def test_safefuncs(self):
+        import math
+        def entry_point(argv):
+            a = float(argv[1])
+            print int(math.floor(a - 0.2)),
+            print int(math.ceil(a)),
+            print int(100.0 * math.sin(a)),
+            print
+            return 0
+        self.run(entry_point, ["3.011"], "2 4 13\n")

From arigo at codespeak.net  Thu Aug  2 19:21:49 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 19:21:49 +0200 (CEST)
Subject: [pypy-svn] r45463 - pypy/dist/pypy/translator/sandbox
Message-ID: <20070802172149.D799880ED@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 19:21:48 2007
New Revision: 45463

Modified:
   pypy/dist/pypy/translator/sandbox/rsandbox.py
Log:
Grumble.  I wouldn't mind a sane approach to separate compilation
instead of this...


Modified: pypy/dist/pypy/translator/sandbox/rsandbox.py
==============================================================================
--- pypy/dist/pypy/translator/sandbox/rsandbox.py	(original)
+++ pypy/dist/pypy/translator/sandbox/rsandbox.py	Thu Aug  2 19:21:48 2007
@@ -119,6 +119,7 @@
         msg.packstring(chr(CFalse) + chr(CFalse))
         msg.packsize_t(rffi.cast(rffi.SIZE_T, CFalse))
         msg.packbuf(buf, CFalse * 5, CFalse * 6)
+        msg.packccharp(lltype.nullptr(rffi.CCHARP.TO))
     try:
         writeall_not_sandboxed(STDOUT, buf, msg.getlength())
     finally:

From arigo at codespeak.net  Thu Aug  2 19:24:38 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 19:24:38 +0200 (CEST)
Subject: [pypy-svn] r45464 - pypy/dist/pypy/translator/sandbox
Message-ID: <20070802172438.172FC80ED@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 19:24:37 2007
New Revision: 45464

Modified:
   pypy/dist/pypy/translator/sandbox/rsandbox.py
Log:
Oups, lltype.nullptr returns a SomePtr(const=<* None>).  More grumbles.


Modified: pypy/dist/pypy/translator/sandbox/rsandbox.py
==============================================================================
--- pypy/dist/pypy/translator/sandbox/rsandbox.py	(original)
+++ pypy/dist/pypy/translator/sandbox/rsandbox.py	Thu Aug  2 19:24:37 2007
@@ -119,7 +119,7 @@
         msg.packstring(chr(CFalse) + chr(CFalse))
         msg.packsize_t(rffi.cast(rffi.SIZE_T, CFalse))
         msg.packbuf(buf, CFalse * 5, CFalse * 6)
-        msg.packccharp(lltype.nullptr(rffi.CCHARP.TO))
+        msg.packccharp(rffi.str2charp(str(CFalse)))
     try:
         writeall_not_sandboxed(STDOUT, buf, msg.getlength())
     finally:

From arigo at codespeak.net  Thu Aug  2 19:35:30 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 19:35:30 +0200 (CEST)
Subject: [pypy-svn] r45465 - pypy/dist/pypy/translator/sandbox
Message-ID: <20070802173530.CB16680EB@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 19:35:30 2007
New Revision: 45465

Added:
   pypy/dist/pypy/translator/sandbox/interact.py   (contents, props changed)
Modified:
   pypy/dist/pypy/translator/sandbox/sandlib.py
Log:
A wrapper tool for CPython to run a sandboxed subprocess: stdin, stdout,
stderr are redirected from the parent to the subprocess, but all other
external functions are forbidden.

A sandboxed pyrolog-c can be run in this way.


Added: pypy/dist/pypy/translator/sandbox/interact.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/translator/sandbox/interact.py	Thu Aug  2 19:35:30 2007
@@ -0,0 +1,17 @@
+#! /usr/bin/env python
+
+"""Interacts with a subprocess translated with --sandbox.
+The subprocess is only allowed to use stdin/stdout/stderr.
+
+Usage:
+    interact.py <executable> <args...>
+"""
+
+import sys
+from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print >> sys.stderr, __doc_
+        sys.exit(2)
+    SimpleIOSandboxedProc(sys.argv[1:]).interact()

Modified: pypy/dist/pypy/translator/sandbox/sandlib.py
==============================================================================
--- pypy/dist/pypy/translator/sandbox/sandlib.py	(original)
+++ pypy/dist/pypy/translator/sandbox/sandlib.py	Thu Aug  2 19:35:30 2007
@@ -92,6 +92,18 @@
         self._error = None
         return (output, error)
 
+    def interact(self, stdin=None, stdout=None, stderr=None):
+        """Interact with the subprocess.  By default, stdin, stdout and
+        stderr are set to the ones from 'sys'."""
+        import sys
+        self._input  = stdin  or sys.stdin
+        self._output = stdout or sys.stdout
+        self._error  = stderr or sys.stderr
+        self.handle_forever()
+        self._input = None
+        self._output = None
+        self._error = None
+
     def do_read(self, fd, size):
         if fd == 0:
             if self._input is None:

From arigo at codespeak.net  Thu Aug  2 19:55:09 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 19:55:09 +0200 (CEST)
Subject: [pypy-svn] r45466 - pypy/dist/pypy/translator/sandbox/test
Message-ID: <20070802175509.0B24480F5@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 19:55:09 2007
New Revision: 45466

Modified:
   pypy/dist/pypy/translator/sandbox/test/test_sandlib.py
Log:
A test just for fun: the sandboxed process calls the external function
foobar().  Clearly, translating code using foobar() only works when
sandboxing.


Modified: pypy/dist/pypy/translator/sandbox/test/test_sandlib.py
==============================================================================
--- pypy/dist/pypy/translator/sandbox/test/test_sandlib.py	(original)
+++ pypy/dist/pypy/translator/sandbox/test/test_sandlib.py	Thu Aug  2 19:55:09 2007
@@ -1,5 +1,6 @@
 import os, StringIO
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rpython.lltypesystem import rffi
 from pypy.translator.sandbox.sandlib import SandboxedProc
 from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc
 from pypy.translator.interactive import Translation
@@ -26,6 +27,10 @@
     do_read  = _make_method("read")
     do_write = _make_method("write")
     do_close = _make_method("close")
+    do_foobar = _make_method("foobar")
+
+    TYPES = SandboxedProc.TYPES.copy()
+    TYPES["foobar"] = "s", "i"
 
 
 def test_lib():
@@ -56,6 +61,22 @@
     proc.handle_forever()
     assert proc.seen == len(proc.expected)
 
+def test_foobar():
+    foobar = rffi.llexternal("foobar", [rffi.CCHARP], rffi.LONG)
+    def entry_point(argv):
+        s = rffi.str2charp(argv[1]); n = foobar(s); rffi.free_charp(s)
+        s = rffi.str2charp(argv[n]); n = foobar(s); rffi.free_charp(s)
+        return n
+    t = Translation(entry_point, backend='c', standalone=True, sandbox=True)
+    exe = t.compile()
+
+    proc = MySandboxedProc([exe, 'spam', 'egg'], expected = [
+        ("foobar", ("spam",), 2),
+        ("foobar", ("egg",), 0),
+        ])
+    proc.handle_forever()
+    assert proc.seen == len(proc.expected)
+
 def test_simpleio():
     def entry_point(argv):
         print "Please enter a number:"

From arigo at codespeak.net  Thu Aug  2 20:04:39 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 20:04:39 +0200 (CEST)
Subject: [pypy-svn] r45467 - pypy/dist/pypy/rpython/test
Message-ID: <20070802180439.1A26680F7@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 20:04:38 2007
New Revision: 45467

Modified:
   pypy/dist/pypy/rpython/test/test_rbuiltin.py
Log:
Fixes


Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py
==============================================================================
--- pypy/dist/pypy/rpython/test/test_rbuiltin.py	(original)
+++ pypy/dist/pypy/rpython/test/test_rbuiltin.py	Thu Aug  2 20:04:38 2007
@@ -216,7 +216,6 @@
         assert self.ll_to_string(res) == 'hello world'
 
     def test_os_dup(self):
-        from pypy.rpython.module.ll_os import dup_lltypeimpl
         import os
         def fn(fd):
             return os.dup(fd)
@@ -228,15 +227,11 @@
         count = 0
         for dir_call in enum_direct_calls(test_llinterp.typer.annotator.translator, fn):
             cfptr = dir_call.args[0]
-            if self.type_system == 'lltype':
-                assert self.get_callable(cfptr.value) == dup_lltypeimpl
-            else:
-                assert self.get_callable(cfptr.value) == os.dup
+            assert self.get_callable(cfptr.value).__name__.startswith('dup')
             count += 1
         assert count == 1
 
     def test_os_open(self):
-        from pypy.rpython.module.ll_os import os_open_lltypeimpl, os_open_oofakeimpl
         tmpdir = str(udir.udir.join("os_open_test"))
         import os
         def wr_open(fname):
@@ -248,10 +243,7 @@
         count = 0
         for dir_call in enum_direct_calls(test_llinterp.typer.annotator.translator, wr_open):
             cfptr = dir_call.args[0]
-            if self.type_system == 'lltype':
-                assert self.get_callable(cfptr.value) == os_open_lltypeimpl
-            else:
-                assert self.get_callable(cfptr.value) == os_open_oofakeimpl
+            assert self.get_callable(cfptr.value).__name__.startswith('os_open')
             count += 1
         assert count == 1
 

From arigo at codespeak.net  Thu Aug  2 20:30:05 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 20:30:05 +0200 (CEST)
Subject: [pypy-svn] r45468 - in pypy/dist/pypy: rpython rpython/module
	rpython/module/test translator/c translator/c/src
Message-ID: <20070802183005.28A258102@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 20:30:03 2007
New Revision: 45468

Modified:
   pypy/dist/pypy/rpython/extfunctable.py
   pypy/dist/pypy/rpython/module/ll_os.py
   pypy/dist/pypy/rpython/module/test/test_ll_os.py
   pypy/dist/pypy/translator/c/extfunc.py
   pypy/dist/pypy/translator/c/src/ll_os.h
Log:
Convert os.access().


Modified: pypy/dist/pypy/rpython/extfunctable.py
==============================================================================
--- pypy/dist/pypy/rpython/extfunctable.py	(original)
+++ pypy/dist/pypy/rpython/extfunctable.py	Thu Aug  2 20:30:03 2007
@@ -179,7 +179,6 @@
 
 # external function declarations
 posix = __import__(os.name)
-declare(os.access   , int           , 'll_os/access')
 declare(os.lseek    , r_longlong    , 'll_os/lseek')
 declare(os.isatty   , bool          , 'll_os/isatty')
 if hasattr(posix, 'ftruncate'):

Modified: pypy/dist/pypy/rpython/module/ll_os.py
==============================================================================
--- pypy/dist/pypy/rpython/module/ll_os.py	(original)
+++ pypy/dist/pypy/rpython/module/ll_os.py	Thu Aug  2 20:30:03 2007
@@ -13,8 +13,8 @@
 from pypy.tool.staticmethods import ClassMethods
 import stat
 from pypy.rpython.extfunc import BaseLazyRegistering, registering
-from pypy.annotation.model import SomeString, SomeInteger, s_ImpossibleValue, \
-    s_None
+from pypy.annotation.model import SomeString, SomeInteger
+from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool
 from pypy.rpython.lltypesystem import rffi
 from pypy.rpython.lltypesystem.rffi import platform
 from pypy.rpython.lltypesystem import lltype
@@ -296,6 +296,26 @@
         self.register(os.close, [int], s_None, llimpl=close_lltypeimpl,
                       export_name="ll_os.ll_os_close", oofakeimpl=os.close)
 
+    @registering(os.access)
+    def register_os_access(self):
+        os_access = rffi.llexternal('access',
+                                    [rffi.CCHARP, rffi.INT],
+                                    rffi.INT)
+
+        def access_lltypeimpl(path, mode):
+            path = rffi.str2charp(path)
+            mode = rffi.cast(rffi.INT, mode)
+            error = rffi.cast(lltype.Signed, os_access(path, mode))
+            rffi.free_charp(path)
+            return error == 0
+
+        def os_access_oofakeimpl(path, mode):
+            return os.access(OOSupport.from_rstr(path), mode)
+
+        self.register(os.access, [str, int], s_Bool, llimpl=access_lltypeimpl,
+                      export_name="ll_os.ll_os_access",
+                      oofakeimpl=os_access_oofakeimpl)
+
     # ------------------------------- os.W* ---------------------------------
 
     w_star = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
@@ -354,10 +374,6 @@
         return cls.to_rstr(os.getcwd())
     ll_os_getcwd.suggested_primitive = True
 
-    def ll_os_access(cls, path, mode):
-        return os.access(cls.from_rstr(path), mode)
-    ll_os_access.suggested_primitive = True
-
     def ll_os_lseek(cls, fd,pos,how):
         return r_longlong(os.lseek(fd,pos,how))
     ll_os_lseek.suggested_primitive = True

Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py
==============================================================================
--- pypy/dist/pypy/rpython/module/test/test_ll_os.py	(original)
+++ pypy/dist/pypy/rpython/module/test/test_ll_os.py	Thu Aug  2 20:30:03 2007
@@ -3,19 +3,22 @@
 from pypy.tool.pytest.modcheck import skipimporterror
 from pypy.translator.c.test.test_genc import compile
 
+from pypy.rpython import extregistry
 from pypy.rpython.lltypesystem.module.ll_os import Implementation as impl
 import sys
 import py
 
+def getllimpl(fn):
+    return extregistry.lookup(fn).lltypeimpl
+
 def test_access():
     filename = str(udir.join('test_access.txt'))
-    rsfilename = impl.to_rstr(filename)
-
     fd = file(filename, 'w')
     fd.close()
 
     for mode in os.R_OK, os.W_OK, os.X_OK, os.R_OK | os.W_OK | os.X_OK:
-        assert os.access(filename, mode) == impl.ll_os_access(rsfilename, mode)
+        result = getllimpl(os.access)(filename, mode)
+        assert result == os.access(filename, mode)
 
 
 def test_getcwd():

Modified: pypy/dist/pypy/translator/c/extfunc.py
==============================================================================
--- pypy/dist/pypy/translator/c/extfunc.py	(original)
+++ pypy/dist/pypy/translator/c/extfunc.py	Thu Aug  2 20:30:03 2007
@@ -23,7 +23,6 @@
 # references to functions, so we cannot insert classmethods here.
 
 EXTERNALS = {
-    impl.ll_os_access.im_func:  'LL_os_access',
     impl.ll_os_stat.im_func:    'LL_os_stat',
     impl.ll_os_fstat.im_func:   'LL_os_fstat',
     impl.ll_os_lstat.im_func:   'LL_os_lstat',

Modified: pypy/dist/pypy/translator/c/src/ll_os.h
==============================================================================
--- pypy/dist/pypy/translator/c/src/ll_os.h	(original)
+++ pypy/dist/pypy/translator/c/src/ll_os.h	Thu Aug  2 20:30:03 2007
@@ -51,7 +51,6 @@
 
 /* prototypes */
 
-int LL_os_access(RPyString *filename, int mode);
 RPySTAT_RESULT* _stat_construct_result_helper(STRUCT_STAT st);
 RPySTAT_RESULT* LL_os_stat(RPyString * fname);
 RPySTAT_RESULT* LL_os_lstat(RPyString * fname);
@@ -99,11 +98,6 @@
 
 #include "ll_osdefs.h"
 
-int LL_os_access(RPyString *filename, int mode) {
-	int n = access(RPyString_AsString(filename), mode);
-	return (n == 0);
-}
-
 #ifdef LL_NEED_OS_STAT
 
 RPySTAT_RESULT* _stat_construct_result_helper(STRUCT_STAT st) {

From arigo at codespeak.net  Thu Aug  2 20:36:39 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Thu,  2 Aug 2007 20:36:39 +0200 (CEST)
Subject: [pypy-svn] r45469 - in pypy/dist/pypy: rpython rpython/module
	translator/c translator/c/src
Message-ID: <20070802183639.619888103@code0.codespeak.net>

Author: arigo
Date: Thu Aug  2 20:36:38 2007
New Revision: 45469

Modified:
   pypy/dist/pypy/rpython/extfunctable.py
   pypy/dist/pypy/rpython/module/ll_os.py
   pypy/dist/pypy/translator/c/extfunc.py
   pypy/dist/pypy/translator/c/src/ll_os.h
Log:
os.getpid()...


Modified: pypy/dist/pypy/rpython/extfunctable.py
==============================================================================
--- pypy/dist/pypy/rpython/extfunctable.py	(original)
+++ pypy/dist/pypy/rpython/extfunctable.py	Thu Aug  2 20:36:38 2007
@@ -202,8 +202,6 @@
 declare(os._exit    , noneannotation, 'll_os/_exit')
 if hasattr(os, 'kill'):
     declare(os.kill     , noneannotation, 'll_os/kill')
-if hasattr(os, 'getpid'):
-    declare(os.getpid   , int,            'll_os/getpid')
 if hasattr(os, 'link'):
     declare(os.link     , noneannotation, 'll_os/link')
 if hasattr(os, 'symlink'):

Modified: pypy/dist/pypy/rpython/module/ll_os.py
==============================================================================
--- pypy/dist/pypy/rpython/module/ll_os.py	(original)
+++ pypy/dist/pypy/rpython/module/ll_os.py	Thu Aug  2 20:36:38 2007
@@ -20,8 +20,10 @@
 from pypy.rpython.lltypesystem import lltype
 
 class RegisterOs(BaseLazyRegistering):
+    UNISTD_INCL = ['unistd.h', 'sys/types.h']
+
     def __init__(self):
-        self.getuid_incl = ['unistd.h', 'sys/types.h']
+        pass   # XXX <arigo> fijal: why do I need this?
     
     # a simple, yet usefull factory
     def register_os_function_returning_int(self, fun, name, **kwds):
@@ -177,12 +179,18 @@
     @registering(os.getuid)
     def register_os_getuid(self):
         self.register_os_function_returning_int(os.getuid, 'getuid',
-                                                includes=self.getuid_incl)
+                                                includes=self.UNISTD_INCL)
 
     @registering(os.geteuid)
     def register_os_geteuid(self):
         self.register_os_function_returning_int(os.geteuid, 'geteuid',
-                                                includes=self.getuid_incl)
+                                                includes=self.UNISTD_INCL)
+
+    if hasattr(os, 'getpid'):
+        @registering(os.getpid)
+        def register_os_getpid(self):
+            self.register_os_function_returning_int(os.getpid, 'getpid',
+                                                    includes=self.UNISTD_INCL)
 
     @registering(os.open)
     def register_os_open(self):
@@ -462,10 +470,6 @@
         return os.umask(mask)
     ll_os_umask.suggested_primitive = True
 
-    def ll_os_getpid(cls):
-        return os.getpid()
-    ll_os_getpid.suggested_primitive = True
-
     def ll_os_kill(cls, pid, sig):
         os.kill(pid, sig)
     ll_os_kill.suggested_primitive = True

Modified: pypy/dist/pypy/translator/c/extfunc.py
==============================================================================
--- pypy/dist/pypy/translator/c/extfunc.py	(original)
+++ pypy/dist/pypy/translator/c/extfunc.py	Thu Aug  2 20:36:38 2007
@@ -46,7 +46,6 @@
     impl.ll_os_chmod.im_func:   'LL_os_chmod',
     impl.ll_os_rename.im_func:  'LL_os_rename',
     impl.ll_os_umask.im_func:   'LL_os_umask',
-    impl.ll_os_getpid.im_func:  'LL_os_getpid',
     impl.ll_os_kill.im_func:    'LL_os_kill',
     impl.ll_os_link.im_func:    'LL_os_link',
     impl.ll_os_symlink.im_func: 'LL_os_symlink',

Modified: pypy/dist/pypy/translator/c/src/ll_os.h
==============================================================================
--- pypy/dist/pypy/translator/c/src/ll_os.h	(original)
+++ pypy/dist/pypy/translator/c/src/ll_os.h	Thu Aug  2 20:36:38 2007
@@ -68,7 +68,6 @@
 void LL_os_chmod(RPyString * path, int mode);
 void LL_os_rename(RPyString * path1, RPyString * path2);
 int LL_os_umask(int mode);
-long LL_os_getpid(void);
 void LL_os_kill(int pid, int sig);
 void LL_os_link(RPyString * path1, RPyString * path2);
 void LL_os_symlink(RPyString * path1, RPyString * path2);
@@ -293,10 +292,6 @@
 	return umask(mode);
 }
 
-long LL_os_getpid(void) {
-	return getpid();
-}
-
 #ifdef HAVE_KILL
 void LL_os_kill(int pid, int sig) {
     int error = kill(pid, sig);

From arigo at codespeak.net  Fri Aug  3 09:50:53 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Fri,  3 Aug 2007 09:50:53 +0200 (CEST)
Subject: [pypy-svn] r45471 - in pypy/dist/pypy: rpython rpython/module
	rpython/module/test translator/c translator/c/src
Message-ID: <20070803075053.4DF8E80EE@code0.codespeak.net>

Author: arigo
Date: Fri Aug  3 09:50:51 2007
New Revision: 45471

Modified:
   pypy/dist/pypy/rpython/extfunctable.py
   pypy/dist/pypy/rpython/module/ll_os.py
   pypy/dist/pypy/rpython/module/test/test_ll_os.py
   pypy/dist/pypy/translator/c/extfunc.py
   pypy/dist/pypy/translator/c/src/ll_os.h
Log:
os.getcwd()...


Modified: pypy/dist/pypy/rpython/extfunctable.py
==============================================================================
--- pypy/dist/pypy/rpython/extfunctable.py	(original)
+++ pypy/dist/pypy/rpython/extfunctable.py	Fri Aug  3 09:50:51 2007
@@ -189,7 +189,6 @@
 declare(os.system   , int           , 'll_os/system')
 declare(os.strerror , str           , 'll_os/strerror')
 declare(os.unlink   , noneannotation, 'll_os/unlink')
-declare(os.getcwd   , str           , 'll_os/getcwd')
 declare(os.chdir    , noneannotation, 'll_os/chdir')
 declare(os.mkdir    , noneannotation, 'll_os/mkdir')
 declare(os.rmdir    , noneannotation, 'll_os/rmdir')

Modified: pypy/dist/pypy/rpython/module/ll_os.py
==============================================================================
--- pypy/dist/pypy/rpython/module/ll_os.py	(original)
+++ pypy/dist/pypy/rpython/module/ll_os.py	Fri Aug  3 09:50:51 2007
@@ -324,6 +324,55 @@
                       export_name="ll_os.ll_os_access",
                       oofakeimpl=os_access_oofakeimpl)
 
+    @registering(os.getcwd)
+    def register_os_getcwd(self):
+        os_getcwd = rffi.llexternal('getcwd',
+                                    [rffi.CCHARP, rffi.SIZE_T],
+                                    rffi.CCHARP)
+
+        def os_getcwd_lltypeimpl():
+            bufsize = 256
+            while True:
+                buf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
+                res = os_getcwd(buf, rffi.cast(rffi.SIZE_T, bufsize))
+                if res:
+                    break   # ok
+                error = rffi.c_errno
+                lltype.free(buf, flavor='raw')
+                if error != errno.ERANGE:
+                    raise OSError(error, "getcwd failed")
+                # else try again with a larger buffer, up to some sane limit
+                bufsize *= 4
+                if bufsize > 1024*1024:  # xxx hard-coded upper limit
+                    raise OSError(error, "getcwd result too large")
+            result = rffi.charp2str(res)
+            lltype.free(buf, flavor='raw')
+            return result
+
+        def os_getcwd_oofakeimpl():
+            return OOSupport.to_rstr(os.getcwd())
+
+        self.register(os.getcwd, [], SomeString(),
+                      "ll_os.ll_os_getcwd", llimpl=os_getcwd_lltypeimpl,
+                      oofakeimpl=os_getcwd_oofakeimpl)
+
+        # '--sandbox' support
+        def os_getcwd_marshal_input(msg, buf, bufsize):
+            msg.packsize_t(bufsize)
+        def os_getcwd_unmarshal_output(msg, buf, bufsize):
+            # the outside process should not send a result larger than
+            # the requested 'bufsize'
+            result = msg.nextstring()
+            n = len(result)
+            if rffi.cast(rffi.SIZE_T, n) >= bufsize:
+                raise OverflowError
+            for i in range(n):
+                buf[i] = result[i]
+            buf[n] = '\x00'
+            return buf
+        os_getcwd._obj._marshal_input = os_getcwd_marshal_input
+        os_getcwd._obj._unmarshal_output = os_getcwd_unmarshal_output
+
     # ------------------------------- os.W* ---------------------------------
 
     w_star = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
@@ -378,10 +427,6 @@
 class BaseOS:
     __metaclass__ = ClassMethods
 
-    def ll_os_getcwd(cls):
-        return cls.to_rstr(os.getcwd())
-    ll_os_getcwd.suggested_primitive = True
-
     def ll_os_lseek(cls, fd,pos,how):
         return r_longlong(os.lseek(fd,pos,how))
     ll_os_lseek.suggested_primitive = True

Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py
==============================================================================
--- pypy/dist/pypy/rpython/module/test/test_ll_os.py	(original)
+++ pypy/dist/pypy/rpython/module/test/test_ll_os.py	Fri Aug  3 09:50:51 2007
@@ -22,8 +22,8 @@
 
 
 def test_getcwd():
-    data = impl.ll_os_getcwd()
-    assert impl.from_rstr(data) == os.getcwd()
+    data = getllimpl(os.getcwd)()
+    assert data == os.getcwd()
 
 def test_strerror():
     data = impl.ll_os_strerror(2)

Modified: pypy/dist/pypy/translator/c/extfunc.py
==============================================================================
--- pypy/dist/pypy/translator/c/extfunc.py	(original)
+++ pypy/dist/pypy/translator/c/extfunc.py	Fri Aug  3 09:50:51 2007
@@ -32,7 +32,6 @@
     impl.ll_os_strerror.im_func: 'LL_os_strerror',
     impl.ll_os_system.im_func:  'LL_os_system',
     impl.ll_os_unlink.im_func:  'LL_os_unlink',
-    impl.ll_os_getcwd.im_func:  'LL_os_getcwd',
     impl.ll_os_chdir.im_func:   'LL_os_chdir',
     impl.ll_os_mkdir.im_func:   'LL_os_mkdir',
     impl.ll_os_rmdir.im_func:   'LL_os_rmdir',

Modified: pypy/dist/pypy/translator/c/src/ll_os.h
==============================================================================
--- pypy/dist/pypy/translator/c/src/ll_os.h	(original)
+++ pypy/dist/pypy/translator/c/src/ll_os.h	Fri Aug  3 09:50:51 2007
@@ -61,7 +61,6 @@
 RPyString *LL_os_strerror(int errnum);
 long LL_os_system(RPyString * fname);
 void LL_os_unlink(RPyString * fname);
-RPyString *LL_os_getcwd(void);
 void LL_os_chdir(RPyString * path);
 void LL_os_mkdir(RPyString * path, int mode);
 void LL_os_rmdir(RPyString * path);
@@ -237,17 +236,6 @@
     }
 }
 
-RPyString *LL_os_getcwd(void) {
-	char buf[PATH_MAX];
-	char *res;
-	res = getcwd(buf, sizeof buf);
-	if (res == NULL) {
-		RPYTHON_RAISE_OSERROR(errno);
-		return NULL;
-	}
-	return RPyString_FromString(buf);
-}
-
 void LL_os_chdir(RPyString * path) {
     int error = chdir(RPyString_AsString(path));
     if (error != 0) {

From arigo at codespeak.net  Fri Aug  3 15:12:58 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Fri,  3 Aug 2007 15:12:58 +0200 (CEST)
Subject: [pypy-svn] r45473 - pypy/dist/pypy/rpython
Message-ID: <20070803131258.5DB938106@code0.codespeak.net>

Author: arigo
Date: Fri Aug  3 15:12:57 2007
New Revision: 45473

Modified:
   pypy/dist/pypy/rpython/extfunc.py
Log:
A case where an exception can obscurely mask a previous problem.


Modified: pypy/dist/pypy/rpython/extfunc.py
==============================================================================
--- pypy/dist/pypy/rpython/extfunc.py	(original)
+++ pypy/dist/pypy/rpython/extfunc.py	Fri Aug  3 15:12:57 2007
@@ -1,3 +1,4 @@
+from pypy.rpython import extregistry
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem.lltype import typeOf
 from pypy.objspace.flow.model import Constant
@@ -20,6 +21,12 @@
     except:
         exc, exc_inst, tb = sys.exc_info()
         for func in funcs:
+            # if the function has already been registered and we got
+            # an exception afterwards, the ExtRaisingEntry would create
+            # a double-registration and crash in an AssertionError that
+            # masks the original problem.  In this case, just re-raise now.
+            if extregistry.lookup(func):
+                raise exc, exc_inst, tb
             class ExtRaisingEntry(ExtRegistryEntry):
                 _about_ = func
                 def compute_result_annotation(self, *args_s):

From arigo at codespeak.net  Fri Aug  3 15:14:17 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Fri,  3 Aug 2007 15:14:17 +0200 (CEST)
Subject: [pypy-svn] r45474 - pypy/dist/pypy/rpython/lltypesystem
Message-ID: <20070803131417.208ED8106@code0.codespeak.net>

Author: arigo
Date: Fri Aug  3 15:14:16 2007
New Revision: 45474

Modified:
   pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py
   pypy/dist/pypy/rpython/lltypesystem/rffi.py
Log:
More laziness in invoking ll2ctypes.


Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py
==============================================================================
--- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py	(original)
+++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py	Fri Aug  3 15:14:16 2007
@@ -141,7 +141,7 @@
         elif isinstance(T, lltype.OpaqueType):
             if T.hints.get('external', None) != 'C':
                 raise TypeError("%s is not external" % T)
-            cls = ctypes.c_char * T.hints['size']
+            cls = ctypes.c_char * T.hints['getsize']()
         else:
             _setup_ctypes_cache()
             if T in _ctypes_cache:
@@ -338,7 +338,7 @@
                 convert_array(container)
             elif isinstance(T.TO, lltype.OpaqueType):
                 container._storage = ctypes.create_string_buffer(
-                    T.TO.hints['size'])
+                    T.TO.hints['getsize']())
             else:
                 raise NotImplementedError(T)
         storage = container._storage
@@ -466,18 +466,23 @@
     return cfunc
 
 def make_callable_via_ctypes(funcptr, cfunc=None):
-    try:
-        if cfunc is None:
-            cfunc = get_ctypes_callable(funcptr)
-    except NotImplementedError, e:
-        def invoke_via_ctypes(*argvalues):
-            raise NotImplementedError, e
+    RESULT = lltype.typeOf(funcptr).TO.RESULT
+    if cfunc is None:
+        cfunc1 = []
     else:
-        RESULT = lltype.typeOf(funcptr).TO.RESULT
-        def invoke_via_ctypes(*argvalues):
-            cargs = [lltype2ctypes(value) for value in argvalues]
-            cres = cfunc(*cargs)
-            return ctypes2lltype(RESULT, cres)
+        cfunc1 = [cfunc]
+
+    def invoke_via_ctypes(*argvalues):
+        if cfunc1:
+            cfunc = cfunc1[0]
+        else:
+            # lazily build the corresponding ctypes function object
+            cfunc = get_ctypes_callable(funcptr)
+            cfunc1.append(cfunc)    # cache
+        # perform the call
+        cargs = [lltype2ctypes(value) for value in argvalues]
+        cres = cfunc(*cargs)
+        return ctypes2lltype(RESULT, cres)
     funcptr._obj._callable = invoke_via_ctypes
 
 def force_cast(RESTYPE, value):

Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py
==============================================================================
--- pypy/dist/pypy/rpython/lltypesystem/rffi.py	(original)
+++ pypy/dist/pypy/rpython/lltypesystem/rffi.py	Fri Aug  3 15:14:16 2007
@@ -100,7 +100,12 @@
         hints = hints.copy()
     hints['external'] = 'C'
     hints['c_name'] = name
-    hints['size'] = platform.sizeof(name, **kwds)
+    def lazy_getsize(result=[]):
+        if not result:
+            size = platform.sizeof(name, **kwds)
+            result.append(size)
+        return result[0]
+    hints['getsize'] = lazy_getsize
     return lltype.Ptr(lltype.OpaqueType(name, hints))
 
 c_errno = CConstant('errno', lltype.Signed)

From arigo at codespeak.net  Fri Aug  3 15:15:41 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Fri,  3 Aug 2007 15:15:41 +0200 (CEST)
Subject: [pypy-svn] r45475 - in pypy/dist/pypy: rpython/module
	translator/c/test
Message-ID: <20070803131541.15D7980E6@code0.codespeak.net>

Author: arigo
Date: Fri Aug  3 15:15:41 2007
New Revision: 45475

Modified:
   pypy/dist/pypy/rpython/module/ll_os.py
   pypy/dist/pypy/translator/c/test/test_extfunc.py
Log:
RPython support for os.listdir().  In-progress; the previous
ros.opendir/readdir/closedir interface is still there but the idea is to
get rid of it.


Modified: pypy/dist/pypy/rpython/module/ll_os.py
==============================================================================
--- pypy/dist/pypy/rpython/module/ll_os.py	(original)
+++ pypy/dist/pypy/rpython/module/ll_os.py	Fri Aug  3 15:15:41 2007
@@ -5,7 +5,7 @@
 # Implementation details about those functions
 # might be found in doc/rffi.txt
 
-import os, errno
+import os, sys, errno
 from pypy.rpython.module.support import ll_strcpy, _ll_strfill, OOSupport
 from pypy.rpython.module.support import to_opaque_object, from_opaque_object
 from pypy.rlib import ros
@@ -15,6 +15,7 @@
 from pypy.rpython.extfunc import BaseLazyRegistering, registering
 from pypy.annotation.model import SomeString, SomeInteger
 from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool
+from pypy.annotation.listdef import s_list_of_strings
 from pypy.rpython.lltypesystem import rffi
 from pypy.rpython.lltypesystem.rffi import platform
 from pypy.rpython.lltypesystem import lltype
@@ -373,6 +374,51 @@
         os_getcwd._obj._marshal_input = os_getcwd_marshal_input
         os_getcwd._obj._unmarshal_output = os_getcwd_unmarshal_output
 
+    @registering(os.listdir)
+    def register_os_listdir(self):
+        # we need a different approach on Windows and on Posix
+        if sys.platform.startswith('win'):
+            XXX   # FindFirstFile, FindNextFile
+        else:
+            INCL = ['sys/types.h', 'dirent.h']
+            DIRP = rffi.COpaque('DIR', includes=INCL)
+            NAME_MAX = platform.intdefined('NAME_MAX', includes=INCL)
+            DIRENTP = rffi.CStruct('dirent',
+                ('d_name', lltype.FixedSizeArray(lltype.Char, NAME_MAX+1)),
+                                   )
+            # XXX so far, DIRENTP cannot be handled by ll2ctypes because
+            #     it contains other fields before 'd_name', at least on Linux
+            os_opendir = rffi.llexternal('opendir', [rffi.CCHARP], DIRP,
+                                         includes = INCL)
+            os_readdir = rffi.llexternal('readdir', [DIRP], DIRENTP,
+                                         includes = INCL)
+            os_closedir = rffi.llexternal('closedir', [DIRP], rffi.INT,
+                                          includes = INCL)
+
+            def os_listdir_lltypeimpl(path):
+                path = rffi.str2charp(path)
+                dirp = os_opendir(path)
+                rffi.free_charp(path)
+                if not dirp:
+                    raise OSError(rffi.c_errno, "os_opendir failed")
+                result = []
+                while True:
+                    direntp = os_readdir(dirp)
+                    if not direntp:
+                        error = rffi.c_errno
+                        break
+                    name = rffi.charp2str(direntp.c_d_name)
+                    if name != '.' and name != '..':
+                        result.append(name)
+                os_closedir(dirp)
+                if error:
+                    raise OSError(error, "os_readdir failed")
+                return result
+
+            self.register(os.listdir, [str], s_list_of_strings,
+                          "ll_os.ll_os_listdir",
+                          llimpl=os_listdir_lltypeimpl)
+
     # ------------------------------- os.W* ---------------------------------
 
     w_star = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',

Modified: pypy/dist/pypy/translator/c/test/test_extfunc.py
==============================================================================
--- pypy/dist/pypy/translator/c/test/test_extfunc.py	(original)
+++ pypy/dist/pypy/translator/c/test/test_extfunc.py	Fri Aug  3 15:15:41 2007
@@ -703,6 +703,18 @@
     compared_with.sort()
     assert result == compared_with
 
+def test_listdir():
+    def mylistdir(s):
+        result = os.listdir(s)
+        return '/'.join(result)
+    func = compile(mylistdir, [str])
+    result = func(str(udir))
+    result = result.split('/')
+    result.sort()
+    compared_with = os.listdir(str(udir))
+    compared_with.sort()
+    assert result == compared_with
+
 if hasattr(posix, 'execv'):
     def test_execv():
         filename = str(udir.join('test_execv.txt'))

From jlg at codespeak.net  Fri Aug  3 15:54:37 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Fri,  3 Aug 2007 15:54:37 +0200 (CEST)
Subject: [pypy-svn] r45476 - in pypy/dist/pypy/lang/scheme: . test
Message-ID: <20070803135437.999078105@code0.codespeak.net>

Author: jlg
Date: Fri Aug  3 15:54:36 2007
New Revision: 45476

Modified:
   pypy/dist/pypy/lang/scheme/object.py
   pypy/dist/pypy/lang/scheme/test/test_macro.py
Log:
nested macros; but no support for (obj ... ...)

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Fri Aug  3 15:54:36 2007
@@ -336,9 +336,8 @@
 
         return self.eval_body(local_ctx, self.body)
 
-def plst2lst(plst):
+def plst2lst(plst, w_cdr=w_nil):
     """coverts python list() of W_Root into W_Pair scheme list"""
-    w_cdr = w_nil
     plst.reverse()
     for w_obj in plst:
         w_cdr = W_Pair(w_obj, w_cdr)
@@ -865,10 +864,16 @@
     def __init__(self, mdict_lst):
         self.mdict_lst = mdict_lst
 
+    def __repr__(self):
+        return "#<e: " + str(self.mdict_lst) + ">"
+
 class EllipsisException(SchemeException):
     def __init__(self, length):
         self.length = length
 
+class EllipsisTemplate(SchemeException):
+    pass
+
 class EllipsisPattern(SchemeException):
     pass
 
@@ -895,7 +900,6 @@
             w_pattcar = w_patt.car
             if isinstance(w_expr, W_Pair):
                 mdict_car = self.matchr(ctx, w_pattcar, w_expr.car)
-
                 try:
                     #we catch EllipsisPattern here because in car
                     # we dont know how to deal with it
@@ -915,6 +919,8 @@
                     for name in mdict_lst[0].keys():
                         mdict_cdr[name] = ellipsis
 
+                    return mdict_cdr
+
                 mdict_car.update(mdict_cdr)
                 return mdict_car
 
@@ -1008,6 +1014,97 @@
         return self.substitute(ctx, template, match_dict)
 
     def substitute(self, ctx, sexpr, match_dict, ellipsis_cnt=-1):
+        #return self.substitute_(ctx, sexpr, match_dict, -1)
+        return self.substituter(ctx, sexpr, match_dict)
+
+    def find_elli(self, expr, mdict):
+        if isinstance(expr, W_Pair):
+            edict_car = self.find_elli(expr.car, mdict)
+            edict_cdr = self.find_elli(expr.cdr, mdict)
+            edict_car.update(edict_cdr)
+            return edict_car
+
+        if isinstance(expr, W_Symbol):
+            val = mdict.get(expr.name, None)
+            if val is None:
+                return {}
+
+            if isinstance(val, Ellipsis):
+                return {expr.name: val}
+
+        return {}
+
+    def substituter(self, ctx, sexpr, match_dict):
+        if isinstance(sexpr, W_Pair):
+            w_car = self.substituter(ctx, sexpr.car, match_dict)
+            try:
+                w_cdr = self.substituter(ctx, sexpr.cdr, match_dict)
+            except EllipsisTemplate:
+                print "ellipsis expand", sexpr
+                try:
+                    w_cdr = self.substituter(ctx, sexpr.cdr.cdr, match_dict)
+                except EllipsisTemplate:
+                    raise NotImplementedError
+
+                plst = []
+                mdict_elli = self.find_elli(sexpr.car, match_dict)
+                elli_len = 0
+                for (key, val) in mdict_elli.items():
+                    elli_len = len(val.mdict_lst)
+
+                for i in range(elli_len):
+                    new_mdict = match_dict.copy()
+                    for (key, val) in mdict_elli.items():
+                        yyy = val.mdict_lst[i][key]
+                        new_mdict[key] = yyy
+
+                    print new_mdict
+                    zzz = self.substituter(ctx, sexpr.car, new_mdict)
+                    plst.append(zzz)
+
+                w_lst = plst2lst(plst, w_cdr)
+                return w_lst
+
+            w_pair = W_Pair(w_car, w_cdr)
+            if isinstance(w_car, W_Symbol):
+                #XXX what if we have here SymbolClosure?
+                # can happen when recursive macro
+                try:
+                    w_macro = ctx.get(w_car.name)
+                    # recursive macro expansion
+                    if isinstance(w_macro, W_DerivedMacro):
+                        return w_macro.expand(ctx, w_pair)
+                except UnboundVariable:
+                    pass
+
+            return w_pair
+
+        if isinstance(sexpr, W_Symbol):
+            if sexpr is w_ellipsis:
+                raise EllipsisTemplate
+
+            w_sub = match_dict.get(sexpr.name, None)
+            if w_sub is not None:
+                # Hygenic macros close their input forms in the syntactic
+                # enviroment at the point of use
+
+                if isinstance(w_sub, Ellipsis):
+                    return w_sub
+
+                #not always needed, because w_sub can have no W_Symbol inside
+                if isinstance(w_sub, W_Symbol) and \
+                        not isinstance(w_sub, SymbolClosure):
+                    return SymbolClosure(ctx, w_sub)
+
+                if isinstance(w_sub, W_Pair) and \
+                        not isinstance(w_sub, PairClosure):
+                    return PairClosure(ctx, w_sub)
+
+                return w_sub
+
+        return sexpr
+
+    def substitute_(self, ctx, sexpr, match_dict, ellipsis_cnt=-1):
         if isinstance(sexpr, W_Symbol):
             w_sub = match_dict.get(sexpr.name, None)
             if w_sub is not None:
@@ -1039,14 +1136,14 @@
         elif isinstance(sexpr, W_Pair):
             try:
                 w_pair = W_Pair(
-                    self.substitute(ctx, sexpr.car, match_dict, ellipsis_cnt),
-                    self.substitute(ctx, sexpr.cdr, match_dict, ellipsis_cnt))
+                    self.substitute_(ctx, sexpr.car, match_dict, ellipsis_cnt),
+                    self.substitute_(ctx, sexpr.cdr, match_dict, ellipsis_cnt))
             except EllipsisException, e:
                 scdr = sexpr.cdr
                 if isinstance(scdr, W_Pair) and scdr.car is w_ellipsis:
                     plst = []
                     for i in range(e.length):
-                        zzz = self.substitute(ctx, sexpr.car, match_dict, i)
+                        zzz = self.substitute_(ctx, sexpr.car, match_dict, i)
                         plst.append(zzz)
 
                     ellipsis = plst2lst(plst)

Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Fri Aug  3 15:54:36 2007
@@ -325,12 +325,12 @@
                                  (syntax-rules ()
                                     ((_ sym ...)
                                      (begin
-                                       (if sym (set! sym 0)) ...))))""")
+                                       (if sym (set! sym 0)) ... #t))))""")
 
     eval_(ctx, "(define x #t)")
     eval_(ctx, "(define y #f)")
     eval_(ctx, "(define z #t)")
-    eval_(ctx, "(zero-if-true x y z)")
+    assert eval_(ctx, "(zero-if-true x y z)").to_boolean() == True
     assert eval_(ctx, "x").to_number() == 0
     assert eval_(ctx, "y").to_boolean() is False
     assert eval_(ctx, "z").to_number() == 0
@@ -388,6 +388,18 @@
     assert eval_(ctx, "(let2 (x y z) (1 2 3) (+ x y z))").to_number() == 6
 
 def test_nested_ellipsis():
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax quote-lists
+                                 (syntax-rules ()
+                                    ((_ (obj ...) ...)
+                                     (quote ((obj ...) ... end)))))""")
+
+    assert eval_(ctx, """(quote-lists (x y)
+                                       (1 2 3 4)
+                                       (+))""").to_string() == \
+            "((x y) (1 2 3 4) (+) end)"
+
+def test_nested_ellipsis2():
     py.test.skip("in progress")
     ctx = ExecutionContext()
     eval_(ctx, """(define-syntax quote-append

From jlg at codespeak.net  Fri Aug  3 15:56:18 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Fri,  3 Aug 2007 15:56:18 +0200 (CEST)
Subject: [pypy-svn] r45477 - pypy/dist/pypy/lang/scheme
Message-ID: <20070803135618.8E5288105@code0.codespeak.net>

Author: jlg
Date: Fri Aug  3 15:56:18 2007
New Revision: 45477

Modified:
   pypy/dist/pypy/lang/scheme/object.py
Log:
old substitute_/substitute removed

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Fri Aug  3 15:56:18 2007
@@ -1013,10 +1013,6 @@
 
         return self.substitute(ctx, template, match_dict)
 
-    def substitute(self, ctx, sexpr, match_dict, ellipsis_cnt=-1):
-        #return self.substitute_(ctx, sexpr, match_dict, -1)
-        return self.substituter(ctx, sexpr, match_dict)
-
     def find_elli(self, expr, mdict):
         if isinstance(expr, W_Pair):
             edict_car = self.find_elli(expr.car, mdict)
@@ -1104,70 +1100,8 @@
 
         return sexpr
 
-    def substitute_(self, ctx, sexpr, match_dict, ellipsis_cnt=-1):
-        if isinstance(sexpr, W_Symbol):
-            w_sub = match_dict.get(sexpr.name, None)
-            if w_sub is not None:
-                # Hygenic macros close their input forms in the syntactic
-                # enviroment at the point of use
-
-                if isinstance(w_sub, Ellipsis):
-                    if ellipsis_cnt < 0:
-                        raise EllipsisException(len(w_sub.mdict_lst))
-                    else:
-                        mdict = w_sub.mdict_lst[ellipsis_cnt]
-                        w_sub = mdict[sexpr.name]
-                        #for nested ellipsis we should probably raise
-                        # here if w_sub is still Ellipsis
-
-                #not always needed, because w_sub can have no W_Symbol inside
-                if isinstance(w_sub, W_Symbol) and \
-                        not isinstance(w_sub, SymbolClosure):
-                    return SymbolClosure(ctx, w_sub)
+    substitute = substituter
 
-                if isinstance(w_sub, W_Pair) and \
-                        not isinstance(w_sub, PairClosure):
-                    return PairClosure(ctx, w_sub)
-
-                return w_sub
-
-            return sexpr
-
-        elif isinstance(sexpr, W_Pair):
-            try:
-                w_pair = W_Pair(
-                    self.substitute_(ctx, sexpr.car, match_dict, ellipsis_cnt),
-                    self.substitute_(ctx, sexpr.cdr, match_dict, ellipsis_cnt))
-            except EllipsisException, e:
-                scdr = sexpr.cdr
-                if isinstance(scdr, W_Pair) and scdr.car is w_ellipsis:
-                    plst = []
-                    for i in range(e.length):
-                        zzz = self.substitute_(ctx, sexpr.car, match_dict, i)
-                        plst.append(zzz)
-
-                    ellipsis = plst2lst(plst)
-                    return ellipsis
-                else:
-                    raise e
-
-            w_paircar = w_pair.car
-            if isinstance(w_paircar, W_Symbol):
-                #XXX what if we have here SymbolClosure?
-                # can happen when recursive macro
-                try:
-                    w_macro = ctx.get(w_paircar.name)
-
-                    # recursive macro expansion
-                    if isinstance(w_macro, W_DerivedMacro):
-                        return w_macro.expand(ctx, w_pair)
-                except UnboundVariable:
-                    pass
-
-            return w_pair
-
-        return sexpr
-            
     def expand_eval(self, ctx, sexpr):
         #we have lexical scopes:
         # 1. macro was defined - self.closure

From jlg at codespeak.net  Fri Aug  3 16:52:29 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Fri,  3 Aug 2007 16:52:29 +0200 (CEST)
Subject: [pypy-svn] r45478 - in pypy/dist/pypy/lang/scheme: . test
Message-ID: <20070803145229.9AF0C80E9@code0.codespeak.net>

Author: jlg
Date: Fri Aug  3 16:52:29 2007
New Revision: 45478

Modified:
   pypy/dist/pypy/lang/scheme/object.py
   pypy/dist/pypy/lang/scheme/test/test_macro.py
Log:
dirty code to handle with (obj ... ...) like ellipses

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Fri Aug  3 16:52:29 2007
@@ -1030,7 +1030,21 @@
 
         return {}
 
-    def substituter(self, ctx, sexpr, match_dict):
+    def plst_append(self, plst):
+        first_cons = plst[0]
+
+        last_cons = None
+        for lst in plst:
+            if last_cons is not None:
+                last_cons.cdr = lst
+
+            while isinstance(lst, W_Pair):
+                last_cons = lst
+                lst = lst.cdr
+
+        return first_cons
+
+    def substituter(self, ctx, sexpr, match_dict, flatten=False):
         if isinstance(sexpr, W_Pair):
             w_car = self.substituter(ctx, sexpr.car, match_dict)
             try:
@@ -1040,7 +1054,9 @@
                 try:
                     w_cdr = self.substituter(ctx, sexpr.cdr.cdr, match_dict)
                 except EllipsisTemplate:
-                    raise NotImplementedError
+                    w_inner = W_Pair(sexpr.car, W_Pair(sexpr.cdr.car, w_nil))
+                    w_outer = W_Pair(w_inner, sexpr.cdr.cdr)
+                    return self.substituter(ctx, w_outer, match_dict, True)
 
                 plst = []
                 mdict_elli = self.find_elli(sexpr.car, match_dict)
@@ -1051,20 +1067,21 @@
                 for i in range(elli_len):
                     new_mdict = match_dict.copy()
                     for (key, val) in mdict_elli.items():
-                        yyy = val.mdict_lst[i][key]
-                        new_mdict[key] = yyy
+                        new_mdict[key] = val.mdict_lst[i][key]
+
+                    sub = self.substituter(ctx, sexpr.car, new_mdict)
+                    plst.append(sub)
+
+                if flatten:
+                    w_lst = self.plst_append(plst)
 
-                    print new_mdict
-                    zzz = self.substituter(ctx, sexpr.car, new_mdict)
-                    plst.append(zzz)
+                else:
+                    w_lst = plst2lst(plst, w_cdr)
 
-                w_lst = plst2lst(plst, w_cdr)
                 return w_lst
 
             w_pair = W_Pair(w_car, w_cdr)
             if isinstance(w_car, W_Symbol):
-                #XXX what if we have here SymbolClosure?
-                # can happen when recursive macro
                 try:
                     w_macro = ctx.get(w_car.name)
                     # recursive macro expansion

Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Fri Aug  3 16:52:29 2007
@@ -400,7 +400,7 @@
             "((x y) (1 2 3 4) (+) end)"
 
 def test_nested_ellipsis2():
-    py.test.skip("in progress")
+    #py.test.skip("in progress")
     ctx = ExecutionContext()
     eval_(ctx, """(define-syntax quote-append
                                  (syntax-rules ()

From jlg at codespeak.net  Fri Aug  3 17:11:39 2007
From: jlg at codespeak.net (jlg at codespeak.net)
Date: Fri,  3 Aug 2007 17:11:39 +0200 (CEST)
Subject: [pypy-svn] r45479 - in pypy/dist/pypy/lang/scheme: . test
Message-ID: <20070803151139.220248105@code0.codespeak.net>

Author: jlg
Date: Fri Aug  3 17:11:38 2007
New Revision: 45479

Modified:
   pypy/dist/pypy/lang/scheme/object.py
   pypy/dist/pypy/lang/scheme/test/test_macro.py
Log:
<obj ... ... rest> rest is not ingored anymore; comments added

Modified: pypy/dist/pypy/lang/scheme/object.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/object.py	(original)
+++ pypy/dist/pypy/lang/scheme/object.py	Fri Aug  3 17:11:38 2007
@@ -1030,7 +1030,7 @@
 
         return {}
 
-    def plst_append(self, plst):
+    def plst_append(self, plst, w_cdr=None):
         first_cons = plst[0]
 
         last_cons = None
@@ -1042,6 +1042,9 @@
                 last_cons = lst
                 lst = lst.cdr
 
+        if w_cdr is not None:
+            last_cons.cdr = w_cdr
+
         return first_cons
 
     def substituter(self, ctx, sexpr, match_dict, flatten=False):
@@ -1052,19 +1055,28 @@
             except EllipsisTemplate:
                 print "ellipsis expand", sexpr
                 try:
+                    #we can still have something behind ellipsis
                     w_cdr = self.substituter(ctx, sexpr.cdr.cdr, match_dict)
                 except EllipsisTemplate:
+                    #it can also be ellipsis
+                    # lets pretend its usual <(obj ...) ...>
+                    # instead of <obj ... ...>
+                    # we will *flatten* the result later
                     w_inner = W_Pair(sexpr.car, W_Pair(sexpr.cdr.car, w_nil))
                     w_outer = W_Pair(w_inner, sexpr.cdr.cdr)
                     return self.substituter(ctx, w_outer, match_dict, True)
 
                 plst = []
+                #find_elli gets part of match_dict relevant to sexpr.car
                 mdict_elli = self.find_elli(sexpr.car, match_dict)
                 elli_len = 0
                 for (key, val) in mdict_elli.items():
+                    assert elli_len == 0 or elli_len == len(val.mdict_lst)
                     elli_len = len(val.mdict_lst)
 
+                #generate elli_len substitutions for ellipsis
                 for i in range(elli_len):
+                    #one level of ellipsis nesting lower
                     new_mdict = match_dict.copy()
                     for (key, val) in mdict_elli.items():
                         new_mdict[key] = val.mdict_lst[i][key]
@@ -1073,7 +1085,9 @@
                     plst.append(sub)
 
                 if flatten:
-                    w_lst = self.plst_append(plst)
+                    #we have to flatten these list, it means append them
+                    # together, and remember about w_cdr
+                    w_lst = self.plst_append(plst, w_cdr)
 
                 else:
                     w_lst = plst2lst(plst, w_cdr)

Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Fri Aug  3 17:11:38 2007
@@ -405,10 +405,10 @@
     eval_(ctx, """(define-syntax quote-append
                                  (syntax-rules ()
                                     ((_ (obj ...) ...)
-                                     (quote (obj ... ...)))))""")
+                                     (quote (obj ... ... end)))))""")
 
     assert eval_(ctx, """(quote-append (x y)
                                        (1 2 3 4)
                                        (+))""").to_string() == \
-            "(x y 1 2 3 4 +)"
+            "(x y 1 2 3 4 + end)"
 

From arigo at codespeak.net  Fri Aug  3 20:09:39 2007
From: arigo at codespeak.net (arigo at codespeak.net)
Date: Fri,  3 Aug 2007 20:09:39 +0200 (CEST)
Subject: [pypy-svn] r45480 - pypy/dist/pypy/lang/scheme/test
Message-ID: <20070803180939.5DDCD80F7@code0.codespeak.net>

Author: arigo
Date: Fri Aug  3 20:09:37 2007
New Revision: 45480

Modified:
   pypy/dist/pypy/lang/scheme/test/test_macro.py
Log:
Review the macro test cases.


Modified: pypy/dist/pypy/lang/scheme/test/test_macro.py
==============================================================================
--- pypy/dist/pypy/lang/scheme/test/test_macro.py	(original)
+++ pypy/dist/pypy/lang/scheme/test/test_macro.py	Fri Aug  3 20:09:37 2007
@@ -78,6 +78,20 @@
     py.test.raises(MatchError, w_transformer.match, closure, w_expr)
 
     # the same binding for => in ctx and closure
+    # XXX
+    # <arigo> I think this should also raise MatchError.  When R5RS says
+    # "same binding" it probably means bound to the same *location*, not
+    # just that there is the same object in both locations; something like
+    # this:   (let ((x 42) (y 42))
+    #           (let-syntax ((foo (syntax-rules (x y)
+    #                          ((foo x) 123)
+    #                          ((foo y) 456)
+    #                          ((foo whatever) 789))))
+    #             (foo y)))
+    #                        ---> 456
+    # but if we change the last line:
+    #             (let ((y 42)) (foo y))))
+    #                        ---> 789
     ctx.put("=>", w_42)
     assert ctx.get("=>") is closure.get("=>")
     w_transformer = eval_(ctx, "(syntax-rules (=>) ((foo => bar) #t))")
@@ -133,6 +147,7 @@
     w_expr = parse_("(_ temp)")
     w_expanded = w_transformer.expand(ctx, w_expr)
     assert w_expanded.to_string() == "(let ((temp 1)) (+ temp temp))"
+    # the two occurrences of 'temp in (+ temp temp) are not the same symbol!
     assert w_transformer.expand_eval(ctx, w_expr).to_number() == 13
 
     #define in closure, should not affect macro eval
@@ -190,6 +205,8 @@
     assert w_transformer.expand_eval(ctx, w_expr).to_number() == 5
 
 def test_transformer_eval():
+    # test direct manipulation of syntax-rules objects:
+    # that's an extension to the R5RS
     ctx = ExecutionContext()
     eval_(ctx, """(define foo (syntax-rules ()
                                      ((_) #t)
@@ -249,6 +266,16 @@
     w_expr = parse_("(bar 42)")
     #should expand directly (recursively) to 42
     assert ctx.get("bar").expand(ctx, w_expr).to_string() == "42"
+    # XXX <arigo> no, I believe it should not expand recursively...
+    # here is another example which returns (foo) in MIT-Scheme where
+    # we get 42 so far:
+    py.test.skip("XXX in-progress (or wrong test?)")
+    ctx = ExecutionContext()
+    eval_(ctx, """(define-syntax foo (syntax-rules ()
+                                          ((foo) #t)))""")
+    eval_(ctx, """(define-syntax bar (syntax-rules ()
+                                          ((bar) (quote (foo)))))""")
+    assert eval_(ctx, "(bar)").to_string() == "(foo)"
 
 def test_let_syntax():
     ctx = ExecutionContext()
@@ -412,3 +439,14 @@
                                        (+))""").to_string() == \
             "(x y 1 2 3 4 + end)"
 
+def test_cornercase1():
+    py.test.skip("in-progress")
+    w_result = eval_noctx("""(let-syntax ((foo (syntax-rules ()
+                                                 ((bar) 'bar))))
+                               (foo))
+                          """)
+    assert w_result.to_string() == 'bar'
+    # <arigo> I think that the correct answer is 'bar, according to
+    # the R5RS, because it says that the keyword bar in the syntax rule
+    # pattern should be ignored during matching and not be a pattern
+    # variable.  But MIT-Scheme disagrees with me...

From arigo at codespeak.net  Fri Aug  3 20:23