From arigo at codespeak.net Sat Dec 1 11:05:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:05:52 +0100 (CET) Subject: [pypy-svn] r49237 - in pypy/dist/pypy: lang/prolog/interpreter translator/backendopt/test translator/goal Message-ID: <20071201100552.7600780C6@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:05:51 2007 New Revision: 49237 Modified: pypy/dist/pypy/lang/prolog/interpreter/term.py pypy/dist/pypy/translator/backendopt/test/test_removenoops.py pypy/dist/pypy/translator/goal/targetrpystonex.py Log: Some remaining usages of id(). Modified: pypy/dist/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/term.py Sat Dec 1 11:05:51 2007 @@ -1,5 +1,6 @@ import math from pypy.rlib.objectmodel import we_are_translated, UnboxedValue +from pypy.rlib.objectmodel import compute_unique_id from pypy.rlib.rarithmetic import intmask from pypy.lang.prolog.interpreter.error import UnificationFailed, UncatchableError from pypy.lang.prolog.interpreter import error @@ -380,7 +381,7 @@ raise UnificationFailed def get_unify_hash(self, heap): - return intmask(id(self) << TAGBITS | self.TAG) + return intmask(hash(self) << TAGBITS | self.TAG) @@ -572,7 +573,7 @@ return c if isinstance(obj1, Var): assert isinstance(obj2, Var) - return rcmp(id(obj1), id(obj2)) + return rcmp(compute_unique_id(obj1), compute_unique_id(obj2)) if isinstance(obj1, Atom): assert isinstance(obj2, Atom) return rcmp(obj1.name, obj2.name) Modified: pypy/dist/pypy/translator/backendopt/test/test_removenoops.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_removenoops.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_removenoops.py Sat Dec 1 11:05:51 2007 @@ -92,7 +92,7 @@ s2 = lltype.malloc(S) llop.keepalive(lltype.Void, s1) llop.keepalive(lltype.Void, s2) - return id(s1) + id(s2) + return lltype.cast_ptr_to_int(s1) + lltype.cast_ptr_to_int(s2) graph, t = get_graph(f, []) remove_superfluous_keep_alive(graph) ops = getops(graph) Modified: pypy/dist/pypy/translator/goal/targetrpystonex.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetrpystonex.py (original) +++ pypy/dist/pypy/translator/goal/targetrpystonex.py Sat Dec 1 11:05:51 2007 @@ -1,4 +1,5 @@ from pypy.translator.test import rpystone +from pypy.rlib.objectmodel import current_object_addr_as_int def make_target_definition(LOOPS): @@ -15,7 +16,7 @@ g.Array2Glob[i][j] = 0 g.PtrGlb = None g.PtrGlbNext = None - return rpystone.pystones(loops), id(g) + return rpystone.pystones(loops), current_object_addr_as_int(g) def target(*args): return entry_point, [int] From arigo at codespeak.net Sat Dec 1 11:20:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:20:54 +0100 (CET) Subject: [pypy-svn] r49238 - in pypy/dist/pypy/rpython: lltypesystem test Message-ID: <20071201102054.1480A80C8@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:20:53 2007 New Revision: 49238 Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py pypy/dist/pypy/rpython/test/test_rclass.py Log: Use again the invert of the address of the object as its default hash, for Boehm. Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rclass.py Sat Dec 1 11:20:53 2007 @@ -701,7 +701,11 @@ return 0 # for None cached = ins.hash_cache if cached == 0: - cached = ins.hash_cache = cast_ptr_to_int(ins) + # XXX this should ideally be done in a GC-dependent way: we only + # need a hash_cache for moving GCs, and we only need the '~' to + # avoid Boehm keeping the object alive if the value is passed + # around + cached = ins.hash_cache = ~cast_ptr_to_int(ins) return cached def ll_inst_type(obj): Modified: pypy/dist/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rclass.py (original) +++ pypy/dist/pypy/rpython/test/test_rclass.py Sat Dec 1 11:20:53 2007 @@ -399,16 +399,19 @@ d = D() def f(): d2 = D() - # xxx check for this CPython peculiarity for now: - # (this is true on top of the llinterp too) - x = ((hash(d2) & sys.maxint) == - (current_object_addr_as_int(d2) & sys.maxint)) - return x, hash(c)+hash(d) + return hash(d2), current_object_addr_as_int(d2), hash(c), hash(d) res = self.interpret(f, []) - - assert res.item0 == True - assert res.item1 == intmask(hash(c)+hash(d)) + # xxx this is too precise, checking the exact implementation + if isinstance(self, OORtypeMixin): + assert res.item0 == res.item1 + else: + assert res.item0 == ~res.item1 + # the following property is essential on top of the lltypesystem + # otherwise prebuilt dictionaries are broken. It's not that + # relevant on top of the ootypesystem though. + assert res.item2 == hash(c) + assert res.item3 == hash(d) def test_type(self): class A: From arigo at codespeak.net Sat Dec 1 11:25:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:25:57 +0100 (CET) Subject: [pypy-svn] r49239 - pypy/dist/pypy/translator/c/test Message-ID: <20071201102557.E256E820D@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:25:57 2007 New Revision: 49239 Modified: pypy/dist/pypy/translator/c/test/test_typed.py Log: Port of r49238. Modified: pypy/dist/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_typed.py (original) +++ pypy/dist/pypy/translator/c/test/test_typed.py Sat Dec 1 11:25:57 2007 @@ -573,6 +573,7 @@ assert f(255) == 255 def test_hash_preservation(self): + from pypy.rlib.objectmodel import current_object_addr_as_int class C: pass class D(C): @@ -581,16 +582,16 @@ d = D() def fn(): d2 = D() - # xxx check for this CPython peculiarity for now: - x = (hash(d2) & sys.maxint) == (id(d2) & sys.maxint) - return x, hash(c)+hash(d) + return hash(d2), current_object_addr_as_int(d2), hash(c), hash(d) f = self.getcompiled(fn) res = f() - assert res[0] == True - assert res[1] == intmask(hash(c)+hash(d)) + # xxx this is too precise, checking the exact implementation + assert res[0] == ~res[1] + assert res[2] == hash(c) + assert res[3] == hash(d) def test_list_basic_ops(self): def list_basic_ops(i, j): From arigo at codespeak.net Sat Dec 1 11:38:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:38:37 +0100 (CET) Subject: [pypy-svn] r49240 - pypy/dist/pypy/rpython/test Message-ID: <20071201103837.196038210@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:38:36 2007 New Revision: 49240 Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py Log: These checks are too precise, e.g. when llvm reuses the test. Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Sat Dec 1 11:38:36 2007 @@ -445,9 +445,12 @@ assert isinstance(x1, int) assert isinstance(x2, (int, r_longlong)) assert isinstance(x3, int) - assert x1 == intmask(x0) # at least on top of llinterp - assert x3 == intmask(x2) # at least on top of llinterp assert x0 != x2 + # the following checks are probably too precise, but work at + # least on top of llinterp + if type(self) is TestLLtype: + assert x1 == intmask(x0) + assert x3 == intmask(x2) class TestLLtype(BaseTestRbuiltin, LLRtypeMixin): From arigo at codespeak.net Sat Dec 1 11:42:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:42:42 +0100 (CET) Subject: [pypy-svn] r49241 - pypy/dist/pypy/translator/tool/test Message-ID: <20071201104242.C317F8210@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:42:42 2007 New Revision: 49241 Modified: pypy/dist/pypy/translator/tool/test/test_cbuild.py Log: This doesn't make much sense any more. Modified: pypy/dist/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/dist/pypy/translator/tool/test/test_cbuild.py Sat Dec 1 11:42:42 2007 @@ -20,26 +20,6 @@ testexec = build_executable([t], eci) out = py.process.cmdexec(testexec) assert out.startswith('hello world') - -def test_compile_threads(): - if sys.platform == 'nt': - py.test.skip("Linux-specific test") - try: - import ctypes - except ImportError: - py.test.skip("Need ctypes for that test") - from pypy.tool.autopath import pypydir - pypydir = py.path.local(pypydir) - csourcedir = pypydir.join('translator', 'c', 'src') - include_dirs = [str(csourcedir.dirpath())] - files = [csourcedir.join('thread.c')] - eci = ExternalCompilationInfo( - include_dirs=include_dirs, - libraries=['pthread'] - ) - mod = compile_c_module(files, '_thread', eci) - cdll = ctypes.CDLL(mod) - assert hasattr(cdll, 'RPyThreadLockInit') class TestEci: def setup_class(cls): From cfbolz at codespeak.net Sat Dec 1 12:49:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Dec 2007 12:49:11 +0100 (CET) Subject: [pypy-svn] r49242 - pypy/dist/pypy/rlib/test Message-ID: <20071201114911.55D038249@code0.codespeak.net> Author: cfbolz Date: Sat Dec 1 12:49:09 2007 New Revision: 49242 Modified: pypy/dist/pypy/rlib/test/test_rope.py Log: the test hit a bug under 2.4 and 2.3 Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sat Dec 1 12:49:09 2007 @@ -479,6 +479,8 @@ def test_find_iterator_unicode(): + if sys.version_info > (2, 5): + py.test.skip("bug in unicode.find that was fixed in 2.5") for searchstring in [ u"\uAAAA\uBBBB\uCCCC", u"\uAAAA", u"", u"\u6666", u"\u6666\u7777\u8888", From cfbolz at codespeak.net Sat Dec 1 12:58:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Dec 2007 12:58:30 +0100 (CET) Subject: [pypy-svn] r49243 - pypy/dist/pypy/rlib/test Message-ID: <20071201115830.27A948249@code0.codespeak.net> Author: cfbolz Date: Sat Dec 1 12:58:29 2007 New Revision: 49243 Modified: pypy/dist/pypy/rlib/test/test_rope.py Log: wrong comparison order - thanks alexander Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sat Dec 1 12:58:29 2007 @@ -479,7 +479,7 @@ def test_find_iterator_unicode(): - if sys.version_info > (2, 5): + if sys.version_info < (2, 5): py.test.skip("bug in unicode.find that was fixed in 2.5") for searchstring in [ u"\uAAAA\uBBBB\uCCCC", u"\uAAAA", u"", u"\u6666", From exarkun at codespeak.net Sat Dec 1 18:05:47 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sat, 1 Dec 2007 18:05:47 +0100 (CET) Subject: [pypy-svn] r49252 - in pypy/dist/pypy: module/posix module/posix/test rpython/module rpython/module/test Message-ID: <20071201170547.B074280C4@code0.codespeak.net> Author: exarkun Date: Sat Dec 1 18:05:46 2007 New Revision: 49252 Added: pypy/dist/pypy/rpython/module/test/execve_tests.py Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/test/test_posix2.py pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_ll_os.py Log: llimpl for os.execve Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sat Dec 1 18:05:46 2007 @@ -75,7 +75,7 @@ interpleveldefs['waitpid'] = 'interp_posix.waitpid' if hasattr(os, 'execv'): interpleveldefs['execv'] = 'interp_posix.execv' - if hasattr(os, 'execve') and 0: # XXX XXX in-progress + if hasattr(os, 'execve'): interpleveldefs['execve'] = 'interp_posix.execve' if False and hasattr(os, 'uname'): interpleveldefs['uname'] = 'interp_posix.uname' Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sat Dec 1 18:05:46 2007 @@ -206,8 +206,9 @@ t([3, "a"]) def test_execve(self): - skip("Not implemented") os = self.posix + if not hasattr(os, "fork"): + skip("Need fork() to test execve()") pid = os.fork() if pid == 0: os.execve("/usr/bin/env", ["env", "python", "-c", "import os; open('onefile', 'w').write(os.environ['ddd'])"], {'ddd':'xxx'}) Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sat Dec 1 18:05:46 2007 @@ -137,6 +137,36 @@ return extdef([str, [str]], s_ImpossibleValue, llimpl=execv_llimpl, export_name="ll_os.ll_os_execv") + + @registering_if(os, 'execve') + def register_os_execve(self): + os_execve = self.llexternal( + 'execve', [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT) + + def execve_llimpl(path, args, env): + # XXX Check path, args, env for \0 and raise TypeErrors as + # appropriate + envstrs = [] + for item in env.iteritems(): + envstrs.append("%s=%s" % item) + + l_args = rffi.liststr2charpp(args) + l_env = rffi.liststr2charpp(envstrs) + os_execve(path, l_args, l_env) + + # XXX untested + rffi.free_charpp(l_env) + rffi.free_charpp(l_args) + + raise OSError(rposix.get_errno(), "execve failed") + + return extdef( + [str, [str], {str: str}], + s_ImpossibleValue, + llimpl=execve_llimpl, + export_name="ll_os.ll_os_execve") + + @registering_if(posix, 'spawnv') def register_os_spawnv(self): os_spawnv = self.llexternal('spawnv', Added: pypy/dist/pypy/rpython/module/test/execve_tests.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/module/test/execve_tests.py Sat Dec 1 18:05:46 2007 @@ -0,0 +1,25 @@ + +""" +Functions which call the llimpl execve function with various arguments. Since +execve replaces the process with a new one in the successful case, these are +here to be run as a child process of the test process. +""" + +import os, sys + +from pypy.rpython.module.test.test_ll_os import EXECVE_ENV, getllimpl + +execve = getllimpl(os.execve) + + +def execve_true(): + execve("/bin/true", ["/bin/true"], {}) + +def execve_false(): + execve("/bin/false", ["/bin/false"], {}) + +def execve_env(): + execve("/usr/bin/env", ["/usr/bin/env"], EXECVE_ENV) + +if __name__ == '__main__': + globals()[sys.argv[1]]() Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_os.py Sat Dec 1 18:05:46 2007 @@ -1,8 +1,11 @@ import os +from py.path import local + from pypy.tool.udir import udir from pypy.translator.c.test.test_genc import compile from pypy.rpython import extregistry +import errno import sys import py @@ -58,6 +61,43 @@ assert file(filename).read().strip() == '2' os.unlink(filename) + +EXECVE_ENV = {"foo": "bar", "baz": "quux"} +execve_tests = str(local(__file__).dirpath().join('execve_tests.py')) + +def test_execve(): + if os.name != 'posix': + py.test.skip('posix specific function') + base = sys.executable + " " + execve_tests + " " + + # Test exit status and code + result = os.system(base + "execve_true") + assert os.WIFEXITED(result) + assert os.WEXITSTATUS(result) == 0 + result = os.system(base + "execve_false") + assert os.WIFEXITED(result) + assert os.WEXITSTATUS(result) == 1 + + # Test environment + result = os.popen(base + "execve_env").read() + assert dict([line.split('=') for line in result.splitlines()]) == EXECVE_ENV + + # These won't actually execute anything, so they don't need a child process + # helper. + execve = getllimpl(os.execve) + + # If the target does not exist, an OSError should result + info = py.test.raises( + OSError, execve, execve_tests + "-non-existent", [], {}) + assert info.value.errno == errno.ENOENT + + # If the target is not executable, an OSError should result + info = py.test.raises( + OSError, execve, execve_tests, [], {}) + assert info.value.errno == errno.EACCES + + + class ExpectTestOs: def setup_class(cls): if not hasattr(os, 'ttyname'): From niko at codespeak.net Sat Dec 1 18:16:58 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Sat, 1 Dec 2007 18:16:58 +0100 (CET) Subject: [pypy-svn] r49253 - pypy/dist/pypy/translator/jvm Message-ID: <20071201171658.76FC88166@code0.codespeak.net> Author: niko Date: Sat Dec 1 18:16:55 2007 New Revision: 49253 Modified: pypy/dist/pypy/translator/jvm/generator.py Log: update the debug info we print out: now the line number corresponds to the line in the file, and we emit local variable debug information with the name of the variable from the rpython source. this helps when viewing code in java debuggers Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Sat Dec 1 18:16:55 2007 @@ -497,8 +497,10 @@ def __init__(self, classty, superclassty): self.class_type = classty self.superclass_type = superclassty + self.line_number = 1 def out(self, arg): self.file.write(arg) + self.line_number += arg.count("\n") class FunctionState(object): """ When you invoked begin_function(), one of these objects is allocated @@ -508,20 +510,27 @@ self.local_vars = {} self.function_arguments = [] self.instr_counter = 0 - def add_var(self, jvar, jtype): + def add_var(self, jvar, jtype, is_param): """ Adds new entry for variable 'jvar', of java type 'jtype' """ idx = self.next_offset self.next_offset += jtype.descriptor.type_width() if jvar: assert jvar.name not in self.local_vars # never been added before - self.local_vars[jvar.name] = idx - self.function_arguments.append((jtype, idx)) + self.local_vars[jvar.name] = (idx, jtype) + if is_param: + self.function_arguments.append((jtype, idx)) return idx def var_offset(self, jvar, jtype): """ Returns offset for variable 'jvar', of java type 'jtype' """ if jvar.name in self.local_vars: - return self.local_vars[jvar.name] - return self.add_var(jvar, jtype) + return self.local_vars[jvar.name][0] + return self.add_var(jvar, jtype, False) + def var_info_list(self): + var_info_list = [None] * self.next_offset + for name, (idx, jtype) in self.local_vars.items(): + var_info_list[idx] = (name, jtype) + return var_info_list + # ___________________________________________________________________________ @@ -654,7 +663,7 @@ for idx, ty in enumerate(argtypes): if idx < len(argvars): var = argvars[idx] else: var = None - self.curfunc.add_var(var, ty) + self.curfunc.add_var(var, ty, True) # Prepare a map for the local variable indices we will add # Let the subclass do the rest of the work; note that it does # not need to know the argvars parameter, so don't pass it @@ -1371,12 +1380,34 @@ funcname, "".join([a.descriptor for a in argtypes]), rettype.descriptor)) - self._abstract_method = abstract + self.abstract_method = abstract + + if not self.abstract_method: + self.function_start_label = self.unique_label( + 'function_start', True) def _end_function(self): - if not self._abstract_method: + + if not self.abstract_method: + function_end_label = self.unique_label('function_end', True) + self.curclass.out('.limit stack 100\n') # HACK, track max offset self.curclass.out('.limit locals %d\n' % self.curfunc.next_offset) + + # Declare debug information for each variable: + var_info_list = self.curfunc.var_info_list() + for idx, data in enumerate(var_info_list): + if data: + name, jtype = data + if jtype is not jVoid: + self.curclass.out( + '.var %d is %s %s from %s to %s\n' % ( + idx, + name, + jtype.descriptor, + self.function_start_label.label, + function_end_label.label)) + self.curclass.out('.end method\n') def mark(self, lbl): @@ -1394,7 +1425,7 @@ return str(arg) strargs = [jasmin_syntax(arg) for arg in args] instr_text = '%s %s' % (jvmstr, " ".join(strargs)) - self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) + self.curclass.out(' .line %d\n' % self.curclass.line_number) self.curclass.out(' %-60s\n' % (instr_text,)) self.curfunc.instr_counter+=1 From niko at codespeak.net Sat Dec 1 18:59:01 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Sat, 1 Dec 2007 18:59:01 +0100 (CET) Subject: [pypy-svn] r49254 - in pypy/dist/pypy/translator: jvm jvm/src/pypy oosupport/test_template Message-ID: <20071201175901.9BCD28128@code0.codespeak.net> Author: niko Date: Sat Dec 1 18:59:01 2007 New Revision: 49254 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/typesystem.py pypy/dist/pypy/translator/oosupport/test_template/builtin.py Log: fix math.modf in the jvm: 1. the code in PyPy.java was slightly wrong 2. the static signature cannot reference generated objects, so when they are returned we must use jObject instead and insert a downcast 3. add a test to oosupport/builtin for modf Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Sat Dec 1 18:59:01 2007 @@ -1047,15 +1047,32 @@ def call_primitive(self, op, module, name): callee = op.args[0].value - argtypes, rettype = self.db.types_for_signature( + jargtypes, jrettype = self.db.types_for_signature( callee._TYPE.ARGS, callee._TYPE.RESULT) + + # Determine what class the primitive is implemented in: if module == 'll_os': jcls = jll_os else: jcls = jPyPy - mthd = Method.v(jcls, name, argtypes, rettype) + + # Determine the method signature: + # n.b.: if the method returns a generated type, then + # it's static type will be Object. This is because + # the method cannot directly refer to the Java type in + # .java source, as its name is not yet known. + if jrettype.is_generated(): + mthd = Method.v(jcls, name, jargtypes, jObject) + else: + mthd = Method.v(jcls, name, jargtypes, jrettype) + + # Invoke the method self.emit(mthd) + # Cast the result, if needed + if jrettype.is_generated(): + self.downcast_jtype(jrettype) + def prepare_call_oostring(self, OOTYPE): # Load the PyPy object pointer onto the stack: self.push_pypy() Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Sat Dec 1 18:59:01 2007 @@ -20,9 +20,9 @@ from pypy.rpython.ootypesystem import \ ootype, rclass from pypy.translator.jvm.typesystem import \ - JvmClassType, jString, jStringArray, jVoid, jThrowable, jInt, \ + JvmGeneratedClassType, jString, jStringArray, jVoid, jThrowable, jInt, \ jObject, JvmType, jStringBuilder, jPyPyInterlink, jCallbackInterfaces, \ - JvmInterfaceType, jPyPy + JvmGeneratedInterfaceType, jPyPy from pypy.translator.jvm.opcodes import \ opcodes from pypy.translator.jvm.option import \ @@ -445,7 +445,7 @@ OOFunction._render_op(self, op) -class StaticMethodInterface(Node, JvmClassType): +class StaticMethodInterface(Node, JvmGeneratedClassType): """ We generate an abstract base class when we need function pointers, which correspond to constants of StaticMethod ootype. We need a @@ -467,7 +467,7 @@ argtypes: list of JvmTypes rettype: JvmType """ - JvmClassType.__init__(self, name) + JvmGeneratedClassType.__init__(self, name) assert isinstance(jrettype, JvmType) self.java_argument_types = [self] + list(jargtypes) self.java_return_type = jrettype @@ -539,7 +539,7 @@ gen.end_class() -class StaticMethodImplementation(Node, JvmClassType): +class StaticMethodImplementation(Node, JvmGeneratedClassType): """ In addition to the StaticMethodInterface, we must generate an implementation for each specific method that is called. These @@ -567,7 +567,7 @@ } """ def __init__(self, name, super_class, bound_to_jty, impl_method): - JvmClassType.__init__(self, name) + JvmGeneratedClassType.__init__(self, name) self.super_class = super_class self.impl_method = impl_method self.dump_method = ConstantStringDumpMethod( @@ -633,13 +633,13 @@ gen.end_function() gen.end_class() -class Interface(Node, JvmInterfaceType): +class Interface(Node, JvmGeneratedInterfaceType): """ Represents an interface to be generated. The only class that we currently generate into an interface is ootype.ROOT. """ def __init__(self, name): - JvmClassType.__init__(self, name) + JvmGeneratedInterfaceType.__init__(self, name) self.super_class = jObject self.rendered = False self.properties = {} @@ -675,7 +675,7 @@ gen.end_class() -class Class(Node, JvmClassType): +class Class(Node, JvmGeneratedClassType): """ Represents a class to be emitted. Note that currently, classes are emitted all in one shot, not piecemeal. """ @@ -685,7 +685,7 @@ 'name' should be a fully qualified Java class name like "java.lang.String", supercls is a Class object """ - JvmClassType.__init__(self, name) + JvmGeneratedClassType.__init__(self, name) self.rendered = False # has rendering occurred? self.abstract = False # is this an abstract class? self.fields = {} # maps field name to jvmgen.Field object Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Sat Dec 1 18:59:01 2007 @@ -988,13 +988,8 @@ } public Object ll_math_modf(double x) { - if (x >= 0) { - double floor_x = Math.floor(x); - return interlink.recordFloatFloat(floor_x, x - floor_x); - } - - double ceil_x = Math.ceil(x); - return interlink.recordFloatFloat(ceil_x, x + ceil_x); + double integer_x = (x >= 0 ? Math.floor(x) : Math.ceil(x)); + return interlink.recordFloatFloat(x - integer_x, integer_x); } public double ll_math_exp(double x) { Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Sat Dec 1 18:59:01 2007 @@ -127,6 +127,11 @@ exist on this type. """ raise NotImplementedException + def is_generated(self): + """ Indicates whether the source for this type is generated by + pypy. """ + return False + def __repr__(self): return "%s<%s>" % (self.__class__.__name__, self.descriptor) @@ -146,9 +151,21 @@ def lookup_method(self, methodnm): raise KeyError(fieldnm) # we treat as opaque type +class JvmGeneratedClassType(JvmClassType): + """ Abstract class extended by the classes in node.py that represent + generated classes """ + def is_generated(self): + return True + class JvmInterfaceType(JvmClassType): pass +class JvmGeneratedInterfaceType(JvmInterfaceType): + """ Abstract class extended by the classes in node.py that represent + generated interfaces """ + def is_generated(self): + return True + jIntegerClass = JvmClassType('java.lang.Integer') jLongClass = JvmClassType('java.lang.Long') jDoubleClass = JvmClassType('java.lang.Double') Modified: pypy/dist/pypy/translator/oosupport/test_template/builtin.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/builtin.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/builtin.py Sat Dec 1 18:59:01 2007 @@ -1,4 +1,4 @@ -import os +import os, math import errno import stat from py.builtin import sorted @@ -138,6 +138,16 @@ # XXX: remember to test ll_os_readlink and ll_os_pipe as soon as # they are implemented + def test_math_modf(self): + def fn(x): + return math.modf(x) + for x in (.5, 1, 1.5): + for y in (1, -1): + act_res = self.interpret(fn, [x*y]) + exp_res = math.modf(x*y) + assert act_res.item0 == exp_res[0] + assert act_res.item1 == exp_res[1] + class BaseTestTime(llBaseTestTime): From niko at codespeak.net Sat Dec 1 19:02:44 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Sat, 1 Dec 2007 19:02:44 +0100 (CET) Subject: [pypy-svn] r49255 - pypy/dist/pypy/translator/jvm Message-ID: <20071201180244.7A49E812A@code0.codespeak.net> Author: niko Date: Sat Dec 1 19:02:44 2007 New Revision: 49255 Modified: pypy/dist/pypy/translator/jvm/node.py Log: use the new is_generated() test in InterlinkFunction to determine when the rewrite the signature to use Object, since it applies there as well -- before we just checked if the type was a reference type Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Sat Dec 1 19:02:44 2007 @@ -795,11 +795,10 @@ self.name = name self.helper = helper - # The functions in Interlink.java either return Object, - # because they are returning an instance of a class generated - # by us which the JVM doesn't know about, or they return a - # scalar. - if self.helper.return_type.descriptor.is_reference(): + # Since the names of classes we generate are not statically + # known, the functions in Interlink.java simply return + # Object when they create an instance of one of those types. + if self.helper.return_type.is_generated(): self.return_type = jObject else: self.return_type = self.helper.return_type From xoraxax at codespeak.net Sat Dec 1 19:14:39 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 1 Dec 2007 19:14:39 +0100 (CET) Subject: [pypy-svn] r49256 - pypy/dist/pypy/translator/goal Message-ID: <20071201181439.C10218193@code0.codespeak.net> Author: xoraxax Date: Sat Dec 1 19:14:39 2007 New Revision: 49256 Modified: pypy/dist/pypy/translator/goal/app_main.py pypy/dist/pypy/translator/goal/nanos.py Log: app_main does not use the normal os module now anymore. Nanos can expose its module even in scenarios without a pypy interpreter - this makes app_main testable again. This should fix the broken -u option. Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Sat Dec 1 19:14:39 2007 @@ -401,8 +401,8 @@ if __name__ == '__main__': - import os import autopath + import nanos # obscure! try removing the following line, see how it crashes, and # guess why... ImStillAroundDontForgetMe = sys.modules['__main__'] @@ -420,5 +420,6 @@ from pypy.module.sys.version import PYPY_VERSION sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path + os = nanos.os_module_for_testing sys.exit(entry_point(sys.argv[0], sys.argv[1:], os)) #sys.exit(entry_point('app_main.py', sys.argv[1:])) Modified: pypy/dist/pypy/translator/goal/nanos.py ============================================================================== --- pypy/dist/pypy/translator/goal/nanos.py (original) +++ pypy/dist/pypy/translator/goal/nanos.py Sat Dec 1 19:14:39 2007 @@ -35,7 +35,7 @@ app_os = applevel(r''' # NOT_RPYTHON - from os import sep, pathsep + from os import sep, pathsep, getenv, name, fdopen try: from os import readlink except ImportError: @@ -53,3 +53,14 @@ space.setattr(w_os, space.wrap('path'), w_os_path) space.setattr(w_os, space.wrap('getenv'), space.wrap(getenv_w)) return w_os + + +# in order to be able to test app_main without the pypy interpreter +# we create the nanos module with the same names here like it would +# be created while translation +path_module_for_testing = type(os)("os.path") +os_module_for_testing = type(os)("os") +os_module_for_testing.path = path_module_for_testing +eval(app_os_path.code, path_module_for_testing.__dict__) +eval(app_os.code, os_module_for_testing.__dict__) + From xoraxax at codespeak.net Sat Dec 1 19:41:52 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 1 Dec 2007 19:41:52 +0100 (CET) Subject: [pypy-svn] r49257 - pypy/dist/pypy/translator/goal Message-ID: <20071201184152.4A36981BC@code0.codespeak.net> Author: xoraxax Date: Sat Dec 1 19:41:51 2007 New Revision: 49257 Removed: pypy/dist/pypy/translator/goal/Ratthing-b246-benchs.txt pypy/dist/pypy/translator/goal/module-list.example pypy/dist/pypy/translator/goal/module-list.pedronis Log: I think these files are not needed anymore. From xoraxax at codespeak.net Sat Dec 1 19:49:32 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 1 Dec 2007 19:49:32 +0100 (CET) Subject: [pypy-svn] r49258 - pypy/dist/pypy/translator Message-ID: <20071201184932.A836A8190@code0.codespeak.net> Author: xoraxax Date: Sat Dec 1 19:49:32 2007 New Revision: 49258 Modified: pypy/dist/pypy/translator/driver.py Log: Remove two status messages in the regular code path. Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Sat Dec 1 19:49:32 2007 @@ -322,12 +322,11 @@ def sanity_check_annotation(self): translator = self.translator irreg = query.qoutput(query.check_exceptblocks_qgen(translator)) - if not irreg: - self.log.info("All exceptblocks seem sane") + if irreg: + self.log.info("Some exceptblocks seem insane") lost = query.qoutput(query.check_methods_qgen(translator)) assert not lost, "lost methods, something gone wrong with the annotation of method defs" - self.log.info("No lost method defs") so = query.qoutput(query.polluted_qgen(translator)) tot = len(translator.graphs) From arigo at codespeak.net Sun Dec 2 12:01:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:01:27 +0100 (CET) Subject: [pypy-svn] r49265 - pypy/dist/pypy/translator/c/test Message-ID: <20071202110127.CD2378166@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:01:27 2007 New Revision: 49265 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: Forgot to check this in. Another forgotten id() usage. Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Sun Dec 2 12:01:27 2007 @@ -841,6 +841,7 @@ should_be_moving = True def test_many_ids(self): + from pypy.rlib.objectmodel import compute_unique_id class A(object): pass def f(): @@ -852,7 +853,7 @@ # remember the ids, it will trigger some collections itself i = 0 while i < len(alist): - idarray[i] = id(alist[i]) + idarray[i] = compute_unique_id(alist[i]) i += 1 j = 0 while j < 2: @@ -860,7 +861,7 @@ [A() for i in range(20000)] i = 0 while i < len(alist): - if idarray[i] != id(alist[i]): + if idarray[i] != compute_unique_id(alist[i]): return j * 1000000 + i i += 1 j += 1 From arigo at codespeak.net Sun Dec 2 12:07:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:07:43 +0100 (CET) Subject: [pypy-svn] r49266 - pypy/dist/pypy/module/posix Message-ID: <20071202110743.A303E814A@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:07:43 2007 New Revision: 49266 Modified: pypy/dist/pypy/module/posix/interp_posix.py Log: * Style fixes. * What was that ValueError case here for? Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Dec 2 12:07:43 2007 @@ -476,16 +476,14 @@ args: iterable of arguments env: dictionary of strings mapping to strings """ + args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)] + env = {} + w_keys = space.call_method(w_env, 'keys') + for w_key in space.unpackiterable(w_keys): + w_value = space.getitem(w_env, w_key) + env[space.str_w(w_key)] = space.str_w(w_value) try: - args = [space.str_w(i) for i in space.unpackiterable(w_args)] - env = {} - keys = space.call_function(space.getattr(w_env, space.wrap('keys'))) - for key in space.unpackiterable(keys): - value = space.getitem(w_env, key) - env[space.str_w(key)] = space.str_w(value) os.execve(command, args, env) - except ValueError, e: - raise OperationError(space.w_ValueError, space.wrap(str(e))) except OSError, e: raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] From arigo at codespeak.net Sun Dec 2 12:36:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:36:27 +0100 (CET) Subject: [pypy-svn] r49267 - pypy/dist/pypy/tool/pytest Message-ID: <20071202113627.F38868170@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:36:26 2007 New Revision: 49267 Modified: pypy/dist/pypy/tool/pytest/appsupport.py Log: We can get a real None here, e.g. if the signature of the app_test function itself is bogus. Modified: pypy/dist/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/dist/pypy/tool/pytest/appsupport.py (original) +++ pypy/dist/pypy/tool/pytest/appsupport.py Sun Dec 2 12:36:26 2007 @@ -112,7 +112,7 @@ def __init__(self, space, apptb): l = [] - while apptb is not space.w_None: + while apptb is not space.w_None and apptb is not None: l.append(self.Entry(space, apptb)) apptb = space.getattr(apptb, space.wrap('tb_next')) list.__init__(self, l) From cfbolz at codespeak.net Sun Dec 2 12:39:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 12:39:32 +0100 (CET) Subject: [pypy-svn] r49268 - in pypy/dist/pypy/rlib: . test Message-ID: <20071202113932.DE29B8135@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 12:39:32 2007 New Revision: 49268 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: fix a rope multiplication: make the resulting ropes as balanced as possible by default. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 2 12:39:32 2007 @@ -8,7 +8,7 @@ NBITS = int(math.log(sys.maxint) / LOG2) + 2 # XXX should optimize the numbers -NEW_NODE_WHEN_LENGTH = 16 +NEW_NODE_WHEN_LENGTH = 32 CONVERT_WHEN_SMALLER = 8 MAX_DEPTH = 32 # maybe should be smaller CONCATENATE_WHEN_MULTIPLYING = 128 @@ -476,37 +476,33 @@ return node.getslice(0, stop) - def multiply(node, times): if times <= 0: return LiteralStringNode.EMPTY if times == 1: return node - end_length = node.length() * times - num_bits = 2 - mask = times >> 2 - while mask: - num_bits += 1 - mask >>= 1 - result = node - mask = 1 << (num_bits - 2) - #import pdb; pdb.set_trace() - for i in range(num_bits - 1): - if mask & times: - if result.length() < CONCATENATE_WHEN_MULTIPLYING: - result = concatenate(result, result) - result = concatenate(result, node) + twopower = node + number = 1 + result = None + while number < times: + if number & times: + if result is None: + result = twopower + elif result.length() < CONCATENATE_WHEN_MULTIPLYING: + result = concatenate(result, twopower) else: - result = BinaryConcatNode(result, result) - result = BinaryConcatNode(result, node) + result = BinaryConcatNode(result, twopower) + try: + number = ovfcheck(number * 2) + except OverflowError: + break + if twopower.length() < CONCATENATE_WHEN_MULTIPLYING: + twopower = concatenate(twopower, twopower) else: - if result.length() < CONCATENATE_WHEN_MULTIPLYING: - result = concatenate(result, result) - else: - result = BinaryConcatNode(result, result) - mask >>= 1 + twopower = BinaryConcatNode(twopower, twopower) return result + def join(node, l): if node.length() == 0: return rebalance(l) @@ -538,7 +534,7 @@ first_node = None while stack: curr = stack.pop() - while isinstance(curr, BinaryConcatNode) and not curr.balanced: + while isinstance(curr, BinaryConcatNode) and not curr.check_balanced(): stack.append(curr.right) curr = curr.left Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sun Dec 2 12:39:32 2007 @@ -838,3 +838,7 @@ res = str_decode_utf8(node) assert res is None + +def test_multiply_result_needs_no_rebalancing(): + r1 = multiply(LiteralStringNode("s"), 2**31 - 2) + assert r1.rebalance() is r1 From arigo at codespeak.net Sun Dec 2 12:44:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:44:50 +0100 (CET) Subject: [pypy-svn] r49269 - in pypy/dist/pypy: module/_socket/test rlib rlib/test Message-ID: <20071202114450.B681D8134@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:44:50 2007 New Revision: 49269 Modified: pypy/dist/pypy/module/_socket/test/test_sock_app.py pypy/dist/pypy/rlib/rsocket.py pypy/dist/pypy/rlib/test/test_rsocket.py Log: issue318 resolved Fix accept() on AF_UNIX sockets. Add tests. Modified: pypy/dist/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/dist/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/dist/pypy/module/_socket/test/test_sock_app.py Sun Dec 2 12:44:50 2007 @@ -1,13 +1,13 @@ from pypy.conftest import gettestobjspace import sys import py +from pypy.tool.udir import udir def setup_module(mod): mod.space = gettestobjspace(usemodules=['_socket']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") - from pypy.tool.udir import udir mod.path = udir.join('fd') mod.path.write('fo') mod.raises = py.test.raises # make raises available from app-level tests @@ -233,6 +233,7 @@ class AppTestSocket: def setup_class(cls): cls.space = space + cls.w_udir = space.wrap(str(udir)) def test_ntoa_exception(self): import _socket @@ -407,7 +408,32 @@ s = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM, 0) s.sendto(buffer(''), ('localhost', 9)) # Send to discard port. s.close() - + + def test_unix_socket_connect(self): + import _socket, os + if not hasattr(_socket, 'AF_UNIX'): + skip('AF_UNIX not supported.') + sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) + + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + s, addr = serversock.accept() + assert not addr + + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' + + clientsock.close() + s.close() + class AppTestSocketTCP: def setup_class(cls): Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Sun Dec 2 12:44:50 2007 @@ -377,7 +377,7 @@ class UNIXAddress(Address): family = AF_UNIX struct = _c.sockaddr_un - minlen = offsetof(_c.sockaddr_un, 'c_sun_path') + 1 + minlen = offsetof(_c.sockaddr_un, 'c_sun_path') maxlen = sizeof(struct) def __init__(self, path): @@ -406,7 +406,7 @@ def get_path(self): a = self.lock(_c.sockaddr_un) maxlength = self.addrlen - offsetof(_c.sockaddr_un, 'c_sun_path') - if _c.linux and a.c_sun_path[0] == '\x00': + if _c.linux and maxlength > 0 and a.c_sun_path[0] == '\x00': # Linux abstract namespace length = maxlength else: Modified: pypy/dist/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket.py Sun Dec 2 12:44:50 2007 @@ -345,6 +345,32 @@ py.test.skip("no inet_ntop()") assert inet_ntop(AF_INET, '\x01\x02\x03\x05') == '1.2.3.5' +def test_unix_socket_connect(): + if getattr(rsocket, 'AF_UNIX', None) is None: + py.test.skip('AF_UNIX not supported.') + from pypy.tool.udir import udir + sockpath = str(udir.join('test_unix_socket_connect')) + a = UNIXAddress(sockpath) + + serversock = RSocket(AF_UNIX) + serversock.bind(a) + serversock.listen(1) + + clientsock = RSocket(AF_UNIX) + clientsock.connect(a) + s, addr = serversock.accept() + + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' + + clientsock.close() + s.close() + + class TestTCP: PORT = 50007 HOST = 'localhost' From arigo at codespeak.net Sun Dec 2 12:59:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:59:57 +0100 (CET) Subject: [pypy-svn] r49271 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071202115957.F049B8167@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:59:57 2007 New Revision: 49271 Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py Log: issue326 in-progress Reduce memory allocations when creating RPython lists. For now there is a choice of either using a prebuilt empty initial array, or always pre-allocating for N elements. Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Sun Dec 2 12:59:57 2007 @@ -71,6 +71,7 @@ ITEMARRAY = GcArray(ITEM, adtmeths = ADTIFixedList({ "ll_newlist": ll_fixed_newlist, + "ll_newemptylist": ll_fixed_newemptylist, "ll_length": ll_fixed_length, "ll_items": ll_fixed_items, ##"list_builder": self.list_builder, @@ -184,6 +185,7 @@ ("items", Ptr(ITEMARRAY)), adtmeths = ADTIList({ "ll_newlist": ll_newlist, + "ll_newemptylist": ll_newemptylist, "ll_length": ll_length, "ll_items": ll_items, ##"list_builder": self.list_builder, @@ -362,6 +364,26 @@ ll_newlist = typeMethod(ll_newlist) ll_newlist.oopspec = 'newlist(length)' +# should empty lists start with no allocated memory, or with a preallocated +# minimal number of entries? XXX compare memory usage versus speed, and +# check how many always-empty lists there are in a typical pypy-c run... +INITIAL_EMPTY_LIST_ALLOCATION = 0 + +def _ll_prebuilt_empty_array(LISTITEM): + return malloc(LISTITEM, 0) +_ll_prebuilt_empty_array._annspecialcase_ = 'specialize:memo' + +def ll_newemptylist(LIST): + l = malloc(LIST) + l.length = 0 + if INITIAL_EMPTY_LIST_ALLOCATION > 0: + l.items = malloc(LIST.items.TO, INITIAL_EMPTY_LIST_ALLOCATION) + else: + l.items = _ll_prebuilt_empty_array(LIST.items.TO) + return l +ll_newemptylist = typeMethod(ll_newemptylist) +ll_newemptylist.oopspec = 'newlist(0)' + def ll_length(l): return l.length @@ -385,6 +407,10 @@ ll_fixed_newlist = typeMethod(ll_fixed_newlist) ll_fixed_newlist.oopspec = 'newlist(length)' +def ll_fixed_newemptylist(LIST): + return ll_fixed_newlist(LIST, 0) +ll_fixed_newemptylist = typeMethod(ll_fixed_newemptylist) + def ll_fixed_length(l): return len(l) @@ -401,8 +427,11 @@ def newlist(llops, r_list, items_v): LIST = r_list.LIST - cno = inputconst(Signed, len(items_v)) - v_result = llops.gendirectcall(LIST.ll_newlist, cno) + if len(items_v) == 0: + v_result = llops.gendirectcall(LIST.ll_newemptylist) + else: + cno = inputconst(Signed, len(items_v)) + v_result = llops.gendirectcall(LIST.ll_newlist, cno) v_func = inputconst(Void, dum_nocheck) for i, v_item in enumerate(items_v): ci = inputconst(Signed, i) From arigo at codespeak.net Sun Dec 2 13:29:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 13:29:06 +0100 (CET) Subject: [pypy-svn] r49272 - in pypy/dist: lib-python/modified-2.4.1 pypy/interpreter/test Message-ID: <20071202122906.5526B80C9@code0.codespeak.net> Author: arigo Date: Sun Dec 2 13:29:04 2007 New Revision: 49272 Added: pypy/dist/lib-python/modified-2.4.1/inspect.py - copied, changed from r49241, pypy/dist/lib-python/2.4.1/inspect.py Modified: pypy/dist/pypy/interpreter/test/test_code.py Log: issue306 in-progress Hacking at inspect.py to let getargs() decode the signature of our internal-code' objects too. Modified: pypy/dist/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_code.py (original) +++ pypy/dist/pypy/interpreter/test/test_code.py Sun Dec 2 13:29:04 2007 @@ -126,3 +126,12 @@ exec "def f(): pass" in d2 assert d1['f'].func_code == d2['f'].func_code assert hash(d1['f'].func_code) == hash(d2['f'].func_code) + + def test_inspect(self): + if not hasattr(len, 'func_code'): + skip("CPython: no func_code attribute on built-in functions") + import inspect + args, varargs, varkw = inspect.getargs(len.func_code) + assert args == ['obj'] + assert varargs is None + assert varkw is None From arigo at codespeak.net Sun Dec 2 13:39:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 13:39:08 +0100 (CET) Subject: [pypy-svn] r49274 - pypy/dist/lib-python/modified-2.4.1 Message-ID: <20071202123908.104CF815C@code0.codespeak.net> Author: arigo Date: Sun Dec 2 13:39:07 2007 New Revision: 49274 Added: pypy/dist/lib-python/modified-2.4.1/pydoc.py - copied, changed from r49241, pypy/dist/lib-python/2.4.1/pydoc.py Modified: pypy/dist/lib-python/modified-2.4.1/inspect.py Log: Fix help(builtin-function) to show the signature on top of PyPy. Modified: pypy/dist/lib-python/modified-2.4.1/inspect.py ============================================================================== --- pypy/dist/lib-python/modified-2.4.1/inspect.py (original) +++ pypy/dist/lib-python/modified-2.4.1/inspect.py Sun Dec 2 13:39:07 2007 @@ -679,7 +679,9 @@ if ismethod(func): func = func.im_func - if not isfunction(func): + if not (isfunction(func) or + isbuiltin(func) and hasattr(func, 'func_code')): + # PyPy extension: this works for built-in functions too raise TypeError('arg is not a Python function') args, varargs, varkw = getargs(func.func_code) return args, varargs, varkw, func.func_defaults From arigo at codespeak.net Sun Dec 2 13:44:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 13:44:24 +0100 (CET) Subject: [pypy-svn] r49276 - pypy/dist/pypy/interpreter Message-ID: <20071202124424.5695F815C@code0.codespeak.net> Author: arigo Date: Sun Dec 2 13:44:24 2007 New Revision: 49276 Modified: pypy/dist/pypy/interpreter/typedef.py Log: Tweak. Modified: pypy/dist/pypy/interpreter/typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/typedef.py (original) +++ pypy/dist/pypy/interpreter/typedef.py Sun Dec 2 13:44:24 2007 @@ -287,7 +287,8 @@ return miniglobals['descr_typecheck_%s' % func.__name__] def unknown_objclass_getter(space): - raise OperationError(space.w_TypeError, + # NB. this is an AttributeError to make inspect.py happy + raise OperationError(space.w_AttributeError, space.wrap("generic property has no __objclass__")) def make_objclass_getter(func, cls, cache={}): From arigo at codespeak.net Sun Dec 2 15:51:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 15:51:34 +0100 (CET) Subject: [pypy-svn] r49280 - pypy/dist/pypy/interpreter Message-ID: <20071202145134.0742B80F9@code0.codespeak.net> Author: arigo Date: Sun Dec 2 15:51:34 2007 New Revision: 49280 Modified: pypy/dist/pypy/interpreter/executioncontext.py Log: ExecutionContext instances should not be seen during translation, but they occasionally do (there is a hard-to-reproduce order dependency). This change might help us locate where they come from. Modified: pypy/dist/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/dist/pypy/interpreter/executioncontext.py (original) +++ pypy/dist/pypy/interpreter/executioncontext.py Sun Dec 2 15:51:34 2007 @@ -258,3 +258,8 @@ def add_pending_action(self, action): self.pending_actions.append(action) self.ticker = 0 + + def _freeze_(self): + raise Exception("ExecutionContext instances should not be seen during" + " translation. Now is a good time to inspect the" + " traceback and see where this one comes from :-)") From arigo at codespeak.net Sun Dec 2 16:53:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 16:53:06 +0100 (CET) Subject: [pypy-svn] r49286 - in pypy/dist/pypy/module: _file marshal marshal/test Message-ID: <20071202155306.F28568138@code0.codespeak.net> Author: arigo Date: Sun Dec 2 16:53:05 2007 New Revision: 49286 Modified: pypy/dist/pypy/module/_file/interp_file.py pypy/dist/pypy/module/marshal/interp_marshal.py pypy/dist/pypy/module/marshal/test/test_marshal.py Log: A performance hack for interp-level code that manipulates app-level file objects. It gives direct access to the underlying stream, shortcutting calls that need to go through app-level (which performs various checks and locking). For marshal, for example, a single lock/unlock pair around the whole operation is enough. Modified: pypy/dist/pypy/module/_file/interp_file.py ============================================================================== --- pypy/dist/pypy/module/_file/interp_file.py (original) +++ pypy/dist/pypy/module/_file/interp_file.py Sun Dec 2 16:53:05 2007 @@ -87,6 +87,36 @@ assert self.slockowner is None return False + def do_read(self, n): + """ + An interface for direct interp-level usage of W_Stream, + e.g. from interp_marshal.py. + NOTE: this assumes that the stream lock is already acquired. + Like os.read(), this can return less than n bytes. + """ + try: + return self.stream.read(n) + except streamio.StreamError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.message)) + except OSError, e: + raise wrap_oserror_as_ioerror(space, e) + + def do_write(self, data): + """ + An interface for direct interp-level usage of W_Stream, + e.g. from interp_marshal.py. + NOTE: this assumes that the stream lock is already acquired. + """ + try: + self.stream.write(data) + except streamio.StreamError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.message)) + except OSError, e: + raise wrap_oserror_as_ioerror(space, e) + + for name, argtypes in streamio.STREAM_METHODS.iteritems(): numargs = len(argtypes) args = ", ".join(["v%s" % i for i in range(numargs)]) @@ -136,3 +166,20 @@ space, streamio.fdopen_as_stream(fd, mode, buffering))) fdopen_as_stream.unwrap_spec = [ObjSpace, int, str, int] + +def file2stream(space, w_f): + """A hack for direct interp-level access to W_Stream objects, + for better performance e.g. when marshalling directly from/to a + real file object. This peels off the app-level layers of the file class + defined in app_file.py. It complains if the file is already closed. + """ + w_stream = space.findattr(w_f, space.wrap('stream')) + if w_stream is None: + return None + w_stream = space.interpclass_w(w_stream) + if not isinstance(w_stream, W_Stream): + return None + if space.is_true(space.getattr(w_f, space.wrap('_closed'))): + raise OperationError(space.w_ValueError, + space.wrap('I/O operation on closed file')) + return w_stream Modified: pypy/dist/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/dist/pypy/module/marshal/interp_marshal.py (original) +++ pypy/dist/pypy/module/marshal/interp_marshal.py Sun Dec 2 16:53:05 2007 @@ -1,6 +1,7 @@ from pypy.interpreter.baseobjspace import ObjSpace from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask +from pypy.module._file.interp_file import file2stream import sys # Py_MARSHAL_VERSION = 2 @@ -14,12 +15,19 @@ def dump(space, w_data, w_f, w_version=Py_MARSHAL_VERSION): """Write the 'data' object into the open file 'f'.""" - writer = FileWriter(space, w_f) - # note: bound methods are currently not supported, - # so we have to pass the instance in, instead. - ##m = Marshaller(space, writer.write, space.int_w(w_version)) - m = Marshaller(space, writer, space.int_w(w_version)) - m.put_w_obj(w_data) + w_stream = file2stream(space, w_f) + if w_stream is not None: + writer = StreamWriter(space, w_stream) + else: + writer = FileWriter(space, w_f) + try: + # note: bound methods are currently not supported, + # so we have to pass the instance in, instead. + ##m = Marshaller(space, writer.write, space.int_w(w_version)) + m = Marshaller(space, writer, space.int_w(w_version)) + m.put_w_obj(w_data) + finally: + writer.finished() def dumps(space, w_data, w_version=Py_MARSHAL_VERSION): """Return the string that would have been written to a file @@ -30,9 +38,17 @@ def load(space, w_f): """Read one value from the file 'f' and return it.""" - reader = FileReader(space, w_f) - u = Unmarshaller(space, reader) - return u.get_w_obj(False) + # special case real files for performance + w_stream = file2stream(space, w_f) + if w_stream is not None: + reader = StreamReader(space, w_stream) + else: + reader = FileReader(space, w_f) + try: + u = Unmarshaller(space, reader) + return u.get_w_obj(False) + finally: + reader.finished() def loads(space, w_str): """Convert a string back to a value. Extra characters in the string are @@ -41,9 +57,22 @@ return u.get_w_obj(False) -class FileWriter(object): - def __init__(self, space, w_f): +class AbstractReaderWriter(object): + def __init__(self, space): self.space = space + + def raise_eof(self): + space = self.space + raise OperationError(space.w_EOFError, space.wrap( + 'EOF read where object expected')) + + def finished(self): + pass + + +class FileWriter(AbstractReaderWriter): + def __init__(self, space, w_f): + AbstractReaderWriter.__init__(self, space) try: self.func = space.getattr(w_f, space.wrap('write')) # XXX how to check if it is callable? @@ -53,19 +82,14 @@ raise OperationError(space.w_TypeError, space.wrap( 'marshal.dump() 2nd arg must be file-like object')) - def raise_eof(self): - space = self.space - raise OperationError(space.w_EOFError, space.wrap( - 'EOF read where object expected')) - def write(self, data): space = self.space space.call_function(self.func, space.wrap(data)) -class FileReader(object): +class FileReader(AbstractReaderWriter): def __init__(self, space, w_f): - self.space = space + AbstractReaderWriter.__init__(self, space) try: self.func = space.getattr(w_f, space.wrap('read')) # XXX how to check if it is callable? @@ -83,10 +107,29 @@ self.raise_eof() return ret - def raise_eof(self): - space = self.space - raise OperationError(space.w_EOFError, space.wrap( - 'EOF read where object expected')) + +class StreamReaderWriter(AbstractReaderWriter): + def __init__(self, space, w_stream): + AbstractReaderWriter.__init__(self, space) + self.w_stream = w_stream + w_stream.descr_lock() + + def finished(self): + self.w_stream.descr_unlock() + +class StreamWriter(StreamReaderWriter): + def write(self, data): + self.w_stream.do_write(data) + +class StreamReader(StreamReaderWriter): + def read(self, n): + result = data = self.w_stream.do_read(n) + while len(result) < n: + if len(data) == 0: + self.raise_eof() + data = self.w_stream.do_read(n) + result += data + return result MAX_MARSHAL_DEPTH = 5000 Modified: pypy/dist/pypy/module/marshal/test/test_marshal.py ============================================================================== --- pypy/dist/pypy/module/marshal/test/test_marshal.py (original) +++ pypy/dist/pypy/module/marshal/test/test_marshal.py Sun Dec 2 16:53:05 2007 @@ -1,5 +1,11 @@ +from pypy.tool.udir import udir + class AppTestMarshal: + def setup_class(cls): + tmpfile = udir.join('AppTestMarshal.tmp') + cls.w_tmpfile = cls.space.wrap(str(tmpfile)) + def test_None(self): import sys hello = "he" @@ -589,6 +595,26 @@ x = marshal.load(f) assert x == case and type(x) is type(case) + def test_stream_reader_writer(self): + # for performance, we have a special case when reading/writing real + # file objects + import marshal + obj1 = [4, ("hello", 7.5)] + obj2 = "foobar" + f = open(self.tmpfile, 'wb') + marshal.dump(obj1, f) + marshal.dump(obj2, f) + f.write('END') + f.close() + f = open(self.tmpfile, 'rb') + obj1b = marshal.load(f) + obj2b = marshal.load(f) + tail = f.read() + f.close() + assert obj1b == obj1 + assert obj2b == obj2 + assert tail == 'END' + class AppTestMultiDict(object): def setup_class(cls): @@ -602,3 +628,4 @@ def setup_class(cls): from pypy.conftest import gettestobjspace cls.space = gettestobjspace(**{"objspace.std.withrope": True}) + AppTestMarshal.setup_class.im_func(cls) From arigo at codespeak.net Sun Dec 2 17:18:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 17:18:15 +0100 (CET) Subject: [pypy-svn] r49287 - pypy/dist/pypy/module/_file Message-ID: <20071202161815.B86BE8171@code0.codespeak.net> Author: arigo Date: Sun Dec 2 17:18:13 2007 New Revision: 49287 Modified: pypy/dist/pypy/module/_file/interp_file.py Log: Typos. Modified: pypy/dist/pypy/module/_file/interp_file.py ============================================================================== --- pypy/dist/pypy/module/_file/interp_file.py (original) +++ pypy/dist/pypy/module/_file/interp_file.py Sun Dec 2 17:18:13 2007 @@ -97,10 +97,10 @@ try: return self.stream.read(n) except streamio.StreamError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.message)) + raise OperationError(self.space.w_ValueError, + self.space.wrap(e.message)) except OSError, e: - raise wrap_oserror_as_ioerror(space, e) + raise wrap_oserror_as_ioerror(self.space, e) def do_write(self, data): """ @@ -111,10 +111,10 @@ try: self.stream.write(data) except streamio.StreamError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.message)) + raise OperationError(self.space.w_ValueError, + self.space.wrap(e.message)) except OSError, e: - raise wrap_oserror_as_ioerror(space, e) + raise wrap_oserror_as_ioerror(self.space, e) for name, argtypes in streamio.STREAM_METHODS.iteritems(): From cfbolz at codespeak.net Sun Dec 2 19:20:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 19:20:23 +0100 (CET) Subject: [pypy-svn] r49292 - pypy/dist/pypy/rlib/test Message-ID: <20071202182023.71BD88140@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 19:20:21 2007 New Revision: 49292 Modified: pypy/dist/pypy/rlib/test/test_rope.py Log: this should really be indented one more level Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sun Dec 2 19:20:21 2007 @@ -400,7 +400,7 @@ for c in result[i + j:]: c2 = iter.nextchar() assert c2 == c - py.test.raises(StopIteration, iter.nextchar) + py.test.raises(StopIteration, iter.nextchar) def test_find_int(): rope, st = make_random_string() From cfbolz at codespeak.net Sun Dec 2 20:59:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 20:59:45 +0100 (CET) Subject: [pypy-svn] r49294 - pypy/dist/pypy/rlib Message-ID: <20071202195945.407EF812C@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 20:59:44 2007 New Revision: 49294 Modified: pypy/dist/pypy/rlib/rope.py Log: off by one error Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 2 20:59:44 2007 @@ -484,7 +484,7 @@ twopower = node number = 1 result = None - while number < times: + while number <= times: if number & times: if result is None: result = twopower From cfbolz at codespeak.net Sun Dec 2 21:02:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 21:02:13 +0100 (CET) Subject: [pypy-svn] r49295 - pypy/dist/pypy/rlib/parsing/test Message-ID: <20071202200213.4B0328150@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 21:02:12 2007 New Revision: 49295 Modified: pypy/dist/pypy/rlib/parsing/test/test_pypackrat.py Log: just for the fun of it: write a future parser with the python packrat parser generator. it's not really any more readable than the one we now use, actually. Modified: pypy/dist/pypy/rlib/parsing/test/test_pypackrat.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_pypackrat.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_pypackrat.py Sun Dec 2 21:02:12 2007 @@ -472,3 +472,80 @@ assert expected == ['a', 'x', 'y'] + def test_python_future(self): + class parser(PackratParser): + r""" + comment: + `#[^\r\n]*` lineend; + lineend: + `(\r|\n)+`; + docstring: + `(\"\"\"[^\\]*(\\[^\\]+)*\"\"\")|(\'\'\'[^\\]*(\\[^\\]+)*\'\'\')` + ignore* + | `(\"[^\\]*(\\[^\\]+)*\")|(\'[^\\]*(\\[^\\]+)*\')` + ignore*; + ignore: + `[ \t]+`; + ignoreline: + `[ \t]*[\r\n]+`; + fromimport: + 'from' ignore+ + '__future__' ignore+ + 'import' ignore+ + what; + identifier: + `[a-zA-Z0-9_]+`; + what: + '(' ignoreline* + g = group + ignoreline* + rest = ([',' ignoreline*] group)* + ')' + return {[g] + rest} + | g = group + rest = ([',' ignore*] group)* + return {[g] + rest}; + group: + name = identifier ignore+ 'as' ignore+ identifier ignore* + return {name} + | name = identifier ignore* + return {name}; + line: + comment + return {None} + | docstring lineend + return {None} + | ignore lineend + return {None} + | t = fromimport + ignore* + lineend + return {t}; + header: + l = line* + return {[elt for sublist in l if sublist is not None for elt in sublist]}; + """ + p = parser("#\n") + lines = p.header() + assert lines == [] + p = parser('''"abc"\n''') + lines = p.header() + assert lines == [] + p = parser(''''abc'\n''') + lines = p.header() + assert lines == [] + p = parser(''''abc'\n''') + lines = p.header() + assert lines == [] + p = parser('''from __future__ import division\n''') + lines = p.header() + assert lines == ['division'] + p = parser('''from __future__ import division, generators\n''') + lines = p.fromimport() + assert lines == ['division', 'generators'] + p = parser('''from __future__ import (division, \ngenerators)\n''') + lines = p.fromimport() + assert lines == ['division', 'generators'] + p = parser('''from __future__ import (division as d, \ngenerators)\n''') + lines = p.fromimport() + assert lines == ['division', 'generators'] From cfbolz at codespeak.net Sun Dec 2 21:05:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 21:05:09 +0100 (CET) Subject: [pypy-svn] r49298 - in pypy/dist/pypy/rlib: . test Message-ID: <20071202200509.1F3FC8169@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 21:05:08 2007 New Revision: 49298 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: completely rewrite the seekable rope iterator to be conceptually more right: seeking now is O(log(distance)), not O(distance). Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 2 21:05:08 2007 @@ -970,48 +970,64 @@ class SeekableItemIterator(object): def __init__(self, node): - self.iter = SeekableFringeIterator(node) - self.node = self.nextnode() - self.nodelength = self.node.length() - self.index = 0 + self.stack = [] + self.tookleft = [] + self.find_downward(node) + assert False not in self.tookleft + + def find_downward(self, node, items=0): + assert 0 <= items < node.length() + while isinstance(node, BinaryConcatNode): + self.stack.append(node) + right = node.right + left = node.left + if items >= left.length(): + items -= left.length() + node = node.right + self.tookleft.append(False) + else: + node = node.left + self.tookleft.append(True) + assert len(self.tookleft) == len(self.stack) + self.node = node + self.nodelength = node.length() + self.index = items + return self.node def nextnode(self): - while 1: - node = self.node = self.iter.next() - nodelength = self.nodelength = node.length() - if nodelength != 0: - break - self.index = 0 - return node - + below = self.node + while self.stack: + tookleft = self.tookleft.pop() + if tookleft: + node = self.stack[-1] + assert isinstance(node, BinaryConcatNode) + self.tookleft.append(False) + self.find_downward(node.right) + return self.node + self.stack.pop() + raise StopIteration + + def getnode(self): + if self.index == self.node.length(): + return self.nextnode() + return self.node - def advance_index(self): - if self.index == self.nodelength - 1: - self.node = None - self.index += 1 - def nextchar(self): - node = self.node - if node is None: - node = self.nextnode() - result = self.node.getchar(self.index) - self.advance_index() + node = self.getnode() + result = node.getchar(self.index) + self.index += 1 return result def nextunichar(self): - node = self.node - if node is None: - node = self.nextnode() - result = self.node.getunichar(self.index) - self.advance_index() + node = self.getnode() + result = node.getunichar(self.index) + self.index += 1 return result def nextint(self): - node = self.node - if node is None: - node = self.nextnode() - result = self.node.getint(self.index) - self.advance_index() + node = self.getnode() + result = node.getint(self.index) + self.index += 1 return result def seekforward(self, numchars): @@ -1019,36 +1035,42 @@ self.index += numchars return numchars -= self.nodelength - self.index - while 1: - node = self.iter.next() - length = node.length() - if length <= numchars: - numchars -= length - else: - self.index = numchars - self.node = node - self.nodelength = node.length() - return + while self.stack: + tookleft = self.tookleft.pop() + if tookleft: + node = self.stack[-1] + assert isinstance(node, BinaryConcatNode) + right = node.right + if right.length() > numchars: + break + numchars -= right.length() + self.stack.pop() + else: + raise StopIteration + self.tookleft.append(False) + self.find_downward(right, numchars) + def seekback(self, numchars): if numchars <= self.index: self.index -= numchars - if self.node is None: - self.iter.seekback() - self.node = self.iter.next() return numchars -= self.index - self.iter.seekback() # for first item - while 1: - node = self.iter.seekback() - length = node.length() - if length < numchars: - numchars -= length - else: - self.index = length - numchars - self.node = self.iter.next() - self.nodelength = self.node.length() - return + while self.stack: + tookleft = self.tookleft.pop() + if not tookleft: + node = self.stack[-1] + assert isinstance(node, BinaryConcatNode) + left = node.left + if left.length() >= numchars: + break + numchars -= left.length() + self.stack.pop() + else: + raise StopIteration + self.tookleft.append(True) + self.find_downward(left, left.length() - numchars) + class FindIterator(object): def __init__(self, node, sub, start=0, stop=-1): Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sun Dec 2 21:05:08 2007 @@ -391,8 +391,6 @@ for j in range(len(result) - 1): for i in range(len(result) - 1 - j): iter = SeekableItemIterator(rope) -# if (j, i) == (3, 1): -# import pdb; pdb.set_trace() for c in result[:j]: c2 = iter.nextchar() assert c2 == c From cfbolz at codespeak.net Sun Dec 2 21:22:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 21:22:23 +0100 (CET) Subject: [pypy-svn] r49300 - pypy/dist/pypy/rlib Message-ID: <20071202202223.D99AC8187@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 21:22:21 2007 New Revision: 49300 Modified: pypy/dist/pypy/rlib/rope.py Log: small beautifications Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 2 21:22:21 2007 @@ -1042,13 +1042,12 @@ assert isinstance(node, BinaryConcatNode) right = node.right if right.length() > numchars: - break + self.tookleft.append(False) + self.find_downward(right, numchars) + return numchars -= right.length() self.stack.pop() - else: - raise StopIteration - self.tookleft.append(False) - self.find_downward(right, numchars) + raise StopIteration def seekback(self, numchars): @@ -1063,13 +1062,12 @@ assert isinstance(node, BinaryConcatNode) left = node.left if left.length() >= numchars: - break + self.tookleft.append(True) + self.find_downward(left, left.length() - numchars) + return numchars -= left.length() self.stack.pop() - else: - raise StopIteration - self.tookleft.append(True) - self.find_downward(left, left.length() - numchars) + raise StopIteration class FindIterator(object): From cfbolz at codespeak.net Sun Dec 2 21:37:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 21:37:50 +0100 (CET) Subject: [pypy-svn] r49303 - pypy/dist/pypy/translator/microbench Message-ID: <20071202203750.8AF48817D@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 21:37:50 2007 New Revision: 49303 Added: pypy/dist/pypy/translator/microbench/test_unicode.py (contents, props changed) Log: a very simple unicode finding benchmark Added: pypy/dist/pypy/translator/microbench/test_unicode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/microbench/test_unicode.py Sun Dec 2 21:37:50 2007 @@ -0,0 +1,5 @@ +N = (2 ** 19 - 1) + +u1 = (u"not the xyz" * N) +def test_find_worstcase(): + u1.find(u"not there") From cfbolz at codespeak.net Mon Dec 3 00:09:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 00:09:49 +0100 (CET) Subject: [pypy-svn] r49305 - in pypy/dist/pypy/rlib: . test Message-ID: <20071202230949.B0EA3817D@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 00:09:49 2007 New Revision: 49305 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: don't search when the pattern string is longer than the search string Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 3 00:09:49 2007 @@ -698,6 +698,8 @@ if (stop - start) < 0: return -1 return start + if len2 >= stop - start: + return -1 restart = construct_restart_positions_node(subnode) return _find_node(node, subnode, start, stop, restart) Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Mon Dec 3 00:09:49 2007 @@ -437,6 +437,8 @@ LiteralStringNode("bacbb")) pos = find(node, LiteralStringNode("a"), 0, node.length()) assert pos == 6 + pos = find(node, LiteralStringNode("aaa"), 0, 2) + assert pos == -1 def test_find_unicode(): From cfbolz at codespeak.net Mon Dec 3 00:18:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 00:18:44 +0100 (CET) Subject: [pypy-svn] r49306 - in pypy/dist/pypy/rlib: . test Message-ID: <20071202231844.335D18188@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 00:18:42 2007 New Revision: 49306 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: an off-by-one error. also optimize the FindIterator for the case where the pattern is longer than the bit of the text string where I am searching in. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 3 00:18:42 2007 @@ -698,7 +698,7 @@ if (stop - start) < 0: return -1 return start - if len2 >= stop - start: + if len2 > stop - start: return -1 restart = construct_restart_positions_node(subnode) return _find_node(node, subnode, start, stop, restart) @@ -1079,16 +1079,20 @@ len1 = self.length = node.length() len2 = sub.length() self.search_length = len2 + self.start = start + if stop == -1 or stop > len1: + stop = len1 + self.stop = stop if len2 == 0: self.restart_positions = None elif len2 == 1: self.restart_positions = None + elif len2 > stop - start: + self.restart_positions = None + # ensure that a StopIteration is immediately raised + self.stop = self.start else: self.restart_positions = construct_restart_positions_node(sub) - self.start = start - if stop == -1 or stop > len1: - stop = len1 - self.stop = stop def next(self): if self.search_length == 0: Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Mon Dec 3 00:18:42 2007 @@ -439,6 +439,8 @@ assert pos == 6 pos = find(node, LiteralStringNode("aaa"), 0, 2) assert pos == -1 + pos = find(node, LiteralStringNode("btf"), 0, 3) + assert pos == 0 def test_find_unicode(): From niko at codespeak.net Mon Dec 3 10:33:11 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Mon, 3 Dec 2007 10:33:11 +0100 (CET) Subject: [pypy-svn] r49308 - in pypy/dist/pypy/translator/jvm: src/pypy test Message-ID: <20071203093311.F2DEC8195@code0.codespeak.net> Author: niko Date: Mon Dec 3 10:33:10 2007 New Revision: 49308 Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java pypy/dist/pypy/translator/jvm/test/test_builtin.py Log: implement unlink, mkdir, frexp, and add a test or two Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Mon Dec 3 10:33:10 2007 @@ -978,11 +978,48 @@ return x % y; } - /* TODO - public double ll_math_frexp(double x) { - } - */ + public Object ll_math_frexp(double x) { + /* + Return the mantissa and exponent of x as the pair (m, e). m + is a float and e is an integer such that x == m * 2**e + exactly. If x is zero, returns (0.0, 0), otherwise 0.5 <= + abs(m) < 1. This is used to "pick apart" the internal + representation of a float in a portable way. + */ + + // NaN: Python returns (NaN, 0) + if (Double.isNaN(x)) + return interlink.recordFloatSigned(x, 0); + + // Infinity: Python throws exception + if (Double.isInfinite(x)) + interlink.throwOverflowError(); + // Extract the various parts of the format: + final long e=11, f=52; // number of bits in IEEE format + long bits = Double.doubleToLongBits(x); + long bits_mantissa = bits & ((1 << f) - 1); + int bits_exponent = (int)((bits >> f) & ((1 << e) - 1)); + int bits_sign = (int)(bits >> (e+f)); + + // [+-]0 + if (bits_exponent == 0 && bits_mantissa == 0) + return interlink.recordFloatSigned(x, 0); + + // TODO: Non-looping impl + double mantissa = x; + int exponent = 0; + while (mantissa > 1) { + mantissa /= 2; + exponent += 1; + } + while (mantissa < 0.5) { + mantissa *= 2; + exponent -= 1; + } + return interlink.recordFloatSigned(mantissa, exponent); + } + public double ll_math_ldexp(double v, int w) { return check(v * Math.pow(2.0, w)); } @@ -996,8 +1033,8 @@ return Math.exp(x); } - public double ll_math_log(double x, double base) { - return Math.log10(x) / Math.log10(base); + public double ll_math_log(double x) { + return Math.log(x); } public double ll_math_log10(double v) { Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Mon Dec 3 10:33:10 2007 @@ -357,6 +357,29 @@ return text.length(); } + public void ll_os_mkdir(String path, int mode) { + File f = new File(path); + if (f.exists()) + throwOSError(PyPy.EEXIST, "File exists: '"+path+"'"); + if (!f.mkdir()) + throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); + } + + public void ll_os_unlink(String path) { + if (STRACE) strace("ll_os_unlink: "+path); + + File f = new File(path); + + if (!f.exists()) + throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); + + if (f.isDirectory()) + throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); + + if (!f.delete()) + throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); + } + public boolean ll_os_isatty(int x) { // XXX: this is not the right behaviour, but it's needed Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Mon Dec 3 10:33:10 2007 @@ -14,12 +14,6 @@ skip_win() BaseTestBuiltin.test_os_write_magic(self) - def test_builtin_math_frexp(self): - py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") - - def test_builtin_math_modf(self): - py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") - def test_os_path_exists(self): py.test.skip("fails in annotation stage, unrelated to JVM I think") From niko at codespeak.net Mon Dec 3 10:41:32 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Mon, 3 Dec 2007 10:41:32 +0100 (CET) Subject: [pypy-svn] r49309 - in pypy/dist/pypy: rpython/test translator/jvm/src/pypy translator/oosupport/test_template Message-ID: <20071203094132.89F848194@code0.codespeak.net> Author: niko Date: Mon Dec 3 10:41:32 2007 New Revision: 49309 Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/oosupport/test_template/builtin.py Log: fix a bug in our frexp for negative numbers, and expand the rpython test to include a wider range of testing values Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Mon Dec 3 10:41:32 2007 @@ -157,9 +157,12 @@ import math def fn(f): return math.frexp(f) - res = self.interpret(fn, [10/3.0]) - mantissa, exponent = math.frexp(10/3.0) - assert self.float_eq(res.item0, mantissa) and self.float_eq(res.item1, exponent) + for x in (.5, 1, 1.5, 10/3.0): + for y in (1, -1): + res = self.interpret(fn, [x*y]) + mantissa, exponent = math.frexp(x*y) + assert (self.float_eq(res.item0, mantissa) and + self.float_eq(res.item1, exponent)) def test_builtin_math_ldexp(self): import math @@ -289,8 +292,8 @@ return os.path.exists(fn) filename = self.string_to_ll(str(py.magic.autopath())) assert self.interpret(f, [filename]) == True - assert self.interpret(f, [ - self.string_to_ll("strange_filename_that_looks_improbable.sde")]) == False + #assert self.interpret(f, [ + # self.string_to_ll("strange_filename_that_looks_improbable.sde")]) == False def test_os_isdir(self): self._skip_llinterpreter("os.stat()") Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Mon Dec 3 10:41:32 2007 @@ -1007,9 +1007,9 @@ return interlink.recordFloatSigned(x, 0); // TODO: Non-looping impl - double mantissa = x; + double mantissa = Math.abs(x); int exponent = 0; - while (mantissa > 1) { + while (mantissa >= 1.0) { mantissa /= 2; exponent += 1; } @@ -1017,6 +1017,7 @@ mantissa *= 2; exponent -= 1; } + mantissa = (x < 0 ? -mantissa : mantissa); return interlink.recordFloatSigned(mantissa, exponent); } Modified: pypy/dist/pypy/translator/oosupport/test_template/builtin.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/builtin.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/builtin.py Mon Dec 3 10:41:32 2007 @@ -148,7 +148,6 @@ assert act_res.item0 == exp_res[0] assert act_res.item1 == exp_res[1] - class BaseTestTime(llBaseTestTime): def test_time_clock(self): From fijal at codespeak.net Mon Dec 3 17:36:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 17:36:42 +0100 (CET) Subject: [pypy-svn] r49316 - pypy/dist/pypy/lib/ctypes Message-ID: <20071203163642.19AB78155@code0.codespeak.net> Author: fijal Date: Mon Dec 3 17:36:40 2007 New Revision: 49316 Modified: pypy/dist/pypy/lib/ctypes/ (props changed) Log: fixeol From fijal at codespeak.net Mon Dec 3 17:38:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 17:38:54 +0100 (CET) Subject: [pypy-svn] r49317 - in pypy/dist/pypy/lib: app_test test2 Message-ID: <20071203163854.698DC8155@code0.codespeak.net> Author: fijal Date: Mon Dec 3 17:38:54 2007 New Revision: 49317 Removed: pypy/dist/pypy/lib/test2/test_binascii.py Modified: pypy/dist/pypy/lib/app_test/test_binascii.py Log: Move test away from using AppTest Modified: pypy/dist/pypy/lib/app_test/test_binascii.py ============================================================================== --- pypy/dist/pypy/lib/app_test/test_binascii.py (original) +++ pypy/dist/pypy/lib/app_test/test_binascii.py Mon Dec 3 17:38:54 2007 @@ -146,3 +146,7 @@ f = getattr(binascii, n) f('') binascii.crc_hqx('', 0) + +def test_incorrect_padding(): + import binascii + raises(binascii.Error, "'x'.decode('base64')") From cfbolz at codespeak.net Mon Dec 3 17:43:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 17:43:15 +0100 (CET) Subject: [pypy-svn] r49318 - pypy/dist/pypy/translator/microbench Message-ID: <20071203164315.71D158155@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 17:43:14 2007 New Revision: 49318 Modified: pypy/dist/pypy/translator/microbench/test_unicode.py Log: some more find microbenches. python2.4 is marginally faster than pypy here and python2.5 is tons faster. we should steal their algorithm: http://effbot.org/zone/stringlib.htm Modified: pypy/dist/pypy/translator/microbench/test_unicode.py ============================================================================== --- pypy/dist/pypy/translator/microbench/test_unicode.py (original) +++ pypy/dist/pypy/translator/microbench/test_unicode.py Mon Dec 3 17:43:14 2007 @@ -3,3 +3,38 @@ u1 = (u"not the xyz" * N) def test_find_worstcase(): u1.find(u"not there") + +def test_count_worstcase(): + u1.count(u"not there") + +u2 = (u"aaa" * 1000) +def test_find_pattern16(): + i = 1 + while i < N: + i += 1 + u2.find(u"bbbbbbbbbbbbbbbb") + +def test_find_pattern8(): + i = 1 + while i < N: + i += 1 + u2.find(u"bbbbbbbb") + +def test_find_pattern4(): + i = 1 + while i < N: + i += 1 + u2.find(u"bbbb") + +def test_find_pattern2(): + i = 1 + while i < N: + i += 1 + u2.find(u"bb") + + +def test_find_pattern1(): + i = 1 + while i < N: + i += 1 + u2.find(u"b") From fijal at codespeak.net Mon Dec 3 17:45:00 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 17:45:00 +0100 (CET) Subject: [pypy-svn] r49319 - in pypy/dist/pypy/lib: app_test test2 Message-ID: <20071203164500.51DF68155@code0.codespeak.net> Author: fijal Date: Mon Dec 3 17:44:59 2007 New Revision: 49319 Added: pypy/dist/pypy/lib/test2/test_binascii.py - copied unchanged from r49316, pypy/dist/pypy/lib/test2/test_binascii.py Modified: pypy/dist/pypy/lib/app_test/test_binascii.py Log: Revert last checkin, was bogus Modified: pypy/dist/pypy/lib/app_test/test_binascii.py ============================================================================== --- pypy/dist/pypy/lib/app_test/test_binascii.py (original) +++ pypy/dist/pypy/lib/app_test/test_binascii.py Mon Dec 3 17:44:59 2007 @@ -146,7 +146,3 @@ f = getattr(binascii, n) f('') binascii.crc_hqx('', 0) - -def test_incorrect_padding(): - import binascii - raises(binascii.Error, "'x'.decode('base64')") From cfbolz at codespeak.net Mon Dec 3 17:55:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 17:55:01 +0100 (CET) Subject: [pypy-svn] r49320 - pypy/dist/pypy/translator/microbench Message-ID: <20071203165501.6F810817D@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 17:55:01 2007 New Revision: 49320 Modified: pypy/dist/pypy/translator/microbench/test_unicode.py Log: a find microbench where pypy is both faster than cpy 2.4 and 2.5. I think I am hitting the case here where our algorithm is O(len(text)) and theirs is O(len(text) * len(pattern)) Modified: pypy/dist/pypy/translator/microbench/test_unicode.py ============================================================================== --- pypy/dist/pypy/translator/microbench/test_unicode.py (original) +++ pypy/dist/pypy/translator/microbench/test_unicode.py Mon Dec 3 17:55:01 2007 @@ -38,3 +38,11 @@ while i < N: i += 1 u2.find(u"b") + +u3 = u"baaaaaaaaaaaaaaa" * 1000 +def test_bad_case_python2_5(): + p = u"a" * 16 + u"b" + u"a" * 16 + i = 0 + while i < 10000: + i += 1 + u3.find(p) From fijal at codespeak.net Mon Dec 3 18:00:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 18:00:54 +0100 (CET) Subject: [pypy-svn] r49321 - pypy/dist/pypy/translator/llvm Message-ID: <20071203170054.D23328169@code0.codespeak.net> Author: fijal Date: Mon Dec 3 18:00:54 2007 New Revision: 49321 Modified: pypy/dist/pypy/translator/llvm/opwriter.py Log: Add ullong_is_true operation Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Mon Dec 3 18:00:54 2007 @@ -282,6 +282,7 @@ opr.argrefs[0], "0") uint_is_true = int_is_true llong_is_true = int_is_true + ullong_is_true = int_is_true def float_is_true(self, opr): self.codewriter.binaryop("fcmp une", opr.retref, opr.argtypes[0], From fijal at codespeak.net Mon Dec 3 20:01:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 20:01:11 +0100 (CET) Subject: [pypy-svn] r49322 - pypy/dist/pypy/objspace/std Message-ID: <20071203190111.100BD8164@code0.codespeak.net> Author: fijal Date: Mon Dec 3 20:01:09 2007 New Revision: 49322 Modified: pypy/dist/pypy/objspace/std/formatting.py Log: A cheap way to cut time a bit. This really needs another approach though. Modified: pypy/dist/pypy/objspace/std/formatting.py ============================================================================== --- pypy/dist/pypy/objspace/std/formatting.py (original) +++ pypy/dist/pypy/objspace/std/formatting.py Mon Dec 3 20:01:09 2007 @@ -317,16 +317,21 @@ def std_wp(self, r): length = len(r) prec = self.prec + if prec == -1 and self.width == 0: + # fast path + self.result.append(const(r)) + return if prec >= 0 and prec < length: length = prec # ignore the end of the string if too long result = self.result padding = self.width - length - if not self.f_ljust: + if not self.f_ljust and padding > 0: result.append(const(' ' * padding)) # add any padding at the left of 'r' padding = 0 result.append(const(r[:length])) # add 'r' itself - result.append(const(' ' * padding)) + if padding > 0: + result.append(const(' ' * padding)) # add any remaining padding at the right std_wp._annspecialcase_ = 'specialize:argtype(1)' From cfbolz at codespeak.net Mon Dec 3 21:26:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 21:26:33 +0100 (CET) Subject: [pypy-svn] r49323 - pypy/dist/pypy/rlib Message-ID: <20071203202633.1096B8158@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 21:26:31 2007 New Revision: 49323 Modified: pypy/dist/pypy/rlib/rope.py Log: wtf? Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 3 21:26:31 2007 @@ -167,8 +167,6 @@ if what >= 256: return -1 result = self.s.find(chr(what), start, stop) - if result == -1: - return -1 return result def literal_concat(self, other): @@ -244,8 +242,6 @@ def find_int(self, what, start, stop): result = self.u.find(unichr(what), start, stop) - if result == -1: - return -1 return result def literal_concat(self, other): From cfbolz at codespeak.net Mon Dec 3 21:43:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 21:43:23 +0100 (CET) Subject: [pypy-svn] r49325 - pypy/dist/pypy/doc/discussion Message-ID: <20071203204323.AAA718142@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 21:43:23 2007 New Revision: 49325 Added: pypy/dist/pypy/doc/discussion/parsing-ideas.txt (contents, props changed) Log: add a discussion file with feature requests for the parsing stuff Added: pypy/dist/pypy/doc/discussion/parsing-ideas.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/discussion/parsing-ideas.txt Mon Dec 3 21:43:23 2007 @@ -0,0 +1,5 @@ +add a way to modularize regular expressions: + +_HEXNUM = "..."; +_DECNUM = "..."; +NUM = "{_HEXNUM}|{_DECNUM}"; From cfbolz at codespeak.net Tue Dec 4 00:01:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 00:01:00 +0100 (CET) Subject: [pypy-svn] r49328 - pypy/dist/pypy/rlib Message-ID: <20071203230100.1A6C18139@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 00:00:59 2007 New Revision: 49328 Modified: pypy/dist/pypy/rlib/rope.py Log: add an XXX Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Tue Dec 4 00:00:59 2007 @@ -864,6 +864,7 @@ self._advance_to(start) def _advance_to(self, index): + # XXX this is O(index), should be O(log(index)) assert index > 0 assert self.index == 0 while 1: From niko at codespeak.net Tue Dec 4 10:07:35 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Tue, 4 Dec 2007 10:07:35 +0100 (CET) Subject: [pypy-svn] r49329 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20071204090735.4B69C819F@code0.codespeak.net> Author: niko Date: Tue Dec 4 10:07:34 2007 New Revision: 49329 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Log: implement getpid, symlink, using jna.jar. Right now if jna.jar is not found these operations fail with EPERM and the message "jna.jar required". Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Tue Dec 4 10:07:34 2007 @@ -8,6 +8,9 @@ import java.util.Iterator; import java.util.Arrays; +import com.sun.jna.Library; +import com.sun.jna.Native; + abstract class FileWrapper { public abstract void write(String buffer); @@ -160,6 +163,25 @@ public class ll_os implements Constants { + /** + * JNA Interface: allows access to functions we don't normally + * have in the Java standard lib + */ + static public interface Libc extends Library { + public int getpid(); + public int symlink(String path1, String path2); + } + static final Libc libc; + static { + Libc res; + try { + res = (Libc) Native.loadLibrary("c", Libc.class); + } catch (Throwable t) { + res = null; + } + libc = res; + } + // NB: these values are those used by Windows and they differs // from the Unix ones; the os module is patched with these // values before flowgraphing to make sure we get the very @@ -365,21 +387,26 @@ throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); } - public void ll_os_unlink(String path) { - if (STRACE) strace("ll_os_unlink: "+path); - + public void delete(String path, boolean should_be_dir) { File f = new File(path); - if (!f.exists()) throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); - - if (f.isDirectory()) + if (f.isDirectory() != should_be_dir) throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); - if (!f.delete()) throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); } + public void ll_os_rmdir(String path) { + if (STRACE) strace("ll_os_rmdir: "+path); + delete(path, true); + } + + public void ll_os_unlink(String path) { + if (STRACE) strace("ll_os_unlink: "+path); + delete(path, false); + } + public boolean ll_os_isatty(int x) { // XXX: this is not the right behaviour, but it's needed @@ -464,4 +491,23 @@ throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); return null; // never reached } + + public void checkLibc() { + if (libc == null) + throwOSError(EPERM, "jna.jar required"); + } + + public int ll_os_getpid() + { + checkLibc(); + return libc.getpid(); + } + + public void ll_os_symlink(String path1, String path2) + { + checkLibc(); + int res = libc.symlink(path1, path2); + if (res != 0) + throwOSError(res, ""); + } } From antocuni at codespeak.net Tue Dec 4 13:50:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 4 Dec 2007 13:50:00 +0100 (CET) Subject: [pypy-svn] r49331 - pypy/dist/pypy/translator Message-ID: <20071204125000.81020817D@code0.codespeak.net> Author: antocuni Date: Tue Dec 4 13:49:59 2007 New Revision: 49331 Modified: pypy/dist/pypy/translator/driver.py Log: uname -o doesn't work on OS/X Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Tue Dec 4 13:49:59 2007 @@ -664,7 +664,7 @@ f = file(newexename, 'w') f.write("""#!/bin/bash LEDIT=`type -p ledit` -if [ `uname -o` = 'Cygwin' ]; then MONO=; else MONO=mono; fi +if [ `uname -s` = 'Cygwin' ]; then MONO=; else MONO=mono; fi $LEDIT $MONO "$(dirname $0)/$(basename $0)-data/%s" "$@" # XXX doesn't work if it's placed in PATH """ % main_exe_name) f.close() From xoraxax at codespeak.net Tue Dec 4 14:08:28 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 4 Dec 2007 14:08:28 +0100 (CET) Subject: [pypy-svn] r49332 - in pypy/dist/pypy: rpython/test translator/cli/test translator/llvm/test Message-ID: <20071204130828.4023F8158@code0.codespeak.net> Author: xoraxax Date: Tue Dec 4 14:08:27 2007 New Revision: 49332 Modified: pypy/dist/pypy/rpython/test/tool.py pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: Reduce pointless code duplication, fix frexp tests on cli and llvm. Modified: pypy/dist/pypy/rpython/test/tool.py ============================================================================== --- pypy/dist/pypy/rpython/test/tool.py (original) +++ pypy/dist/pypy/rpython/test/tool.py Tue Dec 4 14:08:27 2007 @@ -3,6 +3,8 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.test.test_llinterp import gengraph, interpret, interpret_raises +FLOAT_PRECISION = 8 + class BaseRtypingTest(object): def gengraph(self, func, argtypes=[], viewbefore='auto', policy=None, @@ -19,6 +21,10 @@ def float_eq(self, x, y): return x == y + def float_eq_approx(self, x, y): + diff = abs(x-y) + return diff < 10**-FLOAT_PRECISION + def is_of_type(self, x, type_): return type(x) is type_ Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Tue Dec 4 14:08:27 2007 @@ -23,7 +23,6 @@ from pypy.translator.cli.entrypoint import BaseEntryPoint from pypy.translator.oosupport.support import patch_os, unpatch_os -FLOAT_PRECISION = 8 def format_object(TYPE, cts, ilasm): if TYPE is ootype.Void: @@ -292,9 +291,7 @@ else: assert False, 'function did raise no exception at all' - def float_eq(self, x, y): - diff = abs(x-y) - return diff/x < 10**-FLOAT_PRECISION + float_eq = BaseRtypingTest.float_eq_approx def is_of_type(self, x, type_): return True # we can't really test the type Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Tue Dec 4 14:08:27 2007 @@ -10,7 +10,6 @@ optimize_tests = False native_llvm_backend = True MINIMUM_LLVM_VERSION = 2.0 -FLOAT_PRECISION = 8 # prevents resource leaking use_isolate = True @@ -219,10 +218,8 @@ return True else: assert False, 'function did raise no exception at all' - - def float_eq(self, x, y): - diff = abs(x-y) - return diff/x < 10**-FLOAT_PRECISION + + float_eq = BaseRtypingTest.float_eq_approx def is_of_type(self, x, type_): return True # we can't really test the type From cfbolz at codespeak.net Tue Dec 4 14:16:31 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 14:16:31 +0100 (CET) Subject: [pypy-svn] r49333 - pypy/dist/pypy/doc Message-ID: <20071204131631.2740A816A@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 14:16:31 2007 New Revision: 49333 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: some of these were done Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Tue Dec 4 14:16:31 2007 @@ -20,10 +20,6 @@ https://codespeak.net/issue/pypy-dev/issue303 and the fact that we can have more than one translator/annotator around (with the timeshifter) - - unicode strings in RPython - - finish rctypes removal - - think about approaches to id, especially concerning boehm, where the id will - keep the object alive and concerning a moving GC interpreter ----------- From cfbolz at codespeak.net Tue Dec 4 14:17:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 14:17:47 +0100 (CET) Subject: [pypy-svn] r49335 - pypy/dist/pypy/doc Message-ID: <20071204131747.9AB208185@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 14:17:46 2007 New Revision: 49335 Removed: pypy/dist/pypy/doc/rctypes.txt Modified: pypy/dist/pypy/doc/standalone-howto.txt Log: remove rctypes docs Modified: pypy/dist/pypy/doc/standalone-howto.txt ============================================================================== --- pypy/dist/pypy/doc/standalone-howto.txt (original) +++ pypy/dist/pypy/doc/standalone-howto.txt Tue Dec 4 14:17:46 2007 @@ -9,6 +9,8 @@ way to compile it is not as described below, but by writing a target file as described in the `FAQ entries`_.) +**Warning:** the bits of this howto that describe rctypes are outdated + ======================================== First, see the note above. @@ -99,7 +101,7 @@ Now we can sum with different kinds of lists, eg. ``sum([1,2,3])`` and ``sum([1.0,2.0,3.0])``. -Here is an example of using rctypes_:: +Here is an example of using rctypes:: import ctypes from ctypes import c_int, c_char_p From fijal at codespeak.net Tue Dec 4 14:26:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 4 Dec 2007 14:26:41 +0100 (CET) Subject: [pypy-svn] r49336 - pypy/dist/pypy/doc Message-ID: <20071204132641.AEB8D8185@code0.codespeak.net> Author: fijal Date: Tue Dec 4 14:26:40 2007 New Revision: 49336 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: This is all done Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Tue Dec 4 14:26:40 2007 @@ -27,13 +27,4 @@ - review the things implemented at applevel whether they are performance- critical - - rewrite the following rctypes modules using rffi: - - - _ssl - - - move the following from using old-style C-api - - - signal module - - strtod (RPyton level) - - review CPython regression test suite, enable running tests, fix bugs From regmee at codespeak.net Tue Dec 4 15:40:48 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Tue, 4 Dec 2007 15:40:48 +0100 (CET) Subject: [pypy-svn] r49338 - in pypy/branch/clr-module-improvements/pypy/module/clr: . test Message-ID: <20071204144048.99D308158@code0.codespeak.net> Author: regmee Date: Tue Dec 4 15:40:47 2007 New Revision: 49338 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: checking in failing test case for GetEnumerator when assigned to __iter__ Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py Tue Dec 4 15:40:47 2007 @@ -86,27 +86,31 @@ else: type.__setattr__(cls, name, value) +#class Dummy(object): +# def __init__(self, iterObj): +# self.iterObj = iterObj +# self.index = 0 +# +# def next(self): +# temp = self.index +# if self.index == self.iterObj.Count: +# raise StopIteration +# self.index = self.index + 1 +# return self.iterObj.__getitem__(temp) + + class CliClassWrapper(object): __slots__ = ('__cliobj__',) def __init__(self, *args): import clr self.__cliobj__ = clr._CliObject_internal(self.__cliclass__, args) -# self.index = self.__cliobj__.__len__(self) -# self.index = self.__cliobj__.call_method('Count',1) print self.__cliobj__ -# self.index = self.Count - - def __iter__(self): - self.index = self.Count - return self - - def next(self): - if self.index == 0: - raise StopIteration - self.index = self.index - 1 - return self.this[self.index] + print self.Count +# def __iter__(self): +# return Dummy(self) +# return Dummy(self.Count, self.__getitem__(self.Count - 1)) def build_wrapper(namespace, classname, staticmethods, methods, properties, indexers): fullname = '%s.%s' % (namespace, classname) @@ -122,7 +126,9 @@ if method == "GetEnumerator": print "Enumerator found .. Hurray !!!!!" # now add the __iter__ method to the class -# d['__iter__'] = sampleIter().__iter__ + d['__iter__'] = d['GetEnumerator'] +# d['next'] = d['MoveNext'] + assert len(indexers) <= 1 if indexers: @@ -153,3 +159,8 @@ setattr(cls, name, prop) return cls + + + + + Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Tue Dec 4 15:40:47 2007 @@ -148,3 +148,17 @@ assert Environment.CurrentDirectory == os.getcwd() Environment.CurrentDirectory == '/' assert Environment.CurrentDirectory == os.getcwd() + + def test_GetEnumerator(self): + import clr + ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') + x = ArrayList() + x.Add(1) + x.Add(6) + x.Add(31) + x.Add(2) + for i in x: + print i + + + From exarkun at codespeak.net Tue Dec 4 16:26:58 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Tue, 4 Dec 2007 16:26:58 +0100 (CET) Subject: [pypy-svn] r49340 - pypy/dist/pypy/rpython/module/test Message-ID: <20071204152658.D5E658160@code0.codespeak.net> Author: exarkun Date: Tue Dec 4 16:26:58 2007 New Revision: 49340 Modified: pypy/dist/pypy/rpython/module/test/execve_tests.py pypy/dist/pypy/rpython/module/test/test_ll_os.py Log: Change test_ll_os.test_execve so that it works even if pypy is not importable based on $PYTHONPATH Modified: pypy/dist/pypy/rpython/module/test/execve_tests.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/execve_tests.py (original) +++ pypy/dist/pypy/rpython/module/test/execve_tests.py Tue Dec 4 16:26:58 2007 @@ -6,6 +6,7 @@ """ import os, sys +sys.path.append(sys.argv[1]) from pypy.rpython.module.test.test_ll_os import EXECVE_ENV, getllimpl @@ -22,4 +23,4 @@ execve("/usr/bin/env", ["/usr/bin/env"], EXECVE_ENV) if __name__ == '__main__': - globals()[sys.argv[1]]() + globals()[sys.argv[2]]() Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_os.py Tue Dec 4 16:26:58 2007 @@ -1,6 +1,7 @@ import os from py.path import local +import pypy from pypy.tool.udir import udir from pypy.translator.c.test.test_genc import compile @@ -68,7 +69,11 @@ def test_execve(): if os.name != 'posix': py.test.skip('posix specific function') - base = sys.executable + " " + execve_tests + " " + base = " ".join([ + sys.executable, + execve_tests, + str(local(pypy.__file__).join('..', '..')), + '']) # Test exit status and code result = os.system(base + "execve_true") From cfbolz at codespeak.net Tue Dec 4 16:59:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 16:59:20 +0100 (CET) Subject: [pypy-svn] r49343 - in pypy/dist/pypy/rlib: . test Message-ID: <20071204155920.CCAA18182@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 16:59:20 2007 New Revision: 49343 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: some refactorings about the FringeIterator. also contains the beginnings about some crazy ideas about extremely fast string searching. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Tue Dec 4 16:59:20 2007 @@ -53,6 +53,7 @@ class StringNode(object): hash_cache = 0 + charbitmask = 0 def length(self): raise NotImplementedError("base class") @@ -86,6 +87,9 @@ def getslice(self, start, stop): raise NotImplementedError("abstract base class") + def can_contain_int(self, value): + return True #conservative default + def view(self): view([self]) @@ -115,9 +119,13 @@ assert isinstance(s, str) self.s = s is_ascii = True + charbitmask = 0 for c in s: - if ord(c) >= 128: + ordc = ord(c) + if ordc >= 128: is_ascii = False + charbitmask |= 1 << (ordc & 0x1F) + self.charbitmask = charbitmask self._is_ascii = is_ascii def length(self): @@ -158,16 +166,22 @@ def getrope(self, index): return LiteralStringNode.PREBUILT[ord(self.s[index])] + def can_contain_int(self, value): + if value > 255: + return False + if self.is_ascii() and value > 127: + return False + return (1 << (value & 0x1f)) & self.charbitmask + def getslice(self, start, stop): assert 0 <= start <= stop return LiteralStringNode(self.s[start:stop]) def find_int(self, what, start, stop): - if what >= 256: + if not self.can_contain_int(what): return -1 - result = self.s.find(chr(what), start, stop) - return result + return self.s.find(chr(what), start, stop) def literal_concat(self, other): if (isinstance(other, LiteralStringNode) and @@ -198,6 +212,13 @@ def __init__(self, u): assert isinstance(u, unicode) self.u = u + charbitmask = 0 + for c in u: + ordc = ord(c) + if ordc >= 128: + charbitmask |= 1 # be compatible with LiteralStringNode + charbitmask |= 1 << (ordc & 0x1F) + self.charbitmask = charbitmask def length(self): return len(self.u) @@ -236,13 +257,17 @@ return self return LiteralUnicodeNode(unichr(ch)) + def can_contain_int(self, value): + return (1 << (value & 0x1f)) & self.charbitmask + def getslice(self, start, stop): assert 0 <= start <= stop return LiteralUnicodeNode(self.u[start:stop]) def find_int(self, what, start, stop): - result = self.u.find(unichr(what), start, stop) - return result + if not self.can_contain_int(what): + return -1 + return self.u.find(unichr(what), start, stop) def literal_concat(self, other): if (isinstance(other, LiteralUnicodeNode) and @@ -278,6 +303,7 @@ self.balanced = False self._is_ascii = left.is_ascii() and right.is_ascii() self._is_bytestring = left.is_bytestring() and right.is_bytestring() + self.charbitmask = left.charbitmask | right.charbitmask def is_ascii(self): return self._is_ascii @@ -335,6 +361,13 @@ else: return self.left.getrope(index) + def can_contain_int(self, value): + if self.is_bytestring() and value > 255: + return False + if self.is_ascii() and value > 127: + return False + return (1 << (value & 0x1f)) & self.charbitmask + def flatten_string(self): f = fringe(self) return "".join([node.flatten_string() for node in f]) @@ -650,11 +683,6 @@ length = node.length() if stop == -1: stop = length - if start != 0 or stop != length: - newstart, newstop, node = find_straddling(node, start, stop) - offset = start - newstart - start = newstart - stop = newstop assert 0 <= start <= stop if isinstance(node, LiteralNode): pos = node.find_int(what, start, stop) @@ -662,6 +690,9 @@ return pos return pos + offset iter = FringeIterator(node) + newstart = iter._seekforward(start) + offset += start - newstart + start = newstart #import pdb; pdb.set_trace() i = 0 while i < stop: @@ -675,6 +706,8 @@ continue searchstart = max(0, start - i) searchstop = min(stop - i, nodelength) + if searchstop <= 0: + return -1 assert isinstance(fringenode, LiteralNode) pos = fringenode.find_int(what, searchstart, searchstop) if pos != -1: @@ -809,6 +842,29 @@ return curr raise StopIteration + def _seekforward(self, length): + """seek forward up to n characters, returning the number remaining chars. + experimental api""" + curr = None + while self.stack: + curr = self.stack.pop() + if length < curr.length(): + break + length -= curr.length() + else: + raise StopIteration + while isinstance(curr, BinaryConcatNode): + left_length = curr.left.length() + i