From pedronis at codespeak.net Thu Nov 1 12:02:36 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 12:02:36 +0100 (CET) Subject: [pypy-svn] r48231 - in pypy/extradoc/talk: roadshow-google roadshow-ibm Message-ID: <20071101110236.D04A681AB@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 12:02:36 2007 New Revision: 48231 Added: pypy/extradoc/talk/roadshow-google/ pypy/extradoc/talk/roadshow-ibm/ pypy/extradoc/talk/roadshow-ibm/talk.txt - copied, changed from r48230, pypy/extradoc/talk/ep2007/pypy_10_and_jit/talk.txt Log: directories for our two main talks on the US trip From pedronis at codespeak.net Thu Nov 1 12:16:19 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 12:16:19 +0100 (CET) Subject: [pypy-svn] r48232 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101111619.980B581AD@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 12:16:19 2007 New Revision: 48232 Added: pypy/extradoc/talk/roadshow-ibm/graph0.dot pypy/extradoc/talk/roadshow-ibm/graph1.dot pypy/extradoc/talk/roadshow-ibm/overview2.png - copied unchanged from r48230, pypy/extradoc/talk/ep2007/pypy_10_and_jit/overview2.png Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: our slides about the jit arch Added: pypy/extradoc/talk/roadshow-ibm/graph0.dot ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-ibm/graph0.dot Thu Nov 1 12:16:19 2007 @@ -0,0 +1,19 @@ +digraph _generated__graph { +subgraph _generated__ { +_generated__ [shape="box", label="generated", color="black", fillcolor="#a5e6f0", style="filled", width="0.75"]; +edge [label="startblock", style="dashed", color="black", dir="forward", weight="5"]; +_generated__ -> _generated____1 +_generated____1 [shape="box", label="generated__1\ninputargs: v6024\n\nv6025 = cast_pointer(v6024)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6025", style="solid", color="black", dir="forward", weight="5"]; +_generated____1 -> _generated____2 +_generated____2 [shape="box", label="generated__2\ninputargs: v6026\n\nv6027 = getfield(v6026, ('typeptr'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6026 v6027", style="solid", color="black", dir="forward", weight="5"]; +_generated____2 -> _generated____3 +_generated____3 [shape="box", label="generated__3\ninputargs: v6030 v6028\n\nv6029 = cast_ptr_to_int(v6028)\l\lexitswitch: v6029", color="black", fillcolor="#e5ffff", style="filled", width="0.75"]; +edge [label="'default': v6030 v6028", style="dotted", color="red", dir="forward", weight="5"]; +_generated____3 -> _generated____4 +_generated____4 [shape="box", label="generated__4\ninputargs: v6039 v6035\n\nv6031 = getfield((<* struct ExcData { ex...=... }>), ('exc_type'))\lv6032 = getfield((<* struct ExcData { ex...=... }>), ('exc_value'))\lv6033 = setfield((<* struct ExcData { ex...=... }>), ('exc_type'), (<* None>))\lv6034 = setfield((<* struct ExcData { ex...=... }>), ('exc_value'), (<* None>))\lv6036 = direct_call((<* fn ll_continue_comp...blePtr>), (<* struct object { typ...=... }>), v6035)\lv6037 = setfield((<* struct ExcData { ex...=... }>), ('exc_type'), v6031)\lv6038 = setfield((<* struct ExcData { ex...=... }>), ('exc_value'), v6032)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6039 v6035", style="solid", color="black", dir="forward", weight="5"]; +_generated____4 -> _generated____3 +} +} \ No newline at end of file Added: pypy/extradoc/talk/roadshow-ibm/graph1.dot ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-ibm/graph1.dot Thu Nov 1 12:16:19 2007 @@ -0,0 +1,90 @@ +digraph _generated__graph { +subgraph _generated__ { +_generated__ [shape="box", label="generated", color="black", fillcolor="#a5e6f0", style="filled", width="0.75"]; +edge [label="startblock", style="dashed", color="black", dir="forward", weight="5"]; +_generated__ -> _generated____1 +_generated____1 [shape="box", label="generated__1\ninputargs: v6024\n\nv6025 = cast_pointer(v6024)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6025", style="solid", color="black", dir="forward", weight="5"]; +_generated____1 -> _generated____2 +_generated____2 [shape="box", label="generated__2\ninputargs: v6026\n\nv6027 = getfield(v6026, ('typeptr'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6026 v6027", style="solid", color="black", dir="forward", weight="5"]; +_generated____2 -> _generated____3 +_generated____3 [shape="octagon", label="generated__3\ninputargs: v6030 v6028\n\nv6029 = cast_ptr_to_int(v6028)\l\lexitswitch: v6029", color="red", fillcolor="#e5ffff", style="filled", width="0.75"]; +edge [label="24647528: v6030 v6028", style="dotted", color="red", dir="forward", weight="5"]; +_generated____3 -> _generated____4 +edge [label="'default': v6030 v6028", style="dotted", color="red", dir="forward", weight="5"]; +_generated____3 -> _generated____5 +_generated____4 [shape="box", label="generated__4\ninputargs: v10217 v10220\n\nv10218 = cast_pointer(v10217)\lv10219 = getfield(v10218, ('inst_value'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10219 v10217", style="solid", color="black", dir="forward", weight="5"]; +_generated____4 -> _generated____6 +_generated____6 [shape="box", label="generated__6\ninputargs: v10221 v10223\n\nv10222 = int_lt((1), v10221)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10222 v10223", style="solid", color="black", dir="forward", weight="5"]; +_generated____6 -> _generated____7 +_generated____7 [shape="octagon", label="generated__7\ninputargs: v10224 v10225\n\n\lexitswitch: v10224", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: (1)", style="dotted", color="red", dir="forward", weight="5"]; +_generated____7 -> _generated____8 +edge [label="True: v10225", style="dotted", color="red", dir="forward", weight="5"]; +_generated____7 -> _generated____9 +_generated____8 [shape="box", label="generated__8\ninputargs: v10230\n\nv10226 = malloc((GcStruct pypy.jit.tl.tlc.IntObj))\lv10227 = cast_pointer(v10226)\lv10228 = setfield(v10227, ('typeptr'), (<* struct object_vtabl...=... }>))\lv10229 = cast_pointer(v10227)\lv10231 = setfield(v10229, ('inst_value'), v10230)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10227", style="solid", color="black", dir="forward", weight="5"]; +_generated____8 -> _generated____10 +_generated____10 [shape="box", label="generated__10\ninputargs: v10232\n\nv10233 = cast_pointer(v10232)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10233", style="solid", color="black", dir="forward", weight="5"]; +_generated____10 -> _generated____11 +_generated____11 [shape="box", label="generated__11\ninputargs: v10234\n\nreturn v10234", color="black", fillcolor="green", style="filled", width="0.75"]; +_generated____9 [shape="box", label="generated__9\ninputargs: v10236\n\nv10235 = same_as((0))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10236 v10235", style="solid", color="black", dir="forward", weight="5"]; +_generated____9 -> _generated____12 +_generated____12 [shape="box", label="generated__12\ninputargs: v10239 v10237\n\nv10238 = int_is_true(v10237)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10238 v10239", style="solid", color="black", dir="forward", weight="5"]; +_generated____12 -> _generated____13 +_generated____13 [shape="octagon", label="generated__13\ninputargs: v10240 v10241\n\n\lexitswitch: v10240", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: v10241", style="dotted", color="red", dir="forward", weight="5"]; +_generated____13 -> _generated____14 +edge [label="True: v10241", style="dotted", color="red", dir="forward", weight="5"]; +_generated____13 -> _generated____15 +_generated____14 [shape="box", label="generated__14\ninputargs: v10242\n\nv10243 = cast_pointer(v10242)\lv10244 = getfield(v10243, ('inst_value'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10244 v10242", style="solid", color="black", dir="forward", weight="5"]; +_generated____14 -> _generated____16 +_generated____16 [shape="box", label="generated__16\ninputargs: v10245 v10247\n\nv10246 = int_mul((1), v10245)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10247 v10246", style="solid", color="black", dir="forward", weight="5"]; +_generated____16 -> _generated____17 +_generated____17 [shape="box", label="generated__17\ninputargs: v10248 v10251\n\nv10249 = cast_pointer(v10248)\lv10250 = getfield(v10249, ('inst_value'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10250 v10251 v10248", style="solid", color="black", dir="forward", weight="5"]; +_generated____17 -> _generated____18 +_generated____18 [shape="box", label="generated__18\ninputargs: v10252 v10254 v10255\n\nv10253 = int_sub(v10252, (1))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10253 v10254 v10255", style="solid", color="black", dir="forward", weight="5"]; +_generated____18 -> _generated____19 +_generated____19 [shape="box", label="generated__19\ninputargs: v10256 v10258 v10259\n\nv10257 = int_lt((1), v10256)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10257 v10258 v10256 v10259", style="solid", color="black", dir="forward", weight="5"]; +_generated____19 -> _generated____20 +_generated____20 [shape="octagon", label="generated__20\ninputargs: v10260 v10261 v10262 v10263\n\n\lexitswitch: v10260", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: v10261", style="dotted", color="red", dir="forward", weight="5"]; +_generated____20 -> _generated____8 +edge [label="True: v10263 v10261 v10262", style="dotted", color="red", dir="forward", weight="5"]; +_generated____20 -> _generated____21 +_generated____21 [shape="box", label="generated__21\ninputargs: v10265 v10266 v10267\n\nv10264 = same_as((0))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10265 v10266 v10267 v10264", style="solid", color="black", dir="forward", weight="5"]; +_generated____21 -> _generated____22 +_generated____22 [shape="box", label="generated__22\ninputargs: v10270 v10271 v10272 v10268\n\nv10269 = int_is_true(v10268)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10269 v10271 v10272 v10270", style="solid", color="black", dir="forward", weight="5"]; +_generated____22 -> _generated____23 +_generated____23 [shape="octagon", label="generated__23\ninputargs: v10273 v10274 v10275 v10276\n\n\lexitswitch: v10273", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: v10275 v10274 v10276", style="dotted", color="red", dir="forward", weight="5"]; +_generated____23 -> _generated____24 +edge [label="True: v10274", style="dotted", color="red", dir="forward", weight="5"]; +_generated____23 -> _generated____8 +_generated____24 [shape="box", label="generated__24\ninputargs: v10278 v10277 v10280\n\nv10279 = int_mul(v10277, v10278)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10278 v10279 v10280", style="solid", color="black", dir="forward", weight="5"]; +_generated____24 -> _generated____25 +_generated____25 [shape="box", label="generated__25\ninputargs: v10281 v10283 v10284\n\nv10282 = int_sub(v10281, (1))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10282 v10283 v10284", style="solid", color="black", dir="forward", weight="5"]; +_generated____25 -> _generated____19 +_generated____15 [shape="box", label="generated__15\ninputargs: v10286\n\nv10285 = same_as((1))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10285", style="solid", color="black", dir="forward", weight="5"]; +_generated____15 -> _generated____8 +_generated____5 [shape="box", label="generated__5\ninputargs: v6039 v6035\n\nv6031 = getfield((<* struct ExcData { ex...=... }>), ('exc_type'))\lv6032 = getfield((<* struct ExcData { ex...=... }>), ('exc_value'))\lv6033 = setfield((<* struct ExcData { ex...=... }>), ('exc_type'), (<* None>))\lv6034 = setfield((<* struct ExcData { ex...=... }>), ('exc_value'), (<* None>))\lv6036 = direct_call((<* fn ll_continue_comp...blePtr>), (<* struct object { typ...=... }>), v6035)\lv6037 = setfield((<* struct ExcData { ex...=... }>), ('exc_type'), v6031)\lv6038 = setfield((<* struct ExcData { ex...=... }>), ('exc_value'), v6032)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6039 v6035", style="solid", color="black", dir="forward", weight="5"]; +_generated____5 -> _generated____3 +} +} \ No newline at end of file Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 12:16:19 2007 @@ -105,6 +105,8 @@ pypy-c-jit ====================== +.. where to put this? + PyPy 1.0 contains both the dynamic compiler generator and the start of its application to PyPy's Python intepreter. @@ -180,8 +182,6 @@ .. need to save state in a compact form: paths -.. demo? - Promotion (example) ======================== @@ -251,4 +251,32 @@ PyPy proves this a viable approach worth of further exploration. +.. EXTRA MATERIAL: +.. backend material: not for the general talk +.. virtualizables: not for general talk +.. state of gc framework + +Open Issues +============== + +- inlining control + +- promotion switch explosion fallbacks + +- jit only the hot-spots + +- more hints needed in PyPy's Python + +Virtualizable Frames +====================== +- frames need to live in the heap + (tracebacks ...) and be introspectable +- jit code wants local variables to live in registers + and on the stack +- => mark the frame class as "virtualizable" +- jit code uses lazy allocation and stores + some contents (local variables...) in register + and stack +- outside world access gets intercepted + to be able to force lazy virtual data into the heap From pedronis at codespeak.net Thu Nov 1 12:54:05 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 12:54:05 +0100 (CET) Subject: [pypy-svn] r48233 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101115405.3398781B7@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 12:54:04 2007 New Revision: 48233 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: WIP: translation aspects slides Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 12:54:04 2007 @@ -1,5 +1,67 @@ .. include:: + +Translation aspects +======================== + +- The interpreters that we write should be free of low-level details + (this is also required to target platform as different as Posix/C + and the JVM/.NET) + +- Advanced features related to execution should not need wide-spread + changes to the interpreters + +- The interpreters instead should use support offered and inserted by + the framework + +Examples: gc and memory management, stack inspection and manipulation + +Implementation +================== + +- Translation aspects are implemented as transformation of low-level + graphs + +- Calls to library/helper code written in RPython can be inserted + too which will also be analyzed and translated + +GC Framework +=============== + +- RPython has been extended with allocation and adress manipulation + primitives that can be used to express GC in RPython directly + +- GCs are linked by substituting memory allocation operations with calls + into them + +- Right now bookeeping code to keep track of reference counting or + roots is inserted by the GC framework + +- Inlining is used to eliminate call overhead for the fast paths of + allocation + +.. MMTk reference + +Stackless transformation +========================= + +- One translation aspect transformation insertes support code across + around calls such that the stack can be unwind and function asked to + store and reflect their current activation frame state to the heap + +- Chains of saved activation state can then be resumed + +- We have implemented coroutine switching using this + +Resume points +=============== + +- Further this transformation can insert code tha allows to construct + chains of activation states corresponding to labeled points in the + program, we use this to construct resumable low-level activation + chains from information captured as serialized language level frame + chains. + Flexibility vs. Performance ============================= From rxe at codespeak.net Thu Nov 1 13:54:03 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 1 Nov 2007 13:54:03 +0100 (CET) Subject: [pypy-svn] r48234 - pypy/dist/pypy/translator/llvm Message-ID: <20071101125403.3DB8A814F@code0.codespeak.net> Author: rxe Date: Thu Nov 1 13:54:01 2007 New Revision: 48234 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/modwrapper.py Log: make the whole way we make entry names and filenames saner and fix issue with names like propgating through to filenames Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Thu Nov 1 13:54:01 2007 @@ -131,7 +131,7 @@ self.lastdir = py.path.local() self.dirpath.chdir() - return llvmfile.purebasename + return self.genllvm.entry_name def make_module(self): base = self.setup() Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Thu Nov 1 13:54:01 2007 @@ -177,7 +177,18 @@ ptr = getfunctionptr(bk.getdesc(func).getuniquegraph()) c = inputconst(lltype.typeOf(ptr), ptr) self.db.prepare_arg_value(c) - self.entry_func_name = func.func_name + + # ensure unqiue entry node name for testing + entry_node = self.db.obj2node[c.value._obj] + name = entry_node.name + if name in self.function_count: + self.function_count[name] += 1 + name += '_%d' % self.function_count[name] + entry_node.name = name + else: + self.function_count[name] = 1 + + self.entry_name = name[6:] return c.value._obj def generate_ll_externs(self): @@ -201,13 +212,7 @@ def create_codewriter(self): # prevent running the same function twice in a test - if self.entry_func_name in self.function_count: - postfix = '_%d' % self.function_count[self.entry_func_name] - self.function_count[self.entry_func_name] += 1 - else: - postfix = '' - self.function_count[self.entry_func_name] = 1 - filename = udir.join(self.entry_func_name + postfix).new(ext='.ll') + filename = udir.join(self.entry_name).new(ext='.ll') f = open(str(filename), 'w') return CodeWriter(f, self.db), filename Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Thu Nov 1 13:54:01 2007 @@ -132,7 +132,7 @@ """ epilog = """ -__entrypoint__ = _c.__entrypoint__%(name)s +__entrypoint__ = _c.__entrypoint__pypy_%(name)s # %(RT)r to_llargs = %(to_llargs)s @@ -157,7 +157,7 @@ def __init__(self, genllvm, dllname): self.genllvm = genllvm self.dllname = dllname - basename = self.genllvm.filename.purebasename + '_wrapper.py' + basename = self.genllvm.entry_name + '_wrapper.py' self.modfilename = genllvm.filename.new(basename=basename) self.count = 0 @@ -166,7 +166,7 @@ self.file.write(self.prolog % self.dllname) g = self.genllvm.entrynode.graph - name = "pypy_" + g.name + name = self.genllvm.entry_name ARGS = [a.concretetype for a in g.startblock.inputargs] inputargtypes, to_llargs = self.build_args_to_ctypes_to_lltype(ARGS) From antocuni at codespeak.net Thu Nov 1 13:55:44 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 1 Nov 2007 13:55:44 +0100 (CET) Subject: [pypy-svn] r48235 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101125544.630678172@code0.codespeak.net> Author: antocuni Date: Thu Nov 1 13:55:43 2007 New Revision: 48235 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: I guess this should really be y*y. Without the extra ``y``, it shows up as 9+y in firefox. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 13:55:43 2007 @@ -259,7 +259,7 @@ | | | ``def f_(x, y):`` | | | | ``switch x:`` | | | | *case 3:* | -| | | *return 9 + y* | +| | | *return 9 + y*y | | | | ``default:`` | | | | ``compile_more(value=x)`` | | | | | From cfbolz at codespeak.net Thu Nov 1 14:01:02 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 1 Nov 2007 14:01:02 +0100 (CET) Subject: [pypy-svn] r48236 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101130102.3542281B5@code0.codespeak.net> Author: cfbolz Date: Thu Nov 1 14:01:01 2007 New Revision: 48236 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: some fixes, XXXs Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 14:01:01 2007 @@ -12,9 +12,9 @@ changes to the interpreters - The interpreters instead should use support offered and inserted by - the framework + the translation framework -Examples: gc and memory management, stack inspection and manipulation +Examples: GC and memory management, stack inspection and manipulation Implementation ================== @@ -23,32 +23,34 @@ graphs - Calls to library/helper code written in RPython can be inserted - too which will also be analyzed and translated + too which will also be analyzed and translated GC Framework =============== -- RPython has been extended with allocation and adress manipulation +- RPython has been extended with allocation and address manipulation primitives that can be used to express GC in RPython directly - GCs are linked by substituting memory allocation operations with calls into them -- Right now bookeeping code to keep track of reference counting or +- Right now bookkeeping code to keep track of reference counting or roots is inserted by the GC framework - Inlining is used to eliminate call overhead for the fast paths of - allocation + allocation (XXX not only of allocation, of the write barrier and the root bookkeeping too. do you want to mention that?) .. MMTk reference Stackless transformation ========================= -- One translation aspect transformation insertes support code across +- One translation aspect transformation inserts support code across around calls such that the stack can be unwind and function asked to store and reflect their current activation frame state to the heap +XXX mention that this works regardless of the backend or something? + - Chains of saved activation state can then be resumed - We have implemented coroutine switching using this @@ -56,11 +58,14 @@ Resume points =============== -- Further this transformation can insert code tha allows to construct - chains of activation states corresponding to labeled points in the - program, we use this to construct resumable low-level activation + +- this transformation can also insert code that allows to construct + artificial chains of activation states corresponding to labeled points in the + program + +- we use this to construct resumable low-level activation chains from information captured as serialized language level frame - chains. + chains. XXX a bit hard to understand Flexibility vs. Performance ============================= From cfbolz at codespeak.net Thu Nov 1 14:03:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 1 Nov 2007 14:03:35 +0100 (CET) Subject: [pypy-svn] r48237 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101130335.C29A4814F@code0.codespeak.net> Author: cfbolz Date: Thu Nov 1 14:03:35 2007 New Revision: 48237 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: I think this is what was intended Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 14:03:35 2007 @@ -264,7 +264,7 @@ | | | ``def f_(x, y):`` | | | | ``switch x:`` | | | | *case 3:* | -| | | *return 9 + y*y | +| | | *return 9 + y*y* | | | | ``default:`` | | | | ``compile_more(value=x)`` | | | | | From rxe at codespeak.net Thu Nov 1 14:39:56 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 1 Nov 2007 14:39:56 +0100 (CET) Subject: [pypy-svn] r48238 - pypy/dist/pypy/translator/llvm/module Message-ID: <20071101133956.B011C81A9@code0.codespeak.net> Author: rxe Date: Thu Nov 1 14:39:55 2007 New Revision: 48238 Modified: pypy/dist/pypy/translator/llvm/module/excsupport.py pypy/dist/pypy/translator/llvm/module/support.py Log: well, well... so it is seems that we have been running two independent exception strategies in llvm - one for the backend and the tests, and the other one done by transformation. so upshot is none of the externs could raise a real error and our tests never raised an error. ive know idea how long this has been this way - probably a veeery long time. (and it is probably my fault :-) Modified: pypy/dist/pypy/translator/llvm/module/excsupport.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/excsupport.py (original) +++ pypy/dist/pypy/translator/llvm/module/excsupport.py Thu Nov 1 14:39:55 2007 @@ -1,7 +1,7 @@ entrycode = ''' ccc %(returntype)s %%__entrypoint__%(entrypointname)s { - store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type + call fastcc void %%pypy_rpyexc_clear() %%result = call %(cconv)s %(returntype)s%%%(entrypointname)s ret %(returntype)s %%result } @@ -9,18 +9,15 @@ voidentrycode = ''' ccc %(returntype)s %%__entrypoint__%(entrypointname)s { - store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type + call fastcc void %%pypy_rpyexc_clear() call %(cconv)s %(returntype)s%%%(entrypointname)s ret void } ''' raisedcode = ''' -;XXX this should use the transformation data that has the same purpose -ccc int %%__entrypoint__raised_LLVMException() { - %%tmp = load %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type - %%result = cast %%RPYTHON_EXCEPTION_VTABLE* %%tmp to int - ret int %%result +ccc bool %%__entrypoint__raised_LLVMException() { + %%result = call fastcc bool %%pypy_rpyexc_occured() + ret bool %%result } - ''' Modified: pypy/dist/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/support.py (original) +++ pypy/dist/pypy/translator/llvm/module/support.py Thu Nov 1 14:39:55 2007 @@ -1,8 +1,4 @@ - extdeclarations = """ -%last_exception_type = internal global %RPYTHON_EXCEPTION_VTABLE* null -%last_exception_value = internal global %RPYTHON_EXCEPTION* null - declare ccc uint %strlen(sbyte*) declare ccc void %llvm.memsetPOSTFIX(sbyte*, ubyte, UWORD, UWORD) declare ccc void %llvm.memcpyPOSTFIX(sbyte*, sbyte*, UWORD, UWORD) @@ -21,13 +17,6 @@ ret WORD %size } -internal fastcc int %RPyExceptionOccurred() { - %tmp.0 = load %RPYTHON_EXCEPTION_VTABLE** %last_exception_type - %bool_res = setne %RPYTHON_EXCEPTION_VTABLE* %tmp.0, null - %res = cast bool %bool_res to int - ret int %res -} - internal fastcc %RPyString* %RPyString_FromString(sbyte* %s) { %lenu = call ccc uint %strlen(sbyte* %s) %lenuword = cast uint %lenu to UWORD @@ -79,25 +68,9 @@ """ -from sys import maxint -if maxint != 2**31-1: - extfunctions += """ -internal fastcc void %pypy_ll_raise_OSError__Signed(int %errno_0) { - %tmp = cast int %errno_0 to long - call fastcc void %pypy_ll_raise_OSError__Signed(long %tmp) - ret void -} - -internal fastcc void %pypy__RPyListOfString_SetItem__listPtr_Signed_rpy_stringPtr(%RPyListOfString* %l_1, int %index_0, %RPyString* %newstring_0) { - %index_0_long = cast int %index_0 to long - call fastcc void %pypy__RPyListOfString_SetItem__listPtr_Signed_rpy_stringPtr(%RPyListOfString* %l_1, long %index_0_long, %RPyString* %newstring_0) - ret void -} - -""" - extfunctions_standalone = """ """ +from sys import maxint if maxint != 2**31-1: extfunctions_standalone += """ internal fastcc int %pypy_entry_point(%RPyListOfString* %argv) { @@ -108,15 +81,12 @@ """ + def write_raise_exc(c_name, exc_repr, codewriter): l = """ internal fastcc void %%raise%s(sbyte* %%msg) { - %%exception_value = cast %s to %%RPYTHON_EXCEPTION* - %%tmp = getelementptr %%RPYTHON_EXCEPTION* %%exception_value, int 0, uint 0 - %%exception_type = load %%RPYTHON_EXCEPTION_VTABLE** %%tmp - store %%RPYTHON_EXCEPTION_VTABLE* %%exception_type, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type - store %%RPYTHON_EXCEPTION* %%exception_value, %%RPYTHON_EXCEPTION** %%last_exception_value + ;%%exception_value = cast %s to %%RPYTHON_EXCEPTION* ret void } """ % (c_name, exc_repr) From pedronis at codespeak.net Thu Nov 1 14:44:56 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 14:44:56 +0100 (CET) Subject: [pypy-svn] r48239 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101134456.EDF4281BD@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 14:44:56 2007 New Revision: 48239 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: fixes, shuffling stuff around Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 14:44:56 2007 @@ -38,35 +38,21 @@ roots is inserted by the GC framework - Inlining is used to eliminate call overhead for the fast paths of - allocation (XXX not only of allocation, of the write barrier and the root bookkeeping too. do you want to mention that?) + allocation and barriers .. MMTk reference Stackless transformation ========================= -- One translation aspect transformation inserts support code across - around calls such that the stack can be unwind and function asked to +- One translation aspect transformation inserts support code + around calls such that the stack can be unwound and functions asked to store and reflect their current activation frame state to the heap -XXX mention that this works regardless of the backend or something? - - Chains of saved activation state can then be resumed - We have implemented coroutine switching using this -Resume points -=============== - - -- this transformation can also insert code that allows to construct - artificial chains of activation states corresponding to labeled points in the - program - -- we use this to construct resumable low-level activation - chains from information captured as serialized language level frame - chains. XXX a bit hard to understand - Flexibility vs. Performance ============================= @@ -318,7 +304,9 @@ PyPy proves this a viable approach worth of further exploration. -.. EXTRA MATERIAL: +EXTRA MATERIAL +================== + .. backend material: not for the general talk .. virtualizables: not for general talk .. state of gc framework @@ -347,3 +335,14 @@ and stack - outside world access gets intercepted to be able to force lazy virtual data into the heap + + +Resume points +=============== + +- this transformation can also insert code that allows to construct + artificial chains of activation states corresponding to labeled points in the + program + +- we use this to support resuming serialized language-level coroutines + From pedronis at codespeak.net Thu Nov 1 14:48:23 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 14:48:23 +0100 (CET) Subject: [pypy-svn] r48240 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101134823.546AB81D7@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 14:48:22 2007 New Revision: 48240 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: connecting title Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 14:48:22 2007 @@ -53,8 +53,10 @@ - We have implemented coroutine switching using this -Flexibility vs. Performance -============================= +A special aspect: JIT generation +================================== + +Flexibility vs. Performance: * Interpreters are easy to write and evolve From arigo at codespeak.net Thu Nov 1 16:29:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Nov 2007 16:29:54 +0100 (CET) Subject: [pypy-svn] r48242 - pypy/extradoc/talk/roadshow-ibm/ui Message-ID: <20071101152954.BDC65816E@code0.codespeak.net> Author: arigo Date: Thu Nov 1 16:29:53 2007 New Revision: 48242 Added: pypy/extradoc/talk/roadshow-ibm/ui/ - copied from r48241, pypy/extradoc/talk/ep2007/ui/ Log: Add this directory for the rst2s5 HTML to display properly. From arigo at codespeak.net Thu Nov 1 16:44:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Nov 2007 16:44:52 +0100 (CET) Subject: [pypy-svn] r48243 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101154452.B6DF98184@code0.codespeak.net> Author: arigo Date: Thu Nov 1 16:44:52 2007 New Revision: 48243 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Details. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 16:44:52 2007 @@ -5,7 +5,7 @@ ======================== - The interpreters that we write should be free of low-level details - (this is also required to target platform as different as Posix/C + (this is also required to target platforms as different as Posix/C and the JVM/.NET) - Advanced features related to execution should not need wide-spread @@ -60,7 +60,7 @@ * Interpreters are easy to write and evolve -* For high performance dynamic compilation is required +* For high performance, dynamic compilation is required Traditional JIT compilers =============================== From pedronis at codespeak.net Thu Nov 1 19:24:07 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 19:24:07 +0100 (CET) Subject: [pypy-svn] r48244 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101182407.BFE528178@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 19:24:06 2007 New Revision: 48244 Added: pypy/extradoc/talk/roadshow-ibm/abstract.txt (contents, props changed) Log: the abstract we sent ibm Added: pypy/extradoc/talk/roadshow-ibm/abstract.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-ibm/abstract.txt Thu Nov 1 19:24:06 2007 @@ -0,0 +1,24 @@ +PyPy - automatic generation of VMs for dynamic languages - JIT included +------------------------------------------------------------------------ + +PyPy is a framework written in Python for generating virtual machines +for dynamic languages. The VMs are flexibly produced from a +high-level "specification" in the form of a simple interpreter for the +dynamic language. The interpreters are written in a high-level static +subset of Python. + +Low-level details like memory allocation and object layout and stack +inspection are not encoded manually, but inserted by the VM generation process. +This allows us to produce VMs that run within a wide range of execution +environments (from C-like to JVM/.NET). + +The framework has the ability to automatically generate a dynamic compiler +from the interpreter too. A pragmatic application of partial evaluation +techniques guided by a few hints is used for the task. + +Crucial for the effectiveness of dynamic compilation is +the use of run-time information to improve compilation results: in our +approach, a powerful primitive called "promotion" that +"promotes" run-time values to compile-time is used to that effect. + + From lac at codespeak.net Thu Nov 1 19:26:02 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Thu, 1 Nov 2007 19:26:02 +0100 (CET) Subject: [pypy-svn] r48245 - pypy/extradoc/talk/roadshow-google Message-ID: <20071101182602.C6C4C8185@code0.codespeak.net> Author: lac Date: Thu Nov 1 19:26:02 2007 New Revision: 48245 Added: pypy/extradoc/talk/roadshow-google/abstract.txt (contents, props changed) Log: This is the abstract we sent to Google Added: pypy/extradoc/talk/roadshow-google/abstract.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-google/abstract.txt Thu Nov 1 19:26:02 2007 @@ -0,0 +1,35 @@ +In Search of Speed and Flexibility. PyPy and the Art of Generating Virtual Machines +------------------------------------------------------------------------------------ +We all want our favourite dynamic language to be faster, to be even more +dynamic and to pick up nifty ideas from its peers. + +But this is so much work! + +There ought to be a better way to implement dynamic languages that +enables growth and necessary change with less effort and more ease. +Writing yet another interpreter in C or even Java is not it. + +PyPy is a framework to implement dynamic languages that embraces this +challenge. It allows us to write such languages in the form of simple +interpreters, expressed in a rich subset of Python. This means that +the resulting implementations can can be evolved with reasonable +effort. + +PyPy then generates a variety of virtual machines (VMs) from a single +source. We can therefore target a wide range of environments, +including C/Posix, Java, or .NET. This avoids the major source of +fragmentation within a dynamic language community -- the need for a +separate language implementation for each virtual machine. It also +means that languages as distinct as Python, Prolog and Smalltalk can +share the bulk of our compiler machinery. + +The flexible toolchain which is necessary for virtual machine +generation turns out to be good for other things as well. Traditional +implementations for dynamic languages have to make some very hard +decisions early, which then become entrenched in the whole codebase, +making it difficult or impossible to change later. It can be nice to +experiment with different garbage collectors, but not if you have to +write a complete new implementation of your favourite language for +each one. With PyPy you don't have to. You can also generate and +tailor a JIT-compiler for the language at hand, putting to rest the +notion that flexibility must come at the expense of speed. \ No newline at end of file From pedronis at codespeak.net Thu Nov 1 19:28:05 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 19:28:05 +0100 (CET) Subject: [pypy-svn] r48246 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101182805.EAF868185@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 19:28:05 2007 New Revision: 48246 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: checkin the title Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 19:28:05 2007 @@ -1,5 +1,8 @@ .. include:: +======================================================================== +PyPy - automatic generation of VMs for dynamic languages - JIT included +======================================================================== Translation aspects ======================== From tismer at codespeak.net Fri Nov 2 06:08:01 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Fri, 2 Nov 2007 06:08:01 +0100 (CET) Subject: [pypy-svn] r48247 - pypy/dist/pypy/translator/goal Message-ID: <20071102050801.9F33F81A2@code0.codespeak.net> Author: tismer Date: Fri Nov 2 06:08:00 2007 New Revision: 48247 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: this is still not completely correct, but way better than before. Nobody will believe that I hacked a week on this, already, because it cannot be debugged without compilation... Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Fri Nov 2 06:08:00 2007 @@ -16,14 +16,14 @@ --info print translation information about this PyPy executable """ -import sys, os +import sys DEBUG = False # dump exceptions before calling the except hook originalexcepthook = sys.__excepthook__ def run_toplevel(f, *fargs, **fkwds): - """Calls f() and handle all OperationErrors. + """Calls f() and handles all OperationErrors. Intended use is to run the main program or one interactive statement. run_protected() handles details like forwarding exceptions to sys.excepthook(), catching SystemExit, printing a newline after @@ -144,16 +144,120 @@ # ____________________________________________________________ # Main entry point +# faking os, until we really are able to import it from source +""" +Why fake_os ? +------------- + +When pypy is starting, the builtin modules are there, but os.path +is a Python module. The current startup code depends on os.py, +which had the side effect that os.py got frozen into the binary. + +Os.py is a wrapper for system specific built-in modules and tries +to unify the interface. One problem is os.environ which looks +very differently per os. +Due to the way os.py initializes its environ variable on Windows, +it s hard to get a correct os.getenv that uses the actual environment +variables. In order to fix that without modifying os.py, we need to +re-import it several times, before and after compilation. + +When compiling the source, we first create a fake_os class instance. +fake_os contains all of os, when compiling.reduction seems to be not +appropriate, since lots of other modules are sucked in, at the same time. +A complete reduction is possible, but would require much more work. + +In the course of creating fake_os, we import the real os. We keep a record +of which modules were imported, before. + +During start of the entry_point, we call os.setup(). +This repeats an import of the specific parts in os.name. +Depending of the underlying os, this might or might not +cause a redefinition of certain os variables, but this happens +exactly like in os.py's initialization. +We then capture the variable environ, which may or may not come from the +specific os. +After that, we re-initialize our dict to what is expected in standard +os. The effect of all this magic is that we have captured the environ +cariable, again, and we now can redefine os.getenv to use this fresh +variable, being accurate and not frozen. + +At this point, we finally can compute the location of our import. + +As a side effect, the involved modules are re-imported, although they had +been compiled in, so PyPy behaves really conformant, making all .py +modules changeable, again. +""" + +from pypy.rlib.objectmodel import we_are_translated + +class fake_os: + def __init__(self): + import sys + self.pre_import = sys.modules.keys() + import os + self.os = os + # make ourselves a clone of os + self.__dict__.update(self.os.__dict__) + + def setup(self): + # we now repeat the os-specific initialization, which + # must be done before importing os, since os hides + # variables after initialization. + specifics = __import__(self.os.name) + self.__dict__.update(specifics.__dict__) + # depending on the os, we now might or might not have + # a new environ variable. However, we can now + # repeat the environ initialisation from os.py + environ = self.os._Environ(self.environ) + # to be safe, reset our dict to be like os + self.__dict__.update(self.os.__dict__) + # but now we insert the fresh environ + self.environ = environ + del self.getenv + # use our method, instead of the os's one's + assert self.getenv + + def teardown(self): + for mod in sys.modules.keys(): + if mod not in self.pre_import: + del sys.modules[mod] + global os + import os + + def getenv(self, key, default=None): + """Get an environment variable, return None if it doesn't exist. + The optional second argument can specify an alternate default.""" + return self.environ.get(key, default) + +os = fake_os() + AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' +# hack to determine if we are on windows. Maybe there is a better way... +DRIVE_LETTER_SEP = '' +try: + here = os.getcwd() + try: + os.chdir('C:') + DRIVE_LETTER_SEP = ':' + finally: + os.chdir(here) +except os.OSError: + DRIVE_LETTER_SEP = None +IS_WINDOWS = DRIVE_LETTER_SEP is not None +# XXX extend here + def entry_point(executable, argv): # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH - if os.sep not in executable: + os.setup() # this is the faked one + if os.sep not in executable and DRIVE_LETTER_SEP not in executable: path = os.getenv('PATH') if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) + if we_are_translated and IS_WINDOWS and not fn.lower().endswith('.exe'): + fn += '.exe' if os.path.isfile(fn): executable = fn break @@ -177,7 +281,9 @@ continue sys.path = newpath # found! break - + + os.teardown() # from now on this is the real one + go_interactive = False run_command = False import_site = True From santagada at codespeak.net Fri Nov 2 06:37:13 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Fri, 2 Nov 2007 06:37:13 +0100 (CET) Subject: [pypy-svn] r48248 - in pypy/dist/pypy: rpython/raisingops translator/llvm/test Message-ID: <20071102053713.DDE7D80F9@code0.codespeak.net> Author: santagada Date: Fri Nov 2 06:37:12 2007 New Revision: 48248 Added: pypy/dist/pypy/translator/llvm/test/test_rconstantdict.py Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py pypy/dist/pypy/translator/llvm/test/test_exception.py Log: Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py ============================================================================== --- pypy/dist/pypy/rpython/raisingops/raisingops.py (original) +++ pypy/dist/pypy/rpython/raisingops/raisingops.py Fri Nov 2 06:37:12 2007 @@ -1,7 +1,8 @@ import sys from pypy.rlib.rarithmetic import r_longlong, r_uint, intmask from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.lltypesystem.lltype import Signed +from pypy.rpython.lltypesystem.lltype import Signed, SignedLongLong, \ + UnsignedLongLong #XXX original SIGNED_RIGHT_SHIFT_ZERO_FILLS not taken into account #XXX assuming HAVE_LONG_LONG (int_mul_ovf) @@ -33,6 +34,27 @@ else: raise ZeroDivisionError("unsigned integer division") +def llong_floordiv_zer(x, y): + '''#define OP_LLONG_FLOORDIV_ZER(x,y,r) \ + if ((y)) { OP_LLONG_FLOORDIV(x,y,r); } \ + else FAIL_ZER("integer division") + ''' + if y: + return llop.llong_floordiv(SignedLongLong, x, y) + else: + raise ZeroDivisionError("integer division") + +def ullong_floordiv_zer(x, y): + '''#define OP_ULLONG_FLOORDIV_ZER(x,y,r) \ + if ((y)) { OP_ULLONG_FLOORDIV(x,y,r); } \ + else FAIL_ZER("unsigned integer division") + ''' + if y: + return llop.llong_floordiv(UnsignedLongLong, x, y) + else: + raise ZeroDivisionError("unsigned integer division") + + def int_neg_ovf(x): if x == LONG_MIN: raise OverflowError("integer negate") @@ -199,6 +221,16 @@ else: raise ZeroDivisionError("integer modulo") +def llong_mod_zer(x, y): + '''#define OP_LLONG_MOD_ZER(x,y,r) \ + if ((y)) { OP_LLONG_MOD(x,y,r); } \ + else FAIL_ZER("integer modulo") + ''' + if y: + return llop.int_mod(SignedLongLong, x, y) + else: + raise ZeroDivisionError("integer modulo") + # Helpers... def _Py_ARITHMETIC_RIGHT_SHIFT(i, j): Modified: pypy/dist/pypy/translator/llvm/test/test_exception.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_exception.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_exception.py Fri Nov 2 06:37:12 2007 @@ -3,9 +3,12 @@ import py from pypy.translator.test.snippet import try_raise_choose from pypy.rlib.rarithmetic import r_uint, ovfcheck, ovfcheck_lshift - +from pypy.rpython.test.test_exception import BaseTestException from pypy.translator.llvm.test.runtest import * +class TestLLVMException(LLVMTest, BaseTestException): + pass + class TestException(Exception): pass Added: pypy/dist/pypy/translator/llvm/test/test_rconstantdict.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rconstantdict.py Fri Nov 2 06:37:12 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rconstantdict import BaseTestRconstantdict +from pypy.translator.llvm.test.runtest import * + +class TestLLVMRconstantdict(LLVMTest, BaseTestRconstantdict): + pass From arigo at codespeak.net Fri Nov 2 08:42:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Nov 2007 08:42:17 +0100 (CET) Subject: [pypy-svn] r48249 - pypy/branch/roadshow Message-ID: <20071102074217.5918A8195@code0.codespeak.net> Author: arigo Date: Fri Nov 2 08:42:16 2007 New Revision: 48249 Added: pypy/branch/roadshow/ - copied from r48246, pypy/dist/ Log: A branch (or possibly a tag) that is known to translate and be otherwise reasonably free of blocking issues. We'll stick to this branch during our U.S. trip to avoid bad surprizes. From arigo at codespeak.net Fri Nov 2 10:35:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Nov 2007 10:35:07 +0100 (CET) Subject: [pypy-svn] r48251 - pypy/dist/pypy/translator/c/src Message-ID: <20071102093507.8283581A1@code0.codespeak.net> Author: arigo Date: Fri Nov 2 10:35:06 2007 New Revision: 48251 Modified: pypy/dist/pypy/translator/c/src/int.h Log: An XXX. Modified: pypy/dist/pypy/translator/c/src/int.h ============================================================================== --- pypy/dist/pypy/translator/c/src/int.h (original) +++ pypy/dist/pypy/translator/c/src/int.h Fri Nov 2 10:35:06 2007 @@ -60,6 +60,8 @@ OP_INT_ADD(x,y,r); \ if (r >= (x)); \ else FAIL_OVF("integer addition") +/* XXX can a C compiler be too clever and think it can "prove" that + * r >= x always hold above? */ #define OP_INT_SUB(x,y,r) r = (x) - (y) From cfbolz at codespeak.net Fri Nov 2 11:18:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Nov 2007 11:18:22 +0100 (CET) Subject: [pypy-svn] r48252 - pypy/branch/ooexternals Message-ID: <20071102101822.CE53381AF@code0.codespeak.net> Author: cfbolz Date: Fri Nov 2 11:18:21 2007 New Revision: 48252 Added: pypy/branch/ooexternals/ - copied from r48251, pypy/dist/ Log: a branch to explore refactoring ideas for OO externals From antocuni at codespeak.net Fri Nov 2 16:15:45 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 2 Nov 2007 16:15:45 +0100 (CET) Subject: [pypy-svn] r48253 - pypy/dist/pypy/doc Message-ID: <20071102151545.090D580E7@code0.codespeak.net> Author: antocuni Date: Fri Nov 2 16:15:44 2007 New Revision: 48253 Modified: pypy/dist/pypy/doc/cli-backend.txt Log: add instructions about how to run pythonnet Modified: pypy/dist/pypy/doc/cli-backend.txt ============================================================================== --- pypy/dist/pypy/doc/cli-backend.txt (original) +++ pypy/dist/pypy/doc/cli-backend.txt Fri Nov 2 16:15:44 2007 @@ -430,8 +430,31 @@ chars). By contrast, most types are fine for being returned: these include all primitive types, list, tuples and instances. +Installing Python for .NET on Linux +=================================== + +With the CLI backend, you can access .NET libraries from RPython; +programs using .NET libraries will always run when translated, but you +might also want to test them on top of CPython. + +To do so, you can install `Python for .NET`_. Unfortunately, it does +not work out of the box under Linux. + +To make it working, download and unpack the source package of Python +for .NET; the only version tested with PyPy is the 1.0-rc2, but it +might work also with others. Then, you need to create a file named +Python.Runtime.dll.config at the root of the unpacked archive; put the +following lines inside the file:: + + + + + +The installation should be complete now. To run Python for .NET, +simply type ``mono python.exe``. .. _`Standard Ecma 335`: http://www.ecma-international.org/publications/standards/Ecma-335.htm .. _`flow graph`: translation.html#the-flow-model .. _`rtyper`: rtyper.html +.. _`Python for .NET`: http://pythonnet.sourceforge.net/ From pedronis at codespeak.net Fri Nov 2 16:23:32 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 2 Nov 2007 16:23:32 +0100 (CET) Subject: [pypy-svn] r48254 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071102152332.0693B81D7@code0.codespeak.net> Author: pedronis Date: Fri Nov 2 16:23:32 2007 New Revision: 48254 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: put in some stub intro slides from europython, put the jit transformation slides in the extra material part Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Fri Nov 2 16:23:32 2007 @@ -4,6 +4,35 @@ PyPy - automatic generation of VMs for dynamic languages - JIT included ======================================================================== +What is PyPy? +============== + +PyPy is: + +* A fun and challenging open source project with many contributors + +* An open source implementation of Python (written in Python) + +* A tool-chain with a *new paradigm* for building interpreters + +* Dec 2004 - March 2007: a research project funded by the EU + and companies. + +Motivation +============= + +Conventional interpreters face hard trade-offs: +*speed - maintainability - flexibility* + +A lot of decisions are hard-coded pervasively, improvements +and evolution are hampered (e.g. psyco, stackless) + +Lots of duplication of effort to cover many platforms (C, Java, .NET). + +*It should be possible to do better.* + + + Translation aspects ======================== @@ -175,6 +204,9 @@ .. demo f1 +EXTRA MATERIAL +================== + The transformation ================================== @@ -309,8 +341,6 @@ PyPy proves this a viable approach worth of further exploration. -EXTRA MATERIAL -================== .. backend material: not for the general talk .. virtualizables: not for general talk From arigo at codespeak.net Fri Nov 2 18:02:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Nov 2007 18:02:24 +0100 (CET) Subject: [pypy-svn] r48255 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071102170224.24BB481E6@code0.codespeak.net> Author: arigo Date: Fri Nov 2 18:02:22 2007 New Revision: 48255 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Shuffle shuffle. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Fri Nov 2 18:02:22 2007 @@ -4,19 +4,29 @@ PyPy - automatic generation of VMs for dynamic languages - JIT included ======================================================================== + What is PyPy? ============== -PyPy is: +PyPy is a tool-chain with a *new paradigm* for building interpreters: + +* Speed + +* Maintainability + +* Flexibility + +* ...and ease of implementation + +What is PyPy? +============== * A fun and challenging open source project with many contributors * An open source implementation of Python (written in Python) -* A tool-chain with a *new paradigm* for building interpreters - -* Dec 2004 - March 2007: a research project funded by the EU - and companies. +* Dec 2004 - March 2007: a research project funded by the European + Union and companies. Motivation ============= From jacob at codespeak.net Fri Nov 2 18:07:30 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Fri, 2 Nov 2007 18:07:30 +0100 (CET) Subject: [pypy-svn] r48256 - pypy/extradoc/talk/roadshow-google Message-ID: <20071102170730.0B60A81E6@code0.codespeak.net> Author: jacob Date: Fri Nov 2 18:07:29 2007 New Revision: 48256 Added: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp (contents, props changed) Log: Pretty pictures. Added: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp ============================================================================== Binary file. No diff available. From lac at codespeak.net Fri Nov 2 18:29:03 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Fri, 2 Nov 2007 18:29:03 +0100 (CET) Subject: [pypy-svn] r48257 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071102172903.DCABF80D5@code0.codespeak.net> Author: lac Date: Fri Nov 2 18:29:01 2007 New Revision: 48257 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: add better line, work more later Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Fri Nov 2 18:29:01 2007 @@ -10,6 +10,9 @@ PyPy is a tool-chain with a *new paradigm* for building interpreters: +It challenges the following commonly-held assumptions about the +construction of interpreters for dynamic languages. + * Speed * Maintainability From cfbolz at codespeak.net Fri Nov 2 20:14:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Nov 2007 20:14:52 +0100 (CET) Subject: [pypy-svn] r48260 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071102191452.82BF281A4@code0.codespeak.net> Author: cfbolz Date: Fri Nov 2 20:14:50 2007 New Revision: 48260 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: test and fix for NEW_METHOD primitive Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Nov 2 20:14:50 2007 @@ -35,6 +35,8 @@ # Squeak has primitives all the way up to 575 # So all optional primitives will default to the bytecode implementation prim_table = [make_failing(i) for i in range(576)] +# clean up namespace: +del i prim_table_implemented_only = [] # indicates that what is pushed is an index1, but it is unwrapped and @@ -426,8 +428,8 @@ # XXX not sure this is correct assert isinstance(w_method, model.W_CompiledMethod) w_method.literals[constants.METHOD_HEADER_INDEX] = w_header - for i in range(0,literalcount): - w_method.literals[i+1] = objtable.w_nil + for i0 in range(1, literalcount): + w_method.literals[i0] = objtable.w_nil w_method.bytes = "\x00" * bytecount return w_method Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Fri Nov 2 20:14:50 2007 @@ -129,6 +129,8 @@ return model.W_WordsObject(w_cls, extrasize) elif self.instance_kind == BYTES: return model.W_BytesObject(w_cls, extrasize) + elif self.instance_kind == COMPILED_METHOD: + return model.W_CompiledMethod(extrasize) else: raise NotImplementedError(self.instance_kind) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Nov 2 20:14:50 2007 @@ -411,6 +411,17 @@ w_v = prim(primitives.INST_VAR_AT_0, ["q"]) assert w_v.value == ord("q") +def test_new_method(): + bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) + + shadow = mockclass(0).as_class_get_shadow() + w_method = prim(primitives.NEW_METHOD, [classtable.w_CompiledMethod, len(bytecode), 1025]) + assert w_method.literals[0].value == 1025 + assert len(w_method.literals) == 2 + assert w_method.literals[1] is objtable.w_nil + assert w_method.bytes == "\x00" * len(bytecode) + + # Note: # primitives.NEXT is unimplemented as it is a performance optimization # primitives.NEXT_PUT is unimplemented as it is a performance optimization From cfbolz at codespeak.net Fri Nov 2 21:57:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Nov 2007 21:57:57 +0100 (CET) Subject: [pypy-svn] r48262 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071102205757.CBA3D81DA@code0.codespeak.net> Author: cfbolz Date: Fri Nov 2 21:57:57 2007 New Revision: 48262 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: fix test Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Fri Nov 2 21:57:57 2007 @@ -9,6 +9,7 @@ from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import shadow +from pypy.lang.smalltalk import utility # lazy initialization of test data, ie ImageReader and Float class def setup_module(module): @@ -226,7 +227,6 @@ assert w_false is objtable.w_false def test_compile_method(): - py.test.skip("not quite yet") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] @@ -240,15 +240,15 @@ if isinstance(any, str): # assume never have strings of length 1 if len(any) == 1: - return objtable.wrap_chr(any) + return utility.wrap_chr(any) else: - return objtable.wrap_string(any) + return utility.wrap_string(any) if isinstance(any, int): - return objtable.wrap_int(any) + return utility.wrap_int(any) if isinstance(any, bool): - return objtable.wrap_bool(any) + return utility.wrap_bool(any) if isinstance(any, float): - return objtable.wrap_float(any) + return utility.wrap_float(any) else: raise Exception From cfbolz at codespeak.net Fri Nov 2 21:58:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Nov 2007 21:58:23 +0100 (CET) Subject: [pypy-svn] r48263 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071102205823.7E5B481DB@code0.codespeak.net> Author: cfbolz Date: Fri Nov 2 21:58:23 2007 New Revision: 48263 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: should still be skipped Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Fri Nov 2 21:58:23 2007 @@ -227,6 +227,7 @@ assert w_false is objtable.w_false def test_compile_method(): + py.test.skip("not working") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] From lac at codespeak.net Sat Nov 3 09:06:32 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sat, 3 Nov 2007 09:06:32 +0100 (CET) Subject: [pypy-svn] r48264 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071103080632.8492D8207@code0.codespeak.net> Author: lac Date: Sat Nov 3 09:06:31 2007 New Revision: 48264 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Latest attempt at 3 slides covering 'motivation', except that it doesn't quite do that any more. This may be ok. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sat Nov 3 09:06:31 2007 @@ -10,41 +10,51 @@ PyPy is a tool-chain with a *new paradigm* for building interpreters: -It challenges the following commonly-held assumptions about the +It challenges commonly held assumptions about the construction of interpreters for dynamic languages. -* Speed +What we Reject +============== +* Interpreters for Dynamic Languages must be slow -* Maintainability +* Fast, Maintainable, Flexible -- pick one -* Flexibility +* C is the most appropriate language for implementing interpreters. -* ...and ease of implementation +* C++ is the most appropriate language for implementing interpreters. -What is PyPy? -============== +* Requiring type-hints everywhere, thus turning your dynamic language + into a static one, on the sly -* A fun and challenging open source project with many contributors +What we insist upon +=================== -* An open source implementation of Python (written in Python) +* real source code portability between different target implementations -* Dec 2004 - March 2007: a research project funded by the European - Union and companies. +* support for an actual existing dynamic language, not an academic toy -Motivation -============= +* support for the complete language, not a subset + +* The interpreter we write should be free of low-level details as possible -Conventional interpreters face hard trade-offs: -*speed - maintainability - flexibility* +* the ability to experiement with all aspects of the compiler, including + garbage collection strategies, threading models, security models + +* fun + +Implications? +============= -A lot of decisions are hard-coded pervasively, improvements -and evolution are hampered (e.g. psyco, stackless) +* type analysis must be performed on live code objects, not dead source trees -Lots of duplication of effort to cover many platforms (C, Java, .NET). +* parts of the interpreter must be written in a static manner. -*It should be possible to do better.* +* Generate compilers using partial evaluation techniques. Don't write them + by hand! +* Build a toolchain with pluggable components, for e.g. garbage collection +* The translation framework can be reused by _any_ dynamic language! Translation aspects ======================== From arigo at codespeak.net Sat Nov 3 09:18:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Nov 2007 09:18:11 +0100 (CET) Subject: [pypy-svn] r48265 - pypy/dist/pypy/translator/goal Message-ID: <20071103081811.01E2C8200@code0.codespeak.net> Author: arigo Date: Sat Nov 3 09:18:10 2007 New Revision: 48265 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: This is supposedly the official way to know if we are running on NT or Posix. Althought I cannot find right now the code in PyPy that would expose 'module/posix' under the name 'nt' I guess it must be done somewhere - otherwise tons of things would break on NT. The previous code was also untested (i.e. broken) on non-NT platforms. There are still 3 failures in test2/test_app_main.py but I don't have time to investigate now. Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Sat Nov 3 09:18:10 2007 @@ -233,25 +233,19 @@ AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' -# hack to determine if we are on windows. Maybe there is a better way... -DRIVE_LETTER_SEP = '' -try: - here = os.getcwd() - try: - os.chdir('C:') - DRIVE_LETTER_SEP = ':' - finally: - os.chdir(here) -except os.OSError: - DRIVE_LETTER_SEP = None -IS_WINDOWS = DRIVE_LETTER_SEP is not None -# XXX extend here +if 'nt' in sys.builtin_module_names: + IS_WINDOWS = True + DRIVE_LETTER_SEP = ':' +else: + IS_WINDOWS = False def entry_point(executable, argv): # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH os.setup() # this is the faked one - if os.sep not in executable and DRIVE_LETTER_SEP not in executable: + if os.sep in executable or (IS_WINDOWS and DRIVE_LETTER_SEP in executable): + pass # the path is already more than just an executable name + else: path = os.getenv('PATH') if path: for dir in path.split(os.pathsep): From pedronis at codespeak.net Sat Nov 3 09:31:47 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 3 Nov 2007 09:31:47 +0100 (CET) Subject: [pypy-svn] r48266 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071103083147.37CF38208@code0.codespeak.net> Author: pedronis Date: Sat Nov 3 09:31:46 2007 New Revision: 48266 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: some rough technical meat for the ibm talk before I forget about it (samuele, slitghtly more lucid than yesterday) Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sat Nov 3 09:31:46 2007 @@ -56,6 +56,45 @@ * The translation framework can be reused by _any_ dynamic language! +Translation details +======================= + +- RPython translation starts from loaded and initialized RPython + code as python bytecode in a Python VM. + +- PyPy uses abstract interpretation extensively: to construct flow-graphs, + for type inference, to gather information for some optimisations + +- Obviously in the PE based generated Dynamic Compilers + +- Flow-graph transformation and rewriting is also used + +Representation choice +======================== + +A complex part of RPython translation is choosing implementations +and representations of its still rich built-in types that work for +the target platforms (C/LLVM vs. OO VMs) + +We model the classes of targets through different type systems: + +- low-level: data and function pointers, structures, ... +- object oriented: classes, instances, inheritance and dispatching + +Type systems and helpers +=========================== + +We have emulation of the type systems that can run on top of CPython, +we use them for testing but also for: + +- constructing and representing the static data that our approach involves + (we start from live objects) at translation time + +- the implementation of built-in type for the targets require helper functions: + they are expressed using the emulations which our translation knows about + and can translate too + + Translation aspects ======================== From cfbolz at codespeak.net Sat Nov 3 15:07:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 15:07:23 +0100 (CET) Subject: [pypy-svn] r48267 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103140723.A6FBE8216@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 15:07:22 2007 New Revision: 48267 Added: pypy/dist/pypy/translator/backendopt/coalloc.py (contents, props changed) pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (contents, props changed) Modified: pypy/dist/pypy/translator/backendopt/escape.py Log: a so far very simplistic analysis to find potential coallocation sites Added: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 15:07:22 2007 @@ -0,0 +1,307 @@ +from pypy.annotation.model import setunion +from pypy.objspace.flow.model import Variable, Constant +from pypy.rpython.lltypesystem import lltype +from pypy.translator.simplify import get_graph +from pypy.rpython.rmodel import inputconst +from pypy.translator.backendopt import support +from pypy.tool.uid import uid + +class CreationPoint(object): + def __init__(self, creation_method, TYPE): + self.creation_method = creation_method + self.TYPE = TYPE + + def __repr__(self): + return ("CreationPoint(<0x%x>, %r)" % + (uid(self), self.TYPE)) + +class VarState(object): + def __init__(self, crep=None): + self.creation_points = {} + if crep is not None: + self.creation_points[crep] = True + self.returned = False + + def contains(self, other): + for crep in other.creation_points: + if crep not in self.creation_points: + return False + return True + + def merge(self, other): + creation_points = setunion(self.creation_points, other.creation_points) + newstate = VarState() + newstate.creation_points = creation_points + return newstate + + def __repr__(self): + crepsrepr = (", ".join([repr(crep) for crep in self.creation_points]), ) + return "VarState({%s})" % crepsrepr + +class GraphState(object): + def __init__(self, graph): + self.graph = graph + + +class AbstractDataFlowInterpreter(object): + def __init__(self, translation_context): + self.translation_context = translation_context + self.scheduled = {} # block: graph containing it + self.varstates = {} # var-or-const: state + self.creationpoints = {} # var: creationpoint + self.constant_cps = {} # const: creationpoint + self.dependencies = {} # creationpoint: {block: graph containing it} + self.functionargs = {} # graph: list of state of args + self.flown_blocks = {} # block: True + + def getstate(self, var_or_const): + if not isonheap(var_or_const): + return None + if var_or_const in self.varstates: + return self.varstates[var_or_const] + if isinstance(var_or_const, Variable): + varstate = VarState() + else: + if var_or_const not in self.constant_cps: + crep = CreationPoint("constant", var_or_const.concretetype) + self.constant_cps[var_or_const] = crep + else: + crep = self.constant_cps[var_or_const] + varstate = VarState(crep) + self.varstates[var_or_const] = varstate + return varstate + + def getstates(self, varorconstlist): + return [self.getstate(var) for var in varorconstlist] + + def setstate(self, var, state): + self.varstates[var] = state + + def get_creationpoint(self, var, method="?"): + if var in self.creationpoints: + return self.creationpoints[var] + crep = CreationPoint(method, var.concretetype) + self.creationpoints[var] = crep + return crep + + def schedule_function(self, graph): + #print "scheduling function:", graph.name + startblock = graph.startblock + if graph in self.functionargs: + args = self.functionargs[graph] + else: + args = [] + for var in startblock.inputargs: + if not isonheap(var): + varstate = None + else: + crep = self.get_creationpoint(var, "arg") + varstate = VarState(crep) + self.setstate(var, varstate) + args.append(varstate) + self.scheduled[startblock] = graph + self.functionargs[graph] = args + resultstate = self.getstate(graph.returnblock.inputargs[0]) + return resultstate, args + + def flow_block(self, block, graph): + #print "flowing in block %s of function %s" % (block, graph.name) + self.flown_blocks[block] = True + if block is graph.returnblock: + if isonheap(block.inputargs[0]): + self.getstate(block.inputargs[0]).returned = True + return + if block is graph.exceptblock: + return + self.curr_block = block + self.curr_graph = graph + #print "inputargs", self.getstates(block.inputargs) + + for op in block.operations: + self.flow_operation(op) + #print "checking exits..." + for exit in block.exits: + #print "exit", exit + args = self.getstates(exit.args) + targetargs = self.getstates(exit.target.inputargs) + #print " newargs", args + #print " targetargs", targetargs + # flow every block at least once: + if (multicontains(targetargs, args) and + exit.target in self.flown_blocks): + #print " not necessary" + continue + #else: + #print " scheduling for flowin" + for prevstate, origstate, var in zip(args, targetargs, + exit.target.inputargs): + if not isonheap(var): + continue + newstate = prevstate.merge(origstate) + self.setstate(var, newstate) + #print " args", self.getstates(exit.target.inputargs) + self.scheduled[exit.target] = graph + + def flow_operation(self, op): + #print "handling", op + args = self.getstates(op.args) + #print "args:", args + opimpl = getattr(self, 'op_'+op.opname, None) + if opimpl is not None: + res = opimpl(op, *args) + if res is not NotImplemented: + self.setstate(op.result, res) + return + + if isonheap(op.result) or filter(None, args): + raise NotImplementedError("can't handle %s" % (op.opname, )) + #print "assuming that '%s' is irrelevant" % op + + def complete(self): + while self.scheduled: + block, graph = self.scheduled.popitem() + self.flow_block(block, graph) + + def handle_changed(self, changed): + for crep in changed: + if crep not in self.dependencies: + continue + self.scheduled.update(self.dependencies[crep]) + + def register_block_dependency(self, state, block=None, graph=None): + if block is None: + block = self.curr_block + graph = self.curr_graph + for crep in state.creation_points: + self.dependencies.setdefault(crep, {})[block] = graph + + def register_state_dependency(self, state1, state2): + "state1 depends on state2: if state2 does escape/change, so does state1" + # change state1 according to how state2 is now + #print "registering dependency of %s on %s" % (state1, state2) + if state2.does_escape(): + changed = state1.setescapes() # mark all crep's as escaping + self.handle_changed(changed) + if state2.does_change(): + changed = state1.setchanges() # mark all crep's as changing + self.handle_changed(changed) + # register a dependency of the current block on state2: + # that means that if state2 changes the current block will be reflown + # triggering this function again and thus updating state1 + self.register_block_dependency(state2) + + # _____________________________________________________________________ + # operation implementations + + def op_malloc(self, op, typestate, flagsstate): + assert flagsstate is None + flags = op.args[1].value + if flags != {'flavor': 'gc'}: + return NotImplemented + return VarState(self.get_creationpoint(op.result, "malloc")) + + def op_malloc_varsize(self, op, typestate, flagsstate, lengthstate): + assert flagsstate is None + flags = op.args[1].value + if flags != {'flavor': 'gc'}: + return NotImplemented + return VarState(self.get_creationpoint(op.result, "malloc_varsize")) + + def op_keepalive(self, op, state): + return None + + def op_cast_pointer(self, op, state): + return state + + def op_getfield(self, op, objstate, fieldname): + # connectivity-wise the field within is identical to the containing + # structure + return objstate + op_getarrayitem = op_getinteriorfield = op_getfield + + def op_getarraysize(self, op, arraystate): + pass + + def op_setfield(self, op, objstate, fieldname, valuestate): + pass + op_setarrayitem = op_setinteriorfield = op_setfield + + def op_direct_call(self, op, function, *args): +# graph = get_graph(op.args[0], self.translation_context) +# if graph is None: +# for arg in args: +# if arg is None: +# continue +# # an external function can change every parameter: +# changed = arg.setchanges() +# self.handle_changed(changed) +# funcargs = [None] * len(args) +# else: +# result, funcargs = self.schedule_function(graph) +# assert len(args) == len(funcargs) +# for localarg, funcarg in zip(args, funcargs): +# if localarg is None: +# assert funcarg is None +# continue +# if funcarg is not None: +# self.register_state_dependency(localarg, funcarg) + if isonheap(op.result): + # for now assume that a call always creates a new value + return VarState(self.get_creationpoint(op.result, "direct_call")) + + def op_indirect_call(self, op, function, *args): +# graphs = op.args[-1].value +# args = args[:-1] +# if graphs is None: +# for localarg in args: +# if localarg is None: +# continue +# changed = localarg.setescapes() +# self.handle_changed(changed) +# changed = localarg.setchanges() +# self.handle_changed(changed) +# else: +# for graph in graphs: +# result, funcargs = self.schedule_function(graph) +# assert len(args) == len(funcargs) +# for localarg, funcarg in zip(args, funcargs): +# if localarg is None: +# assert funcarg is None +# continue +# self.register_state_dependency(localarg, funcarg) + if isonheap(op.result): + # for now assume that a call always creates a new value + return VarState(self.get_creationpoint(op.result, "indirect_call")) + + def op_ptr_iszero(self, op, ptrstate): + return None + + op_cast_ptr_to_int = op_keepalive = op_ptr_nonzero = op_ptr_iszero + + def op_ptr_eq(self, op, ptr1state, ptr2state): + return None + + op_ptr_ne = op_ptr_eq + + def op_same_as(self, op, objstate): + return objstate + +def isonheap(var_or_const): + return isinstance(var_or_const.concretetype, lltype.Ptr) + +def multicontains(l1, l2): + assert len(l1) == len(l2) + for a, b in zip(l1, l2): + if a is None: + assert b is None + elif not a.contains(b): + return False + return True + +def malloc_to_coalloc(t): + adi = AbstractDataFlowInterpreter(t) + for graph in t.graphs: + if graph.startblock not in adi.flown_blocks: + adi.schedule_function(graph) + adi.complete() + return adi Modified: pypy/dist/pypy/translator/backendopt/escape.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/escape.py (original) +++ pypy/dist/pypy/translator/backendopt/escape.py Sat Nov 3 15:07:22 2007 @@ -386,25 +386,26 @@ for graph in t.graphs: loop_blocks = support.find_loop_blocks(graph) for block, op in graph.iterblockops(): - if op.opname == 'malloc': - STRUCT = op.args[0].value - # must not remove mallocs of structures that have a RTTI with a destructor - try: - destr_ptr = lltype.getRuntimeTypeInfo(STRUCT)._obj.destructor_funcptr - if destr_ptr: - continue - except (ValueError, AttributeError), e: - pass - varstate = adi.getstate(op.result) - assert len(varstate.creation_points) == 1 - crep = varstate.creation_points.keys()[0] - if not crep.escapes: - if block not in loop_blocks: - print "moving object from heap to stack %s in %s" % (op, graph.name) - flags = op.args[1].value - assert flags == {'flavor': 'gc'} - op.args[1] = Constant({'flavor': 'stack'}, lltype.Void) - else: - print "%s in %s is a non-escaping malloc in a loop" % (op, graph.name) + if op.opname != 'malloc': + continue + STRUCT = op.args[0].value + # must not remove mallocs of structures that have a RTTI with a destructor + try: + destr_ptr = lltype.getRuntimeTypeInfo(STRUCT)._obj.destructor_funcptr + if destr_ptr: + continue + except (ValueError, AttributeError), e: + pass + varstate = adi.getstate(op.result) + assert len(varstate.creation_points) == 1 + crep = varstate.creation_points.keys()[0] + if not crep.escapes: + if block not in loop_blocks: + print "moving object from heap to stack %s in %s" % (op, graph.name) + flags = op.args[1].value + assert flags == {'flavor': 'gc'} + op.args[1] = Constant({'flavor': 'stack'}, lltype.Void) + else: + print "%s in %s is a non-escaping malloc in a loop" % (op, graph.name) Added: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 15:07:22 2007 @@ -0,0 +1,114 @@ +from pypy.translator.translator import TranslationContext, graphof +from pypy.translator.backendopt.coalloc import AbstractDataFlowInterpreter +from pypy.rpython.llinterp import LLInterpreter +from pypy.rlib.objectmodel import instantiate +from pypy import conftest + +import py + +def build_adi(function, types): + t = TranslationContext() + t.buildannotator().build_types(function, types) + t.buildrtyper().specialize() + if conftest.option.view: + t.view() + adi = AbstractDataFlowInterpreter(t) + graph = graphof(t, function) + adi.schedule_function(graph) + adi.complete() + return t, adi, graph + +def test_simple(): + class A(object): + pass + def f(): + a = A() + a.x = 1 + return a.x + t, adi, graph = build_adi(f, []) + avar = graph.startblock.operations[0].result + state = adi.getstate(avar) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + +def test_branch(): + class T: + pass + def fn2(x, y): + t = T() + t.x = x + t.y = y + if x > 0: + return t.x + t.y + else: + return t.x - t.y + t, adi, graph = build_adi(fn2, [int, int]) + tvar = graph.startblock.operations[0].result + state = adi.getstate(tvar) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + assert crep.creation_method == "malloc" + +def test_loop(): + class A(object): + pass + def f(): + a = A() + i = 0 + while i < 3: + a.x = i + a = A() + i += 1 + return a.x + t, adi, graph = build_adi(f, []) + avar = graph.startblock.operations[0].result + state = adi.getstate(avar) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + assert crep.creation_method == "malloc" + avarinloop = graph.startblock.exits[0].target.inputargs[1] + state1 = adi.getstate(avarinloop) + assert crep in state1.creation_points + assert len(state1.creation_points) == 2 + +def test_global(): + class A(object): + pass + globala = A() + def f(): + a = A() + a.next = None + globala.next = a + t, adi, graph = build_adi(f, []) + avar = graph.startblock.operations[0].result + state = adi.getstate(avar) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + assert crep.creation_method == "malloc" + const = graph.startblock.operations[-1].args[0] + state = adi.getstate(const) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + assert crep.creation_method == "constant" + +def test_aliasing(): + class A: + pass + def fn6(n): + a1 = A() + a1.x = 5 + a2 = A() + a2.x = 6 + if n > 0: + a = a1 + else: + a = a2 + a.x = 12 + return a1.x + t, adi, graph = build_adi(fn6, [int]) + avar = graph.startblock.exits[0].target.inputargs[1] + state = adi.getstate(avar) + assert len(state.creation_points) == 2 + for crep in state.creation_points.keys(): + assert crep.creation_method == "malloc" + From cfbolz at codespeak.net Sat Nov 3 16:16:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 16:16:13 +0100 (CET) Subject: [pypy-svn] r48268 - in pypy/dist/pypy/rpython: . lltypesystem Message-ID: <20071103151613.80B6381D1@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 16:16:11 2007 New Revision: 48268 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py Log: add coalloc support to the llinterp Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Sat Nov 3 16:16:11 2007 @@ -675,6 +675,10 @@ self.llinterpreter.remember_malloc(ptr, self) return ptr + def op_coalloc(self, obj, coallocator, flags): + # it's always safe to ignore the coallocator + return self.op_malloc(obj, flags) + # only after gc transform def op_cpy_malloc(self, obj, cpytype): # xxx return self.heap.malloc(obj, flavor='cpy', extra_args=(cpytype,)) @@ -694,6 +698,10 @@ except MemoryError: self.make_llexception() + def op_coalloc_varsize(self, obj, coallocator, flags, size): + # it's always safe to ignore the coallocator + return self.op_malloc_varsize(obj, flags, size) + def op_free(self, obj, flavor): assert isinstance(flavor, str) if flavor == 'raw' and self.llinterpreter.malloc_check: Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Sat Nov 3 16:16:11 2007 @@ -316,6 +316,8 @@ 'malloc': LLOp(canraise=(MemoryError,), canunwindgc=True), 'malloc_varsize': LLOp(canraise=(MemoryError,), canunwindgc=True), + 'coalloc': LLOp(canraise=(MemoryError,), canunwindgc=True), + 'coalloc_varsize': LLOp(canraise=(MemoryError,), canunwindgc=True), 'zero_gc_pointers_inside': LLOp(), 'free': LLOp(), 'getfield': LLOp(sideeffects=False, canrun=True), From cfbolz at codespeak.net Sat Nov 3 16:19:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 16:19:46 +0100 (CET) Subject: [pypy-svn] r48269 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103151946.6750D81D1@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 16:19:46 2007 New Revision: 48269 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: replace mallocs by coallocs Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 16:19:46 2007 @@ -7,6 +7,7 @@ from pypy.tool.uid import uid class CreationPoint(object): + constant = None def __init__(self, creation_method, TYPE): self.creation_method = creation_method self.TYPE = TYPE @@ -64,6 +65,7 @@ else: if var_or_const not in self.constant_cps: crep = CreationPoint("constant", var_or_const.concretetype) + crep.constant = var_or_const self.constant_cps[var_or_const] = crep else: crep = self.constant_cps[var_or_const] @@ -154,8 +156,7 @@ return if isonheap(op.result) or filter(None, args): - raise NotImplementedError("can't handle %s" % (op.opname, )) - #print "assuming that '%s' is irrelevant" % op + print "assuming that '%s' is irrelevant" % op def complete(self): while self.scheduled: @@ -304,4 +305,59 @@ if graph.startblock not in adi.flown_blocks: adi.schedule_function(graph) adi.complete() - return adi + look_at = t.graphs[:] + total = 0 + while look_at: + graph = look_at.pop() + for block, op in graph.iterblockops(): + if not op.opname.startswith("set"): + continue + if not isonheap(op.args[-1]): + continue + tovarstate = adi.getstate(op.args[-1]) + fromvarstate = adi.getstate(op.args[0]) + if (len(tovarstate.creation_points) != 1 or + len(fromvarstate.creation_points) != 1): + continue + fromcrep = fromvarstate.creation_points.keys()[0] + tocrep = tovarstate.creation_points.keys()[0] + if not tocrep.creation_method.startswith("malloc"): + continue + if fromcrep.creation_method.startswith("malloc"): + continue # also recently malloced + + num = do_coalloc(adi, graph, fromcrep, tocrep) + + if num: + look_at.append(graph) + print "changed %s mallocs to coallocs in %s" % (num, graph.name) + total += num + return total + + +def do_coalloc(adi, graph, fromcrep, tocrep): + result = 0 + for block, op in graph.iterblockops(): + if not op.opname.startswith("malloc"): + continue + # find coallocation var + if fromcrep.creation_method == "constant": + coallocvar = fromcrep.constant + else: + for var in block.inputargs: + varstate = adi.getstate(var) + assert len(varstate.creation_points) == 1 + crep = varstate.creation_points.keys()[0] + if crep is fromcrep: + coallocvar = var + break + else: + continue + op.opname = "coalloc" + op.opname[len("malloc"):] + op.args.insert(1, coallocvar) + mallocvarstate = adi.getstate(op.result) + assert len(mallocvarstate.creation_points) == 1 + malloccrep = mallocvarstate.creation_points.keys()[0] + malloccrep.creation_method = "coalloc" + result += 1 + return result Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 16:19:46 2007 @@ -1,5 +1,5 @@ from pypy.translator.translator import TranslationContext, graphof -from pypy.translator.backendopt.coalloc import AbstractDataFlowInterpreter +from pypy.translator.backendopt.coalloc import AbstractDataFlowInterpreter, malloc_to_coalloc from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import instantiate from pypy import conftest @@ -18,6 +18,27 @@ adi.complete() return t, adi, graph +def check_malloc_to_coalloc(function, types, args, expected_result, must_remove=-1): + t = TranslationContext() + t.buildannotator().build_types(function, types) + t.buildrtyper().specialize() + interp = LLInterpreter(t.rtyper) + graph = graphof(t, function) + res = interp.eval_graph(graph, args) + assert res == expected_result + num = malloc_to_coalloc(t) + if must_remove == -1: + for block in graph.iterblocks(): + for op in block.operations: + assert op.opname != "malloc" + else: + assert num == must_remove + t.view() + res = interp.eval_graph(graph, args) + assert res == expected_result + return t + + def test_simple(): class A(object): pass @@ -112,3 +133,41 @@ for crep in state.creation_points.keys(): assert crep.creation_method == "malloc" +def test_coalloc_constants(): + class A(object): + pass + a = A() + def f(): + n = A() + a.next = n + return 1 + check_malloc_to_coalloc(f, [], [], 1) + +def test_nocoalloc_aliasing(): + class A: + pass + def fn6(n): + a1 = A() + a1.x = 5 + a2 = A() + a2.x = 6 + if n > 0: + a = a1 + else: + a = a2 + a.x = 12 + return a1.x + t = check_malloc_to_coalloc(fn6, [int], [2], 12, must_remove=0) + +def test_coalloc_with_arg(): + class A(object): + pass + def g(b): + b.x = A() + def f(): + a = A() + g(a) + a.i = 2 + return 4 + t = check_malloc_to_coalloc(f, [], [], 4, must_remove=1) + From cfbolz at codespeak.net Sat Nov 3 18:09:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 18:09:47 +0100 (CET) Subject: [pypy-svn] r48271 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103170947.96E8E822B@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 18:09:47 2007 New Revision: 48271 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: another test, tweaks Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 18:09:47 2007 @@ -316,17 +316,18 @@ continue tovarstate = adi.getstate(op.args[-1]) fromvarstate = adi.getstate(op.args[0]) - if (len(tovarstate.creation_points) != 1 or - len(fromvarstate.creation_points) != 1): + if len(tovarstate.creation_points) != 1: continue - fromcrep = fromvarstate.creation_points.keys()[0] + fromcreps = set(fromvarstate.creation_points.keys()) tocrep = tovarstate.creation_points.keys()[0] if not tocrep.creation_method.startswith("malloc"): continue - if fromcrep.creation_method.startswith("malloc"): - continue # also recently malloced + for fromcrep in fromcreps: + if fromcrep.creation_method.startswith("malloc"): + continue # also recently malloced - num = do_coalloc(adi, graph, fromcrep, tocrep) + num = do_coalloc(adi, graph, op.args[0], block, + fromcreps, tocrep) if num: look_at.append(graph) @@ -335,24 +336,28 @@ return total -def do_coalloc(adi, graph, fromcrep, tocrep): - result = 0 - for block, op in graph.iterblockops(): - if not op.opname.startswith("malloc"): - continue - # find coallocation var - if fromcrep.creation_method == "constant": - coallocvar = fromcrep.constant - else: +def do_coalloc(adi, graph, fromvar, setblock, fromcreps, tocrep): + def find_coalloc_var(): + if block is setblock: + return fromvar + for fromcrep in fromcreps: + if fromcrep.creation_method == "constant": + return fromcrep.constant + for fromcrep in fromcreps: for var in block.inputargs: varstate = adi.getstate(var) assert len(varstate.creation_points) == 1 crep = varstate.creation_points.keys()[0] if crep is fromcrep: - coallocvar = var - break - else: - continue + return var + return None + result = 0 + for block, op in graph.iterblockops(): + if not op.opname.startswith("malloc"): + continue + coallocvar = find_coalloc_var() + if coallocvar is None: + continue op.opname = "coalloc" + op.opname[len("malloc"):] op.args.insert(1, coallocvar) mallocvarstate = adi.getstate(op.result) Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 18:09:47 2007 @@ -33,7 +33,8 @@ assert op.opname != "malloc" else: assert num == must_remove - t.view() + if conftest.option.view: + t.view() res = interp.eval_graph(graph, args) assert res == expected_result return t @@ -171,3 +172,17 @@ return 4 t = check_malloc_to_coalloc(f, [], [], 4, must_remove=1) +def test_coalloc_with_arg_several_creationpoints(): + class A(object): + pass + a1 = A() + def g(cond, b): + if cond: + b = a1 + b.x = A() + def f(cond): + a2 = A() + g(cond, a2) + return 4 + t = check_malloc_to_coalloc(f, [bool], [True], 4, must_remove=1) + From cfbolz at codespeak.net Sat Nov 3 18:11:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 18:11:13 +0100 (CET) Subject: [pypy-svn] r48272 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20071103171113.93AED822C@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 18:11:13 2007 New Revision: 48272 Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: another working test Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 18:11:13 2007 @@ -172,7 +172,7 @@ return 4 t = check_malloc_to_coalloc(f, [], [], 4, must_remove=1) -def test_coalloc_with_arg_several_creationpoints(): +def test_coalloc_with_arg_set_in_same_block(): class A(object): pass a1 = A() @@ -186,3 +186,18 @@ return 4 t = check_malloc_to_coalloc(f, [bool], [True], 4, must_remove=1) +def test_coalloc_with_arg_several_creationpoints(): + class A(object): + pass + a1 = A() + def g(cond, b): + a = A() + if cond: + b = a1 + b.x = a + def f(cond): + a2 = A() + g(cond, a2) + return 4 + t = check_malloc_to_coalloc(f, [bool], [True], 4, must_remove=1) + From tismer at codespeak.net Sat Nov 3 18:30:53 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 3 Nov 2007 18:30:53 +0100 (CET) Subject: [pypy-svn] r48273 - pypy/dist/pypy/translator/goal/test2 Message-ID: <20071103173053.C1884823C@code0.codespeak.net> Author: tismer Date: Sat Nov 3 18:30:52 2007 New Revision: 48273 Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py Log: relaxing test that depends on filenames in windows. os.abspath is incompatible with autopath right now. The real fix is to implement _getfullpathname . The provided fallback in ntpath is pretty insufficient, and a patch would probably be suitable for CPython Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/dist/pypy/translator/goal/test2/test_app_main.py Sat Nov 3 18:30:52 2007 @@ -337,5 +337,8 @@ data = self.run('-m pypy.translator.goal.test2.mymodule extra') assert 'mymodule running' in data assert 'Name: __main__' in data - assert ('File: ' + p) in data - assert ('Argv: ' + repr([p, 'extra'])) in data + # ignoring case for windows. abspath behaves different from autopath + # concerning drive letters right now + # XXX fix nt module to implement _getfullpathname + assert ('File: ' + p).lower() in data.lower() + assert ('Argv: ' + repr([p, 'extra'])).lower() in data.lower() From cfbolz at codespeak.net Sat Nov 3 19:11:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 19:11:55 +0100 (CET) Subject: [pypy-svn] r48274 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103181155.F255E823B@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 19:11:55 2007 New Revision: 48274 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: woops, check whether the malloc we are trying to replace is the correct one. Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 19:11:55 2007 @@ -326,6 +326,7 @@ if fromcrep.creation_method.startswith("malloc"): continue # also recently malloced + #import pdb; pdb.set_trace() num = do_coalloc(adi, graph, op.args[0], block, fromcreps, tocrep) @@ -355,6 +356,8 @@ for block, op in graph.iterblockops(): if not op.opname.startswith("malloc"): continue + if adi.getstate(op.result).creation_points.keys()[0] is not tocrep: + continue coallocvar = find_coalloc_var() if coallocvar is None: continue Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 19:11:55 2007 @@ -201,3 +201,28 @@ return 4 t = check_malloc_to_coalloc(f, [bool], [True], 4, must_remove=1) + +def test_coalloc_list(): + class A(object): + pass + a1 = A() + def f(count): + i = 0 + l = [] + while i < count: + l.append(A()) + i += 1 + return len(l) + t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=2) + + +def test_nocoalloc_bug(): + class A(object): + pass + a1 = A() + def f(count): + a = A() + a.length = count + a.items = A() + return a.length + t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) From tismer at codespeak.net Sat Nov 3 19:15:58 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 3 Nov 2007 19:15:58 +0100 (CET) Subject: [pypy-svn] r48275 - pypy/dist/pypy/translator/goal Message-ID: <20071103181558.0CDB581DC@code0.codespeak.net> Author: tismer Date: Sat Nov 3 19:15:58 2007 New Revision: 48275 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: some of the test cases see to have trouble with imports. "from pypy.rlib.objectmodel import we_are_translated" caused test_options_u_PYTHONINSPECT to fail. Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Sat Nov 3 19:15:58 2007 @@ -188,7 +188,11 @@ modules changeable, again. """ -from pypy.rlib.objectmodel import we_are_translated +try: + # for some reason, some tests break with this import + from pypy.rlib.objectmodel import we_are_translated +except ImportError: + we_are_translated = False class fake_os: def __init__(self): From cfbolz at codespeak.net Sat Nov 3 19:19:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 19:19:28 +0100 (CET) Subject: [pypy-svn] r48276 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103181928.9DF5381DC@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 19:19:28 2007 New Revision: 48276 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: fix two wrong tests, and the code Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 19:19:28 2007 @@ -324,16 +324,17 @@ continue for fromcrep in fromcreps: if fromcrep.creation_method.startswith("malloc"): - continue # also recently malloced + break # also recently malloced + else: + #import pdb; pdb.set_trace() + num = do_coalloc(adi, graph, op.args[0], block, + fromcreps, tocrep) - #import pdb; pdb.set_trace() - num = do_coalloc(adi, graph, op.args[0], block, - fromcreps, tocrep) - - if num: - look_at.append(graph) - print "changed %s mallocs to coallocs in %s" % (num, graph.name) - total += num + if num: + look_at.append(graph) + print "changed %s mallocs to coallocs in %s" % ( + num, graph.name) + total += num return total Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 19:19:28 2007 @@ -213,16 +213,18 @@ l.append(A()) i += 1 return len(l) - t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=2) + t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) def test_nocoalloc_bug(): class A(object): pass a1 = A() + def g(a): + a.items = A() def f(count): a = A() a.length = count - a.items = A() + g(a) return a.length t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) From cfbolz at codespeak.net Sat Nov 3 19:32:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 19:32:27 +0100 (CET) Subject: [pypy-svn] r48277 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103183227.258568232@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 19:32:25 2007 New Revision: 48277 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: some fixes necessary to make coallocation with dictionaries work. Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 19:32:25 2007 @@ -197,14 +197,14 @@ def op_malloc(self, op, typestate, flagsstate): assert flagsstate is None flags = op.args[1].value - if flags != {'flavor': 'gc'}: + if flags['flavor'] != 'gc': return NotImplemented return VarState(self.get_creationpoint(op.result, "malloc")) def op_malloc_varsize(self, op, typestate, flagsstate, lengthstate): assert flagsstate is None flags = op.args[1].value - if flags != {'flavor': 'gc'}: + if flags['flavor'] != 'gc': return NotImplemented return VarState(self.get_creationpoint(op.result, "malloc_varsize")) @@ -214,7 +214,7 @@ def op_cast_pointer(self, op, state): return state - def op_getfield(self, op, objstate, fieldname): + def op_getfield(self, op, objstate, *rest): # connectivity-wise the field within is identical to the containing # structure return objstate @@ -223,7 +223,7 @@ def op_getarraysize(self, op, arraystate): pass - def op_setfield(self, op, objstate, fieldname, valuestate): + def op_setfield(self, op, objstate, *args): pass op_setarrayitem = op_setinteriorfield = op_setfield Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 19:32:25 2007 @@ -215,6 +215,18 @@ return len(l) t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) +def test_coalloc_dict(): + class A(object): + pass + a1 = A() + def f(count): + i = 0 + d = {} + while i < count: + d[i] = A() + i += 1 + return len(d) + t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) def test_nocoalloc_bug(): class A(object): From fijal at codespeak.net Sun Nov 4 12:14:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 12:14:09 +0100 (CET) Subject: [pypy-svn] r48278 - pypy/dist/pypy/rpython/raisingops Message-ID: <20071104111409.A768C81C7@code0.codespeak.net> Author: fijal Date: Sun Nov 4 12:14:07 2007 New Revision: 48278 Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py Log: Kill tabs. Not sure what this stuff is doing here and why I cannot find tests for that. Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py ============================================================================== --- pypy/dist/pypy/rpython/raisingops/raisingops.py (original) +++ pypy/dist/pypy/rpython/raisingops/raisingops.py Sun Nov 4 12:14:07 2007 @@ -36,8 +36,8 @@ def llong_floordiv_zer(x, y): '''#define OP_LLONG_FLOORDIV_ZER(x,y,r) \ - if ((y)) { OP_LLONG_FLOORDIV(x,y,r); } \ - else FAIL_ZER("integer division") + if ((y)) { OP_LLONG_FLOORDIV(x,y,r); } \ + else FAIL_ZER("integer division") ''' if y: return llop.llong_floordiv(SignedLongLong, x, y) @@ -46,8 +46,8 @@ def ullong_floordiv_zer(x, y): '''#define OP_ULLONG_FLOORDIV_ZER(x,y,r) \ - if ((y)) { OP_ULLONG_FLOORDIV(x,y,r); } \ - else FAIL_ZER("unsigned integer division") + if ((y)) { OP_ULLONG_FLOORDIV(x,y,r); } \ + else FAIL_ZER("unsigned integer division") ''' if y: return llop.llong_floordiv(UnsignedLongLong, x, y) @@ -223,8 +223,8 @@ def llong_mod_zer(x, y): '''#define OP_LLONG_MOD_ZER(x,y,r) \ - if ((y)) { OP_LLONG_MOD(x,y,r); } \ - else FAIL_ZER("integer modulo") + if ((y)) { OP_LLONG_MOD(x,y,r); } \ + else FAIL_ZER("integer modulo") ''' if y: return llop.int_mod(SignedLongLong, x, y) From fijal at codespeak.net Sun Nov 4 12:57:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 12:57:49 +0100 (CET) Subject: [pypy-svn] r48279 - pypy/dist/pypy/interpreter Message-ID: <20071104115749.117A481D7@code0.codespeak.net> Author: fijal Date: Sun Nov 4 12:57:49 2007 New Revision: 48279 Modified: pypy/dist/pypy/interpreter/argument.py Log: Mute warnings. Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Sun Nov 4 12:57:49 2007 @@ -145,6 +145,11 @@ """ args_w, kwds_w = self.unpack() return Arguments(self.space, args_w, kwds_w) + + def unpack(self): + """ Purely abstract + """ + raise NotImplementedError() class ArgumentsPrepended(AbstractArguments): def __init__(self, args, w_firstarg): From fijal at codespeak.net Sun Nov 4 12:59:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 12:59:44 +0100 (CET) Subject: [pypy-svn] r48280 - pypy/dist/pypy/interpreter Message-ID: <20071104115944.B3D37819F@code0.codespeak.net> Author: fijal Date: Sun Nov 4 12:59:44 2007 New Revision: 48280 Modified: pypy/dist/pypy/interpreter/argument.py Log: Another purely abstract method Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Sun Nov 4 12:59:44 2007 @@ -150,6 +150,11 @@ """ Purely abstract """ raise NotImplementedError() + + def firstarg(self): + """ Purely abstract + """ + raise NotImplementedError() class ArgumentsPrepended(AbstractArguments): def __init__(self, args, w_firstarg): From fijal at codespeak.net Sun Nov 4 13:04:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 13:04:27 +0100 (CET) Subject: [pypy-svn] r48281 - pypy/dist/pypy/interpreter Message-ID: <20071104120427.0C52181C3@code0.codespeak.net> Author: fijal Date: Sun Nov 4 13:04:26 2007 New Revision: 48281 Modified: pypy/dist/pypy/interpreter/argument.py Log: Another purely-abstract methods Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Sun Nov 4 13:04:26 2007 @@ -155,7 +155,17 @@ """ Purely abstract """ raise NotImplementedError() + + def _match_signature(self, scope_w, argnames, has_vararg=False, has_kwarg=False, defaults_w=[], blindargs=0, extravarargs=None): + """ Purely abstract + """ + raise NotImplementedError() + def fixedunpack(self, argcount): + """ Purely abstract + """ + raise NotImplementedError() + class ArgumentsPrepended(AbstractArguments): def __init__(self, args, w_firstarg): self.space = args.space From lac at codespeak.net Sun Nov 4 13:12:42 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sun, 4 Nov 2007 13:12:42 +0100 (CET) Subject: [pypy-svn] r48282 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071104121242.C8CF581C4@code0.codespeak.net> Author: lac Date: Sun Nov 4 13:12:41 2007 New Revision: 48282 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Ok, try the non-challenging, non-threatening version Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sun Nov 4 13:12:41 2007 @@ -13,48 +13,43 @@ It challenges commonly held assumptions about the construction of interpreters for dynamic languages. -What we Reject -============== -* Interpreters for Dynamic Languages must be slow - -* Fast, Maintainable, Flexible -- pick one - -* C is the most appropriate language for implementing interpreters. - -* C++ is the most appropriate language for implementing interpreters. - -* Requiring type-hints everywhere, thus turning your dynamic language - into a static one, on the sly - -What we insist upon -=================== +Folk Wisdom about Interpreters for Dynamic Languages +==================================================== +* There are unavoidable tradeoffs between flexibilty, maintainability, + and speed -* real source code portability between different target implementations +* Popular dynamic languages such as Python and Ruby are the best we + can do given these tradeoffs and constraints. -* support for an actual existing dynamic language, not an academic toy - -* support for the complete language, not a subset - -* The interpreter we write should be free of low-level details as possible - -* the ability to experiement with all aspects of the compiler, including - garbage collection strategies, threading models, security models - -* fun +* Fast, Maintainable, Flexible -- pick one -Implications? -============= +What this means in Practice +=========================== -* type analysis must be performed on live code objects, not dead source trees +Dynamic languages are not very flexible. -* parts of the interpreter must be written in a static manner. +- Low-level decisions permeate the entire code base. +- One cannot simply plug-in a new garbage collector, or threading model + when one desires to experiment. +- Early decisions come back to haunt you. + +Dynamic languages are hard to maintain. + +- because they are traditionally written in low-level languages, they are + harder to write than they need be. +- the language community, which generates experts in the dynamic langauge, + can not use this expertise in its own maintenance. Instead, expertise + in C, or C++ is usually needed. +- every time a new VM is needed, there is a fork in the language community. + The maintainers of Jython and IronPython, for instance, are lost to the + C Python community. They have enough to do just to keep up with C Python. -* Generate compilers using partial evaluation techniques. Don't write them - by hand! +Dynamic Languages are Slow. -* Build a toolchain with pluggable components, for e.g. garbage collection +Thus it looks like what we really have is: +* Fast, Maintainable, Flexible -- you get NONE. -* The translation framework can be reused by _any_ dynamic language! +There has got to be a better way. Translation details ======================= From fijal at codespeak.net Sun Nov 4 14:31:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 14:31:12 +0100 (CET) Subject: [pypy-svn] r48283 - pypy/dist/pypy/translator/goal Message-ID: <20071104133112.7D43A820A@code0.codespeak.net> Author: fijal Date: Sun Nov 4 14:31:10 2007 New Revision: 48283 Added: pypy/dist/pypy/translator/goal/targetwarningalone.py (contents, props changed) Log: Strange target for rpbc experiments. Added: pypy/dist/pypy/translator/goal/targetwarningalone.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/targetwarningalone.py Sun Nov 4 14:31:10 2007 @@ -0,0 +1,38 @@ + +class Base(object): + pass + +class A(Base): + a = 1 + b = 2 + c = 3 + +class B(Base): + a = 1 + b = 2 + +class C(Base): + b = 8 + c = 6 + +def f(n): + if n > 3: + x = A + elif n > 1: + x = B + else: + x = C + if n > 0: + return x.a + return 8 + +# __________ Entry point __________ + +def entry_point(argv): + f(int(argv[0])) + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None From fijal at codespeak.net Sun Nov 4 14:40:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 14:40:04 +0100 (CET) Subject: [p