From cfbolz at codespeak.net Sat Mar 1 12:24:55 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 12:24:55 +0100 (CET) Subject: [pypy-svn] r51983 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080301112455.07A09168420@codespeak.net> Author: cfbolz Date: Sat Mar 1 12:24:55 2008 New Revision: 51983 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: some support for raising ops Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 12:24:55 2008 @@ -116,6 +116,7 @@ self.fielddescs = [] self.arrayfielddescs = [] self.interiordescs = [] + self.exceptioninstances = [] self.oopspecdescs = [] self.promotiondescs = [] self.called_bytecodes = [] @@ -146,6 +147,8 @@ self.arrayfielddesc_positions = {} # mapping (TYPE, path) to index self.interiordesc_positions = {} + # mapping exception class to index + self.exceptioninstance_positions = {} # mapping (fnobj, can_raise) to index self.oopspecdesc_positions = {} # mapping (fnobj, can_raise) to index @@ -176,6 +179,7 @@ self.fielddescs, self.arrayfielddescs, self.interiordescs, + self.exceptioninstances, self.oopspecdescs, self.promotiondescs, self.called_bytecodes, @@ -371,13 +375,17 @@ args = [] for arg in op.args: args.append(self.serialize_oparg(color, arg)) - self.serialize_opcode(color, op) + opdesc = self.serialize_opcode(color, op) self.emit(*args) if self.hannotator.binding(op.result).is_green(): self.register_greenvar(op.result) else: self.register_redvar(op.result) - + if (opdesc is not None and + opdesc.tryfold and not opdesc.canfold and opdesc.canraise): + exc_class = opdesc.llop.canraise[0] + self.emit("split_raisingop", + self.exceptioninstance_position(exc_class)) def serialize_opcode(self, color, op): opname = op.opname @@ -390,6 +398,7 @@ self.hannotator.binding(op.result), ) index = self.interpreter.make_opcode_implementation(color, opdesc) self.emit(name) + return self.interpreter.opcode_descs[index] def serialize_oparg(self, color, arg): if color == "red": @@ -504,6 +513,18 @@ self.arrayfielddesc_positions[TYPE] = result return result + def exceptioninstance_position(self, exc_class): + if exc_class in self.exceptioninstance_positions: + return self.exceptioninstance_positions[exc_class] + bk = self.rtyper.annotator.bookkeeper + exc_classdef = bk.getuniqueclassdef(exc_class) + ll_exc = self.rtyper.exceptiondata.get_standard_ll_exc_instance( + self.rtyper, exc_classdef) + result = len(self.exceptioninstances) + self.exceptioninstances.append(ll_exc) + self.exceptioninstance_positions[exc_class] = result + return result + def oopspecdesc_position(self, fnobj, canraise): key = fnobj, canraise if key in self.oopspecdesc_positions: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py Sat Mar 1 12:24:55 2008 @@ -149,6 +149,9 @@ elif argspec == "interiordesc": d = jitcode.interiordescs[src.load_2byte()] args.append(d) + elif argspec == "exception": + d = jitcode.exceptioninstances[src.load_2byte()] + args.append(d) else: assert 0, "unknown argtype declaration" Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sat Mar 1 12:24:55 2008 @@ -21,8 +21,8 @@ def __init__(self, name, code, constants, typekinds, redboxclasses, keydescs, structtypedescs, fielddescs, arrayfielddescs, - interiordescs, oopspecdescs, promotiondescs, - called_bytecodes, num_mergepoints, + interiordescs, exceptioninstances, oopspecdescs, + promotiondescs, called_bytecodes, num_mergepoints, graph_color, calldescs, indirectcalldescs, is_portal): self.name = name self.code = code @@ -34,6 +34,7 @@ self.fielddescs = fielddescs self.arrayfielddescs = arrayfielddescs self.interiordescs = interiordescs + self.exceptioninstances = exceptioninstances self.oopspecdescs = oopspecdescs self.promotiondescs = promotiondescs self.called_bytecodes = called_bytecodes @@ -137,6 +138,9 @@ elif argspec == "interiordesc": d = self.frame.bytecode.interiordescs[self.load_2byte()] args += (d, ) + elif argspec == "exception": + d = self.frame.bytecode.exceptioninstances[self.load_2byte()] + args += (d, ) else: assert 0, "unknown argtype declaration" val = func(*args) @@ -364,16 +368,16 @@ @arguments("red", "jumptarget") def opimpl_red_goto_iftrue(self, switchbox, target): # XXX not sure about passing no green vars - descision = rtimeshift.split(self.jitstate, switchbox, self.frame.pc) - if descision: + decision = rtimeshift.split(self.jitstate, switchbox, self.frame.pc) + if decision: self.frame.pc = target @arguments("bool", "red", "red", "jumptarget") def opimpl_red_goto_ifptrnonzero(self, reverse, ptrbox, switchbox, target): # XXX not sure about passing no green vars - descision = rtimeshift.split_ptr_nonzero(self.jitstate, switchbox, + decision = rtimeshift.split_ptr_nonzero(self.jitstate, switchbox, self.frame.pc, ptrbox, reverse) - if descision: + if decision: self.frame.pc = target @arguments("red", "jumptarget") @@ -381,6 +385,16 @@ if valuebox.is_constant(): self.frame.pc = target + @arguments("exception") + def opimpl_split_raisingop(self, ll_evalue): + # XXX not sure about passing no green vars + decision = rtimeshift.split_raisingop(self.jitstate, self.frame.pc, + ll_evalue) + if decision: + self.frame.pc = target + + + @arguments("jumptarget") def opimpl_goto_if_oopcall_was_virtual(self, target): if not rtimeshift.oopspec_was_residual(self.jitstate): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Mar 1 12:24:55 2008 @@ -1495,7 +1495,6 @@ def test_red_int_add_ovf(self): - py.test.skip("not working yet") def f(n, m): try: return ovfcheck(n + m) From antocuni at codespeak.net Sat Mar 1 12:37:37 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 1 Mar 2008 12:37:37 +0100 (CET) Subject: [pypy-svn] r51985 - in pypy/dist/pypy/translator: cli cli/test oosupport Message-ID: <20080301113737.8E052168460@codespeak.net> Author: antocuni Date: Sat Mar 1 12:37:37 2008 New Revision: 51985 Modified: pypy/dist/pypy/translator/cli/constant.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_class.py pypy/dist/pypy/translator/cli/test/test_dotnet.py pypy/dist/pypy/translator/oosupport/constant.py Log: add a way to cast an ootype._record to System.Object and back. Modified: pypy/dist/pypy/translator/cli/constant.py ============================================================================== --- pypy/dist/pypy/translator/cli/constant.py (original) +++ pypy/dist/pypy/translator/cli/constant.py Sat Mar 1 12:37:37 2008 @@ -84,11 +84,19 @@ type = self.cts.lltype_to_cts(EXPECTED_TYPE) gen.ilasm.opcode('castclass', type) + def _get_key_for_const(self, value): + from pypy.translator.cli.dotnet import _record_view + if isinstance(value, _record_view): + return value._record + return BaseConstantGenerator._get_key_for_const(self, value) + def _create_complex_const(self, value): - from pypy.translator.cli.dotnet import _fieldinfo + from pypy.translator.cli.dotnet import _fieldinfo, _record_view if isinstance(value, _fieldinfo): uniq = self.db.unique() return CLIFieldInfoConst(self.db, value.llvalue, uniq) + elif isinstance(value, _record_view): + return self.record_const(value._record) else: return BaseConstantGenerator._create_complex_const(self, value) Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Sat Mar 1 12:37:37 2008 @@ -465,6 +465,7 @@ return hop.genop('cliunbox', [v_obj, c_type], hop.r_result.lowleveltype) + native_exc_cache = {} def NativeException(cliClass): try: @@ -632,6 +633,71 @@ v_inst = hop.inputarg(hop.args_r[0], arg=0) return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) +class _record_view(object): + + def __init__(self, record): + self._record = record + self._TYPE = CLR.System.Object._INSTANCE + + def __ne__(self, other): + return not (self == other) + + def __eq__(self, other): + if isinstance(other, ootype._record): + return self._record == other + assert isinstance(other, _record_view) + return self._record == other._record + + def __hash__(self): + return hash(self._record) + + def __nonzero__(self): + return bool(self._record) + + +def cast_record_to_object(record): + T = ootype.typeOf(record) + assert isinstance(T, ootype.Record) + return _record_view(record) + +def cast_object_to_record(T, obj): + assert isinstance(T, ootype.Record) + assert isinstance(obj, _record_view) + record = obj._record + assert ootype.typeOf(record) == T + return record + +class Entry(ExtRegistryEntry): + _about_ = cast_record_to_object + + def compute_result_annotation(self, s_value): + T = s_value.ootype + assert isinstance(T, ootype.Record) + can_be_None = getattr(s_value, 'can_be_None', False) + return SomeOOInstance(CLR.System.Object._INSTANCE, can_be_None=can_be_None) + + def specialize_call(self, hop): + assert isinstance(hop.args_s[0], annmodel.SomeOOInstance) + v_obj, = hop.inputargs(*hop.args_r) + hop.exception_cannot_occur() + return hop.genop('ooupcast', [v_obj], hop.r_result.lowleveltype) + +class Entry(ExtRegistryEntry): + _about_ = cast_object_to_record + + def compute_result_annotation(self, s_type, s_value): + assert s_type.is_constant() + T = s_type.const + assert isinstance(T, ootype.Record) + can_be_None = getattr(s_value, 'can_be_None', False) + return SomeOOInstance(T, can_be_None) + + def specialize_call(self, hop): + assert hop.args_s[0].is_constant() + TYPE = hop.args_s[0].const + v_obj = hop.inputarg(hop.args_r[1], arg=1) + return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype) + class _fieldinfo(object): def __init__(self, llvalue): self._TYPE = CLR.System.Reflection.FieldInfo._INSTANCE Modified: pypy/dist/pypy/translator/cli/test/test_class.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_class.py (original) +++ pypy/dist/pypy/translator/cli/test/test_class.py Sat Mar 1 12:37:37 2008 @@ -2,6 +2,8 @@ from pypy.translator.cli.test.runtest import CliTest from pypy.translator.oosupport.test_template.class_ import BaseTestClass, BaseTestSpecialcase +# ====> ../../oosupport/test_template/class_.py + class TestCliClass(CliTest, BaseTestClass): pass Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Sat Mar 1 12:37:37 2008 @@ -8,7 +8,7 @@ from pypy.translator.cli.dotnet import SomeCliClass, SomeCliStaticMethod,\ NativeInstance, CLR, box, unbox, OverloadingResolver, NativeException,\ native_exc, new_array, init_array, typeof, eventhandler, clidowncast,\ - fieldinfo_for_const, classof + fieldinfo_for_const, classof, cast_record_to_object, cast_object_to_record System = CLR.System ArrayList = CLR.System.Collections.ArrayList @@ -641,6 +641,29 @@ res = self.interpret(fn, [True]) assert res == 'Int32' + def test_mix_record_and_object(self): + T = ootype.Record({'x': ootype.Signed}) + record = ootype.new(T) + def fn(flag): + if flag: + obj = cast_record_to_object(record) + else: + obj = System.Object() + record2 = cast_object_to_record(T, obj) + return record is record2 + res = self.interpret(fn, [True]) + assert res + + def test_cast_record_pbc(self): + T = ootype.Record({'x': ootype.Signed}) + record = ootype.new(T) + record.x = 42 + obj = cast_record_to_object(record) + def fn(): + record2 = cast_object_to_record(T, obj) + return record is record2 + res = self.interpret(fn, []) + assert res class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet Modified: pypy/dist/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/constant.py Sat Mar 1 12:37:37 2008 @@ -159,7 +159,10 @@ constant will be stored in the field of a singleton object. """ pass - + + def _get_key_for_const(self, value): + return value + # _________________________________________________________________ # Constant Object Creation # @@ -175,7 +178,8 @@ if value in self.cache: return self.cache[value] const = self._create_complex_const(value) - self.cache[value] = const + key = self._get_key_for_const(value) + self.cache[key] = const self._init_constant(const) const.record_dependencies() return const From cfbolz at codespeak.net Sat Mar 1 12:54:39 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 12:54:39 +0100 (CET) Subject: [pypy-svn] r51986 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080301115439.6036416843D@codespeak.net> Author: cfbolz Date: Sat Mar 1 12:54:37 2008 New Revision: 51986 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: add an assert that the merge point is in the portal Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 12:54:37 2008 @@ -223,6 +223,7 @@ cand = 0 if (op.opname == 'hint' and op.args[1].value == {'global_merge_point': True}): + assert not self.is_portal, "global_merge_point can appare only in portal" hashint = True if block is startblock or len(entrymap[block]) > 1: global_merge_blocks[block] = True Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Mar 1 12:54:37 2008 @@ -1639,11 +1639,6 @@ py.test.skip("not working yet") class MetaG: - __metaclass__ = cachedtype - - def __init__(self, hrtyper): - pass - def _freeze_(self): return True @@ -1699,7 +1694,6 @@ assert res == f(0) def test_misplaced_global_merge_point(self): - py.test.skip("not working yet") def g(n): hint(None, global_merge_point=True) return n+1 From cfbolz at codespeak.net Sat Mar 1 12:57:04 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 12:57:04 +0100 (CET) Subject: [pypy-svn] r51987 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080301115704.4899916844B@codespeak.net> Author: cfbolz Date: Sat Mar 1 12:57:02 2008 New Revision: 51987 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: this works now Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Mar 1 12:57:02 2008 @@ -981,7 +981,6 @@ self.check_insns({}) def test_compile_time_const_tuple(self): - py.test.skip("no clue what's wrong") d = {(4, 5): 42, (6, 7): 12} def f(a, b): d1 = hint(d, deepfreeze=True) From antocuni at codespeak.net Sat Mar 1 13:34:37 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 1 Mar 2008 13:34:37 +0100 (CET) Subject: [pypy-svn] r51990 - pypy/extradoc/proposal Message-ID: <20080301123437.AD27A16846A@codespeak.net> Author: antocuni Date: Sat Mar 1 13:34:37 2008 New Revision: 51990 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: reword a paragraph Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 13:34:37 2008 @@ -41,7 +41,7 @@ Moreover, there is an experimental JIT backend that emits code for the CLI; it is still a work in progress and very incomplete, but it shows -that the is possible to adapt the PyPy JIT to emit code for object +that it is possible to adapt the PyPy JIT to emit code for object oriented virtual machines. @@ -126,11 +126,11 @@ compiled with `gcc -O0`. Making the Python interpreter to exploit the full potential of the JIT -is a separate task and it is out of the scope of this proposal; it is -important to underline that once the JVM backend for the JIT is -complete, the resulting pypy-jvm will automatically take advantage of -all the optimizations written for the other backends. XXX expand -this by one sentence because this is unclear as written +is a separate task and since it requires some small changes to the +interpreter, it is out of the scope of this proposal. It is important +to underline that once the PyPy interpreter is fully optimized for the +JIT, PyPy for the JVM will automatically take advantage of these speed +ups, without needing to change the JIT backend for the JVM. We also expect to find benchmarks in which the JIT that targets the MLVM will perform better than the JIT that targets the plain JVM, From cfbolz at codespeak.net Sat Mar 1 13:50:27 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 13:50:27 +0100 (CET) Subject: [pypy-svn] r51992 - pypy/extradoc/proposal Message-ID: <20080301125027.1F8A1168020@codespeak.net> Author: cfbolz Date: Sat Mar 1 13:50:21 2008 New Revision: 51992 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: fix a few things, add two XXXs Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 13:50:21 2008 @@ -26,11 +26,11 @@ such as garbage collection, microthreading (like `Stackless Python`_), etc. -The most exciting feature of the TT is the ability to automatically -turn the interpreter into a JIT compiler that exploits partial -evaluation techniques to dynamically generate efficient code. The +The most exciting feature of the TT is the ability apply partial evaluation +techniques to automatically turn the interpreter into a JIT compiler which +generates efficient code dynamically. The (XXX not so novel: hotspot does it too) novel idea behind PyPy JIT is to delay the compilation until we know -all the informations useful for emitting optimized code, thus +all the information useful for emitting optimized code, thus being potentially much more efficient than all the current other alternatives (see the "Related Work" section). @@ -109,7 +109,7 @@ Project completion ------------------ -PyPy JIT is still under heavy development; potentially, the resulting +The PyPy JIT is still under heavy development; potentially, the resulting JIT compiler will be able to optimize a large number of Python programs, but at the moment it gives the best results only with computational intensive functions that use only operations between @@ -125,11 +125,12 @@ JIT support, it runs roughly at the same speed as its C equivalent compiled with `gcc -O0`. -Making the Python interpreter to exploit the full potential of the JIT -is a separate task and since it requires some small changes to the +Supporting and thus speeding up more parts of the Python language +is a separate task and since it requires changes to the interpreter, it is out of the scope of this proposal. It is important -to underline that once the PyPy interpreter is fully optimized for the -JIT, PyPy for the JVM will automatically take advantage of these speed +to underline that this work is independent from the backend being used, +so once the PyPy interpreter is fully optimized for the +JIT, PyPy for the JVM will automatically take advantage of these improvements ups, without needing to change the JIT backend for the JVM. We also expect to find benchmarks in which the JIT that targets the @@ -145,11 +146,13 @@ languages which run on top of the JVM. Even if currently Jython_ is the only usable implementation of Python for the JVM, PyPy has the potential to become the reference implementation in the future. +(XXX careful: this enters politics country, I guess we don't want to piss of the +Jython people) To have a working JIT for the JVM is an important step towards making PyPy the fastest Python for the JVM, ever. Moreover, due to the innovative ideas implemented by PyPy, it is likely that Python could become -the fastest dynamic language that runs on the top of the JVM. +the fastest dynamic language of its class that runs on the top of the JVM. Finally, PyPy is not limited to Python: it is entirely possible to write interpreters for languages other than Python and translate them @@ -160,7 +163,7 @@ Since the JIT generator is independent of the Python languages, it will be possible to automatically add a JIT compiler to every language written using the PyPy TT; thus, PyPy could become a very attractive -platform to develop dynamic languages for the JVM. +environment to develop dynamic languages for the JVM. Dependencies on Sun @@ -187,7 +190,7 @@ really works ahead of time (AOT), because the code is fully emitted before the program starts, and it doesn't exploit additional informations that would be available only at runtime - (e.g., informations about the types that each variable can + (most importantly informations about the types that each variable can assume); - JRuby supports interpretation, AOT compilation and JIT @@ -212,7 +215,7 @@ at the same speed of manually written Java code. Moreover, the JIT compiler is automatically generated by the TT: we -believe, based on previous experiences as Psyco_, that manually +believe, based on previous experiences with Psyco_, that manually writing a JIT compiler of that kind is hard and error prone, especially when the source language is as complex as Python; by writing a JIT compiler generator, we get JIT compilers that are From cfbolz at codespeak.net Sat Mar 1 14:03:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 14:03:09 +0100 (CET) Subject: [pypy-svn] r51995 - pypy/extradoc/proposal Message-ID: <20080301130309.F10B2168487@codespeak.net> Author: cfbolz Date: Sat Mar 1 14:03:09 2008 New Revision: 51995 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: add myself Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 14:03:09 2008 @@ -223,18 +223,22 @@ free. -Developer ---------- +Developers +---------- Antonio Cuni is one of the core developers of PyPy; he is the main author of the CLI backend, and the coauthor of the JVM backend; -recently, it began working on the experimental CLI backend for the +recently, he began working on the experimental CLI backend for the JIT. Currently, he is a PhD student at Univerist? degli Studi di Genova, doing research in the area of implementation of dynamic languages on top of object oriented virtual machines. +Carl Friedrich Bolz also is a core developer of PyPy. He worked on various areas +including the garbage collectors, optimizations and the Python interpreter. He +is currently doing a Master's thesis at the Heinrich-Heine-Universit?t +D?sseldorf about improving PyPy's JIT compiler generator. .. _PyPy: http://codespeak.net/pypy .. _RPython: http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#rpython From jacob at codespeak.net Sat Mar 1 14:05:58 2008 From: jacob at codespeak.net (jacob at codespeak.net) Date: Sat, 1 Mar 2008 14:05:58 +0100 (CET) Subject: [pypy-svn] r51996 - pypy/extradoc/proposal Message-ID: <20080301130558.BEA351684C2@codespeak.net> Author: jacob Date: Sat Mar 1 14:05:57 2008 New Revision: 51996 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: Language tweaks. Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 14:05:57 2008 @@ -76,7 +76,7 @@ are not limited to: dynamic invocation, lightweight bytecode loading, tail calls, etc. -Implementation wise, the JIT backends for the plain JVM and for the +Implementation-wise, the JIT backends for the plain JVM and for the MLVM could share most of the code, with the latter making use of the special features when needed. @@ -150,7 +150,7 @@ Jython people) To have a working JIT for the JVM is an important step towards making PyPy -the fastest Python for the JVM, ever. Moreover, due to the innovative +the fastest Python for the JVM. Moreover, due to the innovative ideas implemented by PyPy, it is likely that Python could become the fastest dynamic language of its class that runs on the top of the JVM. @@ -228,7 +228,7 @@ Antonio Cuni is one of the core developers of PyPy; he is the main author of the CLI backend, and the coauthor of the JVM backend; -recently, he began working on the experimental CLI backend for the +recently, he began working on an experimental CLI backend for the JIT. Currently, he is a PhD student at Univerist? degli Studi di Genova, From cfbolz at codespeak.net Sat Mar 1 14:22:13 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 14:22:13 +0100 (CET) Subject: [pypy-svn] r51998 - pypy/extradoc/proposal Message-ID: <20080301132213.A5D2F16843D@codespeak.net> Author: cfbolz Date: Sat Mar 1 14:22:13 2008 New Revision: 51998 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: kill political paragraph Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 14:22:13 2008 @@ -142,13 +142,6 @@ Relevance to the community -------------------------- -Recently the community has shown a lot of interest in dynamic -languages which run on top of the JVM. Even if currently Jython_ is -the only usable implementation of Python for the JVM, PyPy has the -potential to become the reference implementation in the future. -(XXX careful: this enters politics country, I guess we don't want to piss of the -Jython people) - To have a working JIT for the JVM is an important step towards making PyPy the fastest Python for the JVM. Moreover, due to the innovative ideas implemented by PyPy, it is likely that Python could become From cfbolz at codespeak.net Sat Mar 1 16:18:27 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 16:18:27 +0100 (CET) Subject: [pypy-svn] r52002 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080301151827.12CB71684E2@codespeak.net> Author: cfbolz Date: Sat Mar 1 16:18:26 2008 New Revision: 52002 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: implement switch handling Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 16:18:26 2008 @@ -1,3 +1,4 @@ +import py from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import we_are_translated from pypy.objspace.flow import model as flowmodel @@ -10,6 +11,8 @@ from pypy.jit.timeshifter.greenkey import KeyDesc from pypy.jit.rainbow.interpreter import JitCode, JitInterpreter from pypy.translator.backendopt.removenoops import remove_same_as +from pypy.translator.backendopt.ssa import SSA_to_SSI +from pypy.translator.unsimplify import varoftype class CallDesc: @@ -96,12 +99,14 @@ self.unfinished_graphs = [] self.num_global_mergepoints = 0 self.ptr_to_jitcode = {} + self.transformer = GraphTransformer(hannotator) def can_raise(self, op): return self.raise_analyzer.analyze(op) def make_bytecode(self, graph, is_portal=True): - remove_same_as(graph) + self.transformer.transform_graph(graph) + #graph.show() if is_portal: bytecode = JitCode.__new__(JitCode) bytecode.is_portal = True @@ -323,7 +328,30 @@ self.emit(*truerenaming) self.make_bytecode_block(linktrue.target, insert_goto=True) else: - XXX + assert self.varcolor(block.exitswitch) == "green" + for link in block.exits: + if link.exitcase == 'default': + defaultlink = link + switchlinks = [link for link in block.exits + if link is not defaultlink] + + renamings = [self.insert_renaming(link) for link in switchlinks] + defaultrenaming = self.insert_renaming(defaultlink) + cases = [flowmodel.Constant(link.exitcase, + block.exitswitch.concretetype) + for link in switchlinks] + cases = [self.serialize_oparg("green", case) for case in cases] + targets = [tlabel(link) for link in switchlinks] + self.emit("green_switch") + self.emit(self.serialize_oparg("green", block.exitswitch)) + self.emit(len(cases), *cases) + self.emit(len(targets), *targets) + self.emit(*defaultrenaming) + self.make_bytecode_block(defaultlink.target, insert_goto=True) + for renaming, link in zip(renamings, switchlinks): + self.emit(label(link)) + self.emit(*renaming) + self.make_bytecode_block(link.target, insert_goto=True) def insert_merges(self, block): if block is self.graph.returnblock: @@ -1174,6 +1202,158 @@ return c +class GraphTransformer(object): + def __init__(self, hannotator): + self.hannotator = hannotator + + def transform_graph(self, graph): + self.graph = graph + remove_same_as(graph) + self.insert_splits() + + def insert_splits(self): + hannotator = self.hannotator + for block in list(self.graph.iterblocks()): + if block.exitswitch is not None: + assert isinstance(block.exitswitch, flowmodel.Variable) + hs_switch = hannotator.binding(block.exitswitch) + if not hs_switch.is_green(): + if block.exitswitch.concretetype is not lltype.Bool: + self.insert_switch_handling(block) + + def insert_switch_handling(self, block): + v_redswitch = block.exitswitch + T = v_redswitch.concretetype + range_start = -py.std.sys.maxint-1 + range_stop = py.std.sys.maxint+1 + if T is not lltype.Signed: + if T is lltype.Char: + opcast = 'cast_char_to_int' + range_start = 0 + range_stop = 256 + elif T is lltype.UniChar: + opcast = 'cast_unichar_to_int' + range_start = 0 + elif T is lltype.Unsigned: + opcast = 'cast_uint_to_int' + else: + raise AssertionError(T) + v_redswitch = self.genop(block, opcast, [v_redswitch], + resulttype=lltype.Signed, red=True) + block.exitswitch = v_redswitch + # for now, we always turn the switch back into a chain of tests + # that perform a binary search + blockset = {block: True} # reachable from outside + cases = {} + defaultlink = None + for link in block.exits: + if link.exitcase == 'default': + defaultlink = link + blockset[link.target] = False # not reachable from outside + else: + assert lltype.typeOf(link.exitcase) == T + intval = lltype.cast_primitive(lltype.Signed, link.exitcase) + cases[intval] = link + link.exitcase = None + link.llexitcase = None + self.insert_integer_search(block, cases, defaultlink, blockset, + range_start, range_stop) + SSA_to_SSI(blockset, self.hannotator) + + def insert_integer_search(self, block, cases, defaultlink, blockset, + range_start, range_stop): + # fix the exit of the 'block' to check for the given remaining + # 'cases', knowing that if we get there then the value must + # be contained in range(range_start, range_stop). + if not cases: + assert defaultlink is not None + block.exitswitch = None + block.recloseblock(flowmodel.Link(defaultlink.args, defaultlink.target)) + elif len(cases) == 1 and (defaultlink is None or + range_start == range_stop-1): + block.exitswitch = None + block.recloseblock(cases.values()[0]) + else: + intvalues = cases.keys() + intvalues.sort() + if len(intvalues) <= 3: + # not much point in being clever with no more than 3 cases + intval = intvalues[-1] + remainingcases = cases.copy() + link = remainingcases.pop(intval) + c_intval = flowmodel.Constant(intval, lltype.Signed) + v = self.genop(block, 'int_eq', [block.exitswitch, c_intval], + resulttype=lltype.Bool, red=True) + link.exitcase = True + link.llexitcase = True + falseblock = flowmodel.Block([]) + falseblock.exitswitch = block.exitswitch + blockset[falseblock] = False + falselink = flowmodel.Link([], falseblock) + falselink.exitcase = False + falselink.llexitcase = False + block.exitswitch = v + block.recloseblock(falselink, link) + if defaultlink is None or intval == range_stop-1: + range_stop = intval + self.insert_integer_search(falseblock, remainingcases, + defaultlink, blockset, + range_start, range_stop) + else: + intval = intvalues[len(intvalues) // 2] + c_intval = flowmodel.Constant(intval, lltype.Signed) + v = self.genop(block, 'int_ge', [block.exitswitch, c_intval], + resulttype=lltype.Bool, red=True) + falseblock = flowmodel.Block([]) + falseblock.exitswitch = block.exitswitch + trueblock = flowmodel.Block([]) + trueblock.exitswitch = block.exitswitch + blockset[falseblock] = False + blockset[trueblock] = False + falselink = flowmodel.Link([], falseblock) + falselink.exitcase = False + falselink.llexitcase = False + truelink = flowmodel.Link([], trueblock) + truelink.exitcase = True + truelink.llexitcase = True + block.exitswitch = v + block.recloseblock(falselink, truelink) + falsecases = {} + truecases = {} + for intval1, link1 in cases.items(): + if intval1 < intval: + falsecases[intval1] = link1 + else: + truecases[intval1] = link1 + self.insert_integer_search(falseblock, falsecases, + defaultlink, blockset, + range_start, intval) + self.insert_integer_search(trueblock, truecases, + defaultlink, blockset, + intval, range_stop) + + def genop(self, block, opname, args, resulttype=None, result_like=None, red=False): + # 'result_like' can be a template variable whose hintannotation is + # copied + if resulttype is not None: + v_res = varoftype(resulttype) + if red: + hs = hintmodel.SomeLLAbstractVariable(resulttype) + else: + hs = hintmodel.SomeLLAbstractConstant(resulttype, {}) + self.hannotator.setbinding(v_res, hs) + elif result_like is not None: + v_res = copyvar(self.hannotator, result_like) + else: + v_res = self.new_void_var() + + spaceop = flowmodel.SpaceOperation(opname, args, v_res) + if isinstance(block, list): + block.append(spaceop) + else: + block.operations.append(spaceop) + return v_res + class label(object): def __init__(self, name): self.name = name Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py Sat Mar 1 16:18:26 2008 @@ -105,6 +105,9 @@ args.append(jitcode.typekinds[src.load_2byte()]) elif argspec == "jumptarget": args.append(src.load_jumptarget()) + elif argspec == "jumptargets": + num = src.load_2byte() + args.append([src.load_jumptarget() for i in range(num)], ) elif argspec == "bool": args.append(src.load_bool()) elif argspec == "redboxcls": Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sat Mar 1 16:18:26 2008 @@ -95,6 +95,9 @@ args += (self.frame.bytecode.typekinds[self.load_2byte()], ) elif argspec == "jumptarget": args += (self.load_4byte(), ) + elif argspec == "jumptargets": + num = self.load_2byte() + args += ([self.load_4byte() for i in range(num)], ) elif argspec == "bool": args += (self.load_bool(), ) elif argspec == "redboxcls": @@ -365,6 +368,16 @@ if arg: self.frame.pc = target + @arguments("green", "green_varargs", "jumptargets") + def opimpl_green_switch(self, exitcase, cases, targets): + arg = exitcase.revealconst(lltype.Signed) + assert len(cases) == len(targets) + for i in range(len(cases)): + if arg == cases[i].revealconst(lltype.Signed): + self.frame.pc = targets[i] + break + + @arguments("red", "jumptarget") def opimpl_red_goto_iftrue(self, switchbox, target): # XXX not sure about passing no green vars Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Mar 1 16:18:26 2008 @@ -1588,21 +1588,20 @@ assert res == -7 def test_switch(self): - py.test.skip("not working yet") - def g(n): + def g(n, x): if n == 0: - return 12 + return 12 + x elif n == 1: - return 34 + return 34 + x elif n == 3: - return 56 + return 56 + x elif n == 7: - return 78 + return 78 + x else: - return 90 + return 90 + x def f(n, m): - x = g(n) # gives a red switch - y = g(hint(m, concrete=True)) # gives a green switch + x = g(n, n) # gives a red switch + y = g(hint(m, concrete=True), n) # gives a green switch return x - y res = self.interpret(f, [7, 2], backendoptimize=True) @@ -1611,22 +1610,21 @@ assert res == 90 - 34 def test_switch_char(self): - py.test.skip("not working yet") - def g(n): + def g(n, x): n = chr(n) if n == '\x00': - return 12 + return 12 + x elif n == '\x01': - return 34 + return 34 + x elif n == '\x02': - return 56 + return 56 + x elif n == '\x03': - return 78 + return 78 + x else: - return 90 + return 90 + x def f(n, m): - x = g(n) # gives a red switch - y = g(hint(m, concrete=True)) # gives a green switch + x = g(n, n) # gives a red switch + y = g(hint(m, concrete=True), n) # gives a green switch return x - y res = self.interpret(f, [3, 0], backendoptimize=True) From cfbolz at codespeak.net Sat Mar 1 16:23:12 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 16:23:12 +0100 (CET) Subject: [pypy-svn] r52003 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080301152312.6CF621684E2@codespeak.net> Author: cfbolz Date: Sat Mar 1 16:23:11 2008 New Revision: 52003 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Log: nonsense assertion Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 16:23:11 2008 @@ -228,7 +228,7 @@ cand = 0 if (op.opname == 'hint' and op.args[1].value == {'global_merge_point': True}): - assert not self.is_portal, "global_merge_point can appare only in portal" + assert self.is_portal, "global_merge_point can appare only in portal" hashint = True if block is startblock or len(entrymap[block]) > 1: global_merge_blocks[block] = True From antocuni at codespeak.net Sat Mar 1 16:46:23 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 1 Mar 2008 16:46:23 +0100 (CET) Subject: [pypy-svn] r52004 - pypy/extradoc/proposal Message-ID: <20080301154623.4B72C1684DF@codespeak.net> Author: antocuni Date: Sat Mar 1 16:46:20 2008 New Revision: 52004 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: fix the last XXX. Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 16:46:20 2008 @@ -26,13 +26,13 @@ such as garbage collection, microthreading (like `Stackless Python`_), etc. -The most exciting feature of the TT is the ability apply partial evaluation -techniques to automatically turn the interpreter into a JIT compiler which -generates efficient code dynamically. The (XXX not so novel: hotspot does it too) -novel idea behind PyPy JIT is to delay the compilation until we know -all the information useful for emitting optimized code, thus -being potentially much more efficient than all the current other -alternatives (see the "Related Work" section). +The most exciting feature of the TT is the ability apply partial +evaluation techniques to automatically turn the interpreter into a JIT +compiler which generates efficient code dynamically. The key idea +behind PyPy JIT is to delay the compilation until we know all the +information useful for emitting optimized code, thus being potentially +much more efficient than all the current other alternatives (see the +"Related Work" section). Currently, the PyPy JIT works only in conjunction with the C backend. Early results are very good. The resulting Python interpreter @@ -131,7 +131,7 @@ to underline that this work is independent from the backend being used, so once the PyPy interpreter is fully optimized for the JIT, PyPy for the JVM will automatically take advantage of these improvements -ups, without needing to change the JIT backend for the JVM. +without needing to change the JIT backend for the JVM. We also expect to find benchmarks in which the JIT that targets the MLVM will perform better than the JIT that targets the plain JVM, @@ -222,9 +222,7 @@ Antonio Cuni is one of the core developers of PyPy; he is the main author of the CLI backend, and the coauthor of the JVM backend; recently, he began working on an experimental CLI backend for the -JIT. - -Currently, he is a PhD student at Univerist? degli Studi di Genova, +JIT. Currently, he is a PhD student at Univerist? degli Studi di Genova, doing research in the area of implementation of dynamic languages on top of object oriented virtual machines. From cfbolz at codespeak.net Sat Mar 1 16:55:00 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 16:55:00 +0100 (CET) Subject: [pypy-svn] r52005 - pypy/extradoc/proposal Message-ID: <20080301155500.035A11684E2@codespeak.net> Author: cfbolz Date: Sat Mar 1 16:55:00 2008 New Revision: 52005 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: add another sentence, fix english. still not quite happy about it, but probably fine. Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 16:55:00 2008 @@ -26,13 +26,14 @@ such as garbage collection, microthreading (like `Stackless Python`_), etc. -The most exciting feature of the TT is the ability apply partial +The most exciting feature of the TT is the ability to apply partial evaluation techniques to automatically turn the interpreter into a JIT compiler which generates efficient code dynamically. The key idea -behind PyPy JIT is to delay the compilation until we know all the +behind the PyPy JIT is to systematically delay compilation until we know all the information useful for emitting optimized code, thus being potentially much more efficient than all the current other alternatives (see the -"Related Work" section). +"Related Work" section). This is done using a mechanism which can be seen as a +generalized version of polymorphic inline caches. Currently, the PyPy JIT works only in conjunction with the C backend. Early results are very good. The resulting Python interpreter From cfbolz at codespeak.net Sat Mar 1 18:02:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 18:02:09 +0100 (CET) Subject: [pypy-svn] r52008 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080301170209.4C5F21684EB@codespeak.net> Author: cfbolz Date: Sat Mar 1 18:02:07 2008 New Revision: 52008 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: Enable the lazy exception path. This requires the insertion of some global_merge_point hints and a hack around the residual call handling. One test is going into an infinite loop, investigating. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 18:02:07 2008 @@ -90,7 +90,7 @@ etrafo = hannotator.exceptiontransformer type_system = self.rtyper.type_system.name self.exceptiondesc = exception.ExceptionDesc( - RGenOp, etrafo, type_system, False) + RGenOp, etrafo, type_system, True) self.interpreter = JitInterpreter(self.exceptiondesc, RGenOp) self.RGenOp = RGenOp self.current_block = None @@ -821,7 +821,6 @@ self.emit("goto_if_oopcall_was_virtual", tlabel(("oop_call", op))) self.emit("after_oop_residual_call") self.emit(self.promotiondesc_position(lltype.Signed)) - self.emit(label(("oop_call", op))) def handle_green_call(self, op, withexc): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sat Mar 1 18:02:07 2008 @@ -601,7 +601,13 @@ check_forced = False flagbox = rtimeshift.after_residual_call(self.jitstate, exceptiondesc, check_forced) - done = rtimeshift.promote(self.jitstate, flagbox, promotiondesc) + # XXX slightly hackish: the flagbox needs to be in local_boxes + # to be passed along to the new block + self.frame.local_boxes.append(flagbox) + try: + done = rtimeshift.promote(self.jitstate, flagbox, promotiondesc) + finally: + self.frame.local_boxes.pop() if done: return self.dispatch() gv_flag = flagbox.getgenvar(self.jitstate) @@ -623,7 +629,13 @@ exceptiondesc = None flagbox = rtimeshift.after_residual_call(self.jitstate, exceptiondesc, True) - done = rtimeshift.promote(self.jitstate, flagbox, promotiondesc) + # XXX slightly hackish: the flagbox needs to be in local_boxes + # to be passed along to the new block + self.frame.local_boxes.append(flagbox) + try: + done = rtimeshift.promote(self.jitstate, flagbox, promotiondesc) + finally: + self.frame.local_boxes.pop() if done: return self.dispatch() gv_flag = flagbox.getgenvar(self.jitstate) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Mar 1 18:02:07 2008 @@ -1014,6 +1014,7 @@ return 2*h(x) def f(x): + hint(None, global_merge_point=True) try: return g(x) except ValueError: @@ -1260,6 +1261,7 @@ else: return -12 + y def f(x, y): + hint(None, global_merge_point=True) x = hint(x, concrete=True) if x == 1: return g(lltype.nullptr(S), y) @@ -1312,6 +1314,7 @@ ll_assert(bool(s), "please don't give me a null") return 5 def f(m): + hint(None, global_merge_point=True) s = h() n = g(s) if not s: @@ -1354,6 +1357,7 @@ return h(n) + x def f(n, x): + hint(None, global_merge_point=True) try: return g(n, x) except ValueError: From cfbolz at codespeak.net Sat Mar 1 18:42:45 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 18:42:45 +0100 (CET) Subject: [pypy-svn] r52009 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080301174245.868D016842A@codespeak.net> Author: cfbolz Date: Sat Mar 1 18:42:44 2008 New Revision: 52009 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py Log: support for green indirect calls Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 18:42:44 2008 @@ -80,6 +80,7 @@ self.graphs = [graph for (graph, tsgraph) in graph2tsgraph] self.jitcodes = values + self.calldesc = CallDesc(codewriter.RGenOp, lltype.typeOf(fnptr)) class BytecodeWriter(object): @@ -733,6 +734,8 @@ def serialize_op_indirect_call(self, op): kind, withexc = self.guess_call_kind(op) + if kind == "green": + return self.handle_green_call(op, withexc, exclude_last=True) targets = dict(self.graphs_from(op)) fnptrindex = self.serialize_oparg("red", op.args[0]) has_result = (self.varcolor(op.result) != "gray" and @@ -772,10 +775,11 @@ elif kind == "gray": self.emit("red_after_direct_call") else: - XXX + assert 0, "unknown call kind %s" % (kind, ) self.emit(label(("after indirect call", op))) + def handle_oopspec_call(self, op, withexc): from pypy.jit.timeshifter.oop import Index fnobj = op.args[0].value._obj @@ -823,17 +827,21 @@ self.emit(self.promotiondesc_position(lltype.Signed)) self.emit(label(("oop_call", op))) - def handle_green_call(self, op, withexc): - voidargs = [const.value for const in op.args[1:] + def handle_green_call(self, op, withexc, exclude_last=False): + if exclude_last: + args = op.args[1:-1] + else: + args = op.args[1:] + voidargs = [const.value for const in args if const.concretetype == lltype.Void] fnptr = op.args[0] - pos = self.calldesc_position(lltype.typeOf(fnptr.value), *voidargs) + pos = self.calldesc_position(fnptr.concretetype, *voidargs) func = self.serialize_oparg("green", fnptr) emitted_args = [] for v in op.args[1:]: if v.concretetype != lltype.Void: emitted_args.append(self.serialize_oparg("green", v)) - self.emit("green_direct_call") + self.emit("green_call") self.emit(func, pos) self.emit(len(emitted_args)) self.emit(*emitted_args) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sat Mar 1 18:42:44 2008 @@ -525,7 +525,7 @@ self.portalstate.portal_reentry(greenargs, redargs) @arguments("green", "calldesc", "green_varargs") - def opimpl_green_direct_call(self, fnptr_gv, calldesc, greenargs): + def opimpl_green_call(self, fnptr_gv, calldesc, greenargs): calldesc.green_call(self, fnptr_gv, greenargs) @arguments("green_varargs", "red_varargs", "bytecode") Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Mar 1 18:42:44 2008 @@ -1445,7 +1445,6 @@ self.check_insns({'direct_call': 1}) def test_indirect_sometimes_residual_pure_but_fixed_red_call(self): - py.test.skip("not working yet") def h1(x): return x-2 def h2(x): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py Sat Mar 1 18:42:44 2008 @@ -144,7 +144,6 @@ self.check_insns({}) def test_frozen_list_indexerror(self): - py.test.skip("implement me") lst = [5, 7, 9] def ll_function(x): mylist = hint(lst, deepfreeze=True) From antocuni at codespeak.net Sat Mar 1 19:01:36 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 1 Mar 2008 19:01:36 +0100 (CET) Subject: [pypy-svn] r52010 - pypy/extradoc/proposal Message-ID: <20080301180136.85C981684E2@codespeak.net> Author: antocuni Date: Sat Mar 1 19:01:35 2008 New Revision: 52010 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: add email addresses Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 19:01:35 2008 @@ -220,17 +220,18 @@ Developers ---------- -Antonio Cuni is one of the core developers of PyPy; he is the main -author of the CLI backend, and the coauthor of the JVM backend; -recently, he began working on an experimental CLI backend for the -JIT. Currently, he is a PhD student at Univerist? degli Studi di Genova, -doing research in the area of implementation of dynamic languages on -top of object oriented virtual machines. +Antonio Cuni (anto.cuni at gmail.com) is one of the core developers of +PyPy; he is the main author of the CLI backend, and the coauthor of +the JVM backend; recently, he began working on an experimental CLI +backend for the JIT. Currently, he is a PhD student at Univerist? +degli Studi di Genova, doing research in the area of implementation of +dynamic languages on top of object oriented virtual machines. -Carl Friedrich Bolz also is a core developer of PyPy. He worked on various areas -including the garbage collectors, optimizations and the Python interpreter. He -is currently doing a Master's thesis at the Heinrich-Heine-Universit?t -D?sseldorf about improving PyPy's JIT compiler generator. +Carl Friedrich Bolz (cfbolz at gmx.de) also is a core developer of +PyPy. He worked on various areas including the garbage collectors, +optimizations and the Python interpreter. He is currently doing a +Master's thesis at the Heinrich-Heine-Universit?t D?sseldorf about +improving PyPy's JIT compiler generator. .. _PyPy: http://codespeak.net/pypy .. _RPython: http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#rpython From arigo at codespeak.net Sat Mar 1 19:06:56 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Mar 2008 19:06:56 +0100 (CET) Subject: [pypy-svn] r52011 - pypy/extradoc/proposal Message-ID: <20080301180656.0A70B1684EB@codespeak.net> Author: arigo Date: Sat Mar 1 19:06:56 2008 New Revision: 52011 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: Minor clarification (IMHO) Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 19:06:56 2008 @@ -182,7 +182,7 @@ has already been started, by generating and loading bytecode on the fly; despite emitting code at runtime, this kind of compiler really works ahead of time (AOT), because the code is fully - emitted before the program starts, and it doesn't exploit + emitted before the (Python) program starts, and it doesn't exploit additional informations that would be available only at runtime (most importantly informations about the types that each variable can assume); From niko at codespeak.net Sat Mar 1 19:22:01 2008 From: niko at codespeak.net (niko at codespeak.net) Date: Sat, 1 Mar 2008 19:22:01 +0100 (CET) Subject: [pypy-svn] r52012 - pypy/extradoc/proposal Message-ID: <20080301182201.258671684EA@codespeak.net> Author: niko Date: Sat Mar 1 19:21:59 2008 New Revision: 52012 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: tweak slightly Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Sat Mar 1 19:21:59 2008 @@ -233,6 +233,11 @@ Master's thesis at the Heinrich-Heine-Universit?t D?sseldorf about improving PyPy's JIT compiler generator. +Nicholas Matsakis (nicholas.matsakis at inf.ethz.ch) is also a core developer of +PyPy. He was the primary developer of the current JVM backend for PyPy, and +has participated in several PyPy sprints on various topics. He is currently +doing his PhD at ETH Zurich under the tutelage of Professor Thomas Gross. + .. _PyPy: http://codespeak.net/pypy .. _RPython: http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#rpython .. _`Stackless Python`: http://www.stackless.com/ From cfbolz at codespeak.net Sat Mar 1 19:36:34 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 19:36:34 +0100 (CET) Subject: [pypy-svn] r52013 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080301183634.4A8991684FB@codespeak.net> Author: cfbolz Date: Sat Mar 1 19:36:33 2008 New Revision: 52013 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: this belonged to the last commit Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Sat Mar 1 19:36:33 2008 @@ -289,7 +289,7 @@ writer, jitcode = self.serialize(ll_function, [int]) assert jitcode.code == assemble(writer.interpreter, - "green_direct_call", -1, 0, 1, 0, + "green_call", -1, 0, 1, 0, "make_redbox", 1, 0, "make_new_redvars", 1, 0, "make_new_greenvars", 0, From cfbolz at codespeak.net Sat Mar 1 19:41:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 19:41:09 +0100 (CET) Subject: [pypy-svn] r52014 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080301184109.6B723168507@codespeak.net> Author: cfbolz Date: Sat Mar 1 19:41:07 2008 New Revision: 52014 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Log: support for green calls raising exceptions Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 19:41:07 2008 @@ -18,7 +18,7 @@ class CallDesc: __metaclass__ = cachedtype - def __init__(self, RGenOp, FUNCTYPE, voidargs=()): + def __init__(self, RGenOp, rtyper, FUNCTYPE, voidargs=()): self.sigtoken = RGenOp.sigToken(FUNCTYPE.TO) self.result_kind = RGenOp.kindToken(FUNCTYPE.TO.RESULT) # xxx what if the result is virtualizable? @@ -49,8 +49,17 @@ try: result = rgenop.genconst(fnptr(*args)) except Exception, e: - XXX # set exception - return rgenop.genconst(whatever_return_value) + if not we_are_translated(): + # since we have a normal exception instance here + # we need to turn it into a low level one + bk = rtyper.annotator.bookkeeper + exc_classdef = bk.getuniqueclassdef(type(e)) + ll_exc = rtyper.exceptiondata.get_standard_ll_exc_instance( + rtyper, exc_classdef) + interpreter.jitstate.residual_ll_exception(ll_exc) + else: + interpreter.jitstate.residual_exception(ll_exc) + result = rgenop.genconst(whatever_return_value) interpreter.green_result(result) self.green_call = green_call @@ -80,7 +89,8 @@ self.graphs = [graph for (graph, tsgraph) in graph2tsgraph] self.jitcodes = values - self.calldesc = CallDesc(codewriter.RGenOp, lltype.typeOf(fnptr)) + self.calldesc = CallDesc(codewriter.RGenOp, codewriter.rtyper, + lltype.typeOf(fnptr)) class BytecodeWriter(object): @@ -591,7 +601,7 @@ return self.calldesc_positions[key] result = len(self.calldescs) self.calldescs.append( - CallDesc(self.RGenOp, FUNCTYPE, voidargs)) + CallDesc(self.RGenOp, self.rtyper, FUNCTYPE, voidargs)) self.calldesc_positions[key] = result return result From cfbolz at codespeak.net Sat Mar 1 20:26:10 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 20:26:10 +0100 (CET) Subject: [pypy-svn] r52015 - in pypy/branch/jit-refactoring/pypy/jit: rainbow timeshifter Message-ID: <20080301192610.F2A461684EA@codespeak.net> Author: cfbolz Date: Sat Mar 1 20:26:08 2008 New Revision: 52015 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: try to support virtual oop operations that raise Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 20:26:08 2008 @@ -14,6 +14,15 @@ from pypy.translator.backendopt.ssa import SSA_to_SSI from pypy.translator.unsimplify import varoftype +def residual_exception_nontranslated(jitstate, e, rtyper): + # since we have a normal exception instance here + # we need to turn it into a low level one + assert not we_are_translated() + bk = rtyper.annotator.bookkeeper + exc_classdef = bk.getuniqueclassdef(type(e)) + ll_exc = rtyper.exceptiondata.get_standard_ll_exc_instance( + rtyper, exc_classdef) + jitstate.residual_ll_exception(ll_exc) class CallDesc: __metaclass__ = cachedtype @@ -50,15 +59,9 @@ result = rgenop.genconst(fnptr(*args)) except Exception, e: if not we_are_translated(): - # since we have a normal exception instance here - # we need to turn it into a low level one - bk = rtyper.annotator.bookkeeper - exc_classdef = bk.getuniqueclassdef(type(e)) - ll_exc = rtyper.exceptiondata.get_standard_ll_exc_instance( - rtyper, exc_classdef) - interpreter.jitstate.residual_ll_exception(ll_exc) + residual_exception_nontranslated(interpreter.jitstate, e, rtyper) else: - interpreter.jitstate.residual_exception(ll_exc) + interpreter.jitstate.residual_exception(e) result = rgenop.genconst(whatever_return_value) interpreter.green_result(result) self.green_call = green_call Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py Sat Mar 1 20:26:08 2008 @@ -1,10 +1,11 @@ -from pypy.rpython.lltypesystem import lltype -from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.jit.timeshifter.rcontainer import cachedtype from pypy.jit.timeshifter import rvalue, rtimeshift -from pypy.translator import exceptiontransform +from pypy.jit.timeshifter.rcontainer import cachedtype +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable +from pypy.rpython.extregistry import ExtRegistryEntry +from pypy.rpython.lltypesystem import lltype from pypy.tool.sourcetools import func_with_new_name +from pypy.translator import exceptiontransform class SegfaultException(Exception): @@ -21,6 +22,7 @@ do_call = None def __init__(self, RGenOp, rtyper, fnobj, can_raise): + self.rtyper = rtyper ll_func = fnobj._callable FUNCTYPE = lltype.typeOf(fnobj) nb_args = len(FUNCTYPE.ARGS) @@ -151,8 +153,12 @@ return self.redboxbuilder(self.result_kind, gv_result) def residual_exception(self, jitstate, ExcCls): - ll_evalue = get_ll_instance_for_exccls(ExcCls) - jitstate.residual_ll_exception(ll_evalue) + from pypy.jit.rainbow.codewriter import residual_exception_nontranslated + if we_are_translated(): + ll_evalue = get_ll_instance_for_exccls(ExcCls) + jitstate.residual_ll_exception(ll_evalue) + else: + residual_exception_nontranslated(jitstate, ExcClass(), self.rtyper) return self.errorbox residual_exception._annspecialcase_ = 'specialize:arg(2)' Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Sat Mar 1 20:26:08 2008 @@ -555,7 +555,7 @@ jitstate.next = dispatchqueue.return_chain dispatchqueue.return_chain = jitstate -def ll_learn_nonzeroness(jitstate, ptrbox, nonzeroness): +def learn_nonzeroness(jitstate, ptrbox, nonzeroness): ptrbox.learn_nonzeroness(jitstate, nonzeroness) ##def ll_gvar_from_redbox(jitstate, redbox): @@ -1204,7 +1204,6 @@ def residual_exception(self, e): self.residual_ll_exception(cast_instance_to_base_ptr(e)) - def get_resuming(self): return self.frame.dispatchqueue.resuming From cfbolz at codespeak.net Sat Mar 1 20:57:59 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 20:57:59 +0100 (CET) Subject: [pypy-svn] r52016 - pypy/branch/jit-refactoring/pypy/jit/timeshifter/test Message-ID: <20080301195759.C332C1684EA@codespeak.net> Author: cfbolz Date: Sat Mar 1 20:57:58 2008 New Revision: 52016 Removed: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_promotion.py Log: these are all ported to the rainbow interp From cfbolz at codespeak.net Sat Mar 1 21:10:36 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 21:10:36 +0100 (CET) Subject: [pypy-svn] r52017 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter/test Message-ID: <20080301201036.537D116851E@codespeak.net> Author: cfbolz Date: Sat Mar 1 21:10:34 2008 New Revision: 52017 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_0tlc.py - copied, changed from r51886, pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_0tlc.py Removed: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_0tlc.py Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Log: port over one of the tiny language test files Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 21:10:34 2008 @@ -722,6 +722,14 @@ def handle_reverse_split_queue_hint(self, op, arg, result): self.emit("reverse_split_queue") + def handle_forget_hint(self, op, arg, result): + # a hint for testing only + assert self.varcolor(result) == "green" + assert self.varcolor(arg) != "green" + self.emit("revealconst") + self.emit(self.serialize_oparg("red", arg)) + self.register_greenvar(result) + def args_of_call(self, args, colored_as): result = [] reds, greens = self.sort_by_color(args, colored_as) @@ -1036,7 +1044,6 @@ destboxindex = self.serialize_oparg("red", args[0]) indexboxindex = self.serialize_oparg("red", args[1]) valboxindex = self.serialize_oparg("red", args[2]) - fieldname = args[1].value fielddescindex = self.arrayfielddesc_position(PTRTYPE.TO) if fielddescindex == -1: # Void field return Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sat Mar 1 21:10:34 2008 @@ -358,6 +358,10 @@ kind = self.frame.bytecode.typekinds[typeid] return redboxcls(kind, genconst) + @arguments("red", returns="green_from_red") + def opimpl_revealconst(self, box): + return box + @arguments("jumptarget") def opimpl_goto(self, target): self.frame.pc = target From cfbolz at codespeak.net Sat Mar 1 21:13:39 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 21:13:39 +0100 (CET) Subject: [pypy-svn] r52018 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter/test Message-ID: <20080301201339.36C7A1684DF@codespeak.net> Author: cfbolz Date: Sat Mar 1 21:13:38 2008 New Revision: 52018 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_1tl.py - copied, changed from r51886, pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_1tl.py Removed: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_1tl.py Log: this test just passes From cfbolz at codespeak.net Sat Mar 1 21:31:53 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 21:31:53 +0100 (CET) Subject: [pypy-svn] r52019 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter timeshifter/test Message-ID: <20080301203153.F16A2168504@codespeak.net> Author: cfbolz Date: Sat Mar 1 21:31:53 2008 New Revision: 52019 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_exception.py - copied, changed from r51886, pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_exception.py Removed: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_exception.py Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py Log: port over exception tests, most of which pass Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Mar 1 21:31:53 2008 @@ -10,7 +10,7 @@ from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.rpython.lltypesystem import lltype, rstr -from pypy.rpython.llinterp import LLInterpreter +from pypy.rpython.llinterp import LLInterpreter, LLException from pypy.rpython.module.support import LLSupport from pypy.annotation import model as annmodel from pypy.objspace.flow.model import summary, Variable @@ -144,7 +144,7 @@ def serialize(self, func, values, policy=None, inline=None, backendoptimize=False, - portal=None): + portal=None, **kwds): key = func, backendoptimize try: cache, argtypes = self._cache[key] @@ -205,9 +205,26 @@ graph.show() llinterp = LLInterpreter( self.rtyper, exc_data_ptr=writer.exceptiondesc.exc_data_ptr) - res = llinterp.eval_graph(graph, residualargs) + + if 'check_raises' not in kwds: + res = llinterp.eval_graph(graph, residualargs) + else: + try: + llinterp.eval_graph(graph, residualargs) + except LLException, e: + exc = kwds['check_raises'] + assert llinterp.find_exception(e) is exc, ( + "wrong exception type") + else: + raise AssertionError("DID NOT RAISE") + return True return res + def interpret_raises(self, ExcCls, ll_function, values, opt_consts=[], + *args, **kwds): + kwds['check_raises'] = ExcCls + return self.interpret(ll_function, values, opt_consts, *args, **kwds) + def get_residual_graph(self): return self.residual_graph Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py Sat Mar 1 21:31:53 2008 @@ -158,7 +158,7 @@ ll_evalue = get_ll_instance_for_exccls(ExcCls) jitstate.residual_ll_exception(ll_evalue) else: - residual_exception_nontranslated(jitstate, ExcClass(), self.rtyper) + residual_exception_nontranslated(jitstate, ExcCls(), self.rtyper) return self.errorbox residual_exception._annspecialcase_ = 'specialize:arg(2)' From cfbolz at codespeak.net Sat Mar 1 21:33:24 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 21:33:24 +0100 (CET) Subject: [pypy-svn] r52020 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter/test Message-ID: <20080301203324.EB3CB168504@codespeak.net> Author: cfbolz Date: Sat Mar 1 21:33:24 2008 New Revision: 52020 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_tlr.py - copied, changed from r51886, pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_tlr.py Removed: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_tlr.py Log: yet another ported test file From cfbolz at codespeak.net Sat Mar 1 22:39:33 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 22:39:33 +0100 (CET) Subject: [pypy-svn] r52021 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter timeshifter/test Message-ID: <20080301213933.2CE8A1684DF@codespeak.net> Author: cfbolz Date: Sat Mar 1 22:39:32 2008 New Revision: 52021 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_frontend.py - copied, changed from r51886, pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_frontend.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_virtualizable.py - copied, changed from r51886, pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_virtualizable.py Removed: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_frontend.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_virtualizable.py Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rvirtualizable.py Log: copy remaining tests over. fix most virtualizable problems. way too easy, somehow. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sat Mar 1 22:39:32 2008 @@ -927,6 +927,21 @@ self.emit("yellow_retrieve_result") self.register_greenvar(op.result) + def handle_vable_call(self, op, withexc): + assert op.opname == 'direct_call' + oopspec = op.args[0].value._obj._callable.oopspec + name, _ = oopspec.split('(') + kind, name = name.split('_', 1) + + if kind == 'vable.get': + opname = 'getfield' + else: + assert kind == 'vable.set' + opname = 'setfield' + args = op.args[1:] + args.insert(1, flowmodel.Constant(name, lltype.Void)) + newop = flowmodel.SpaceOperation(opname, args, op.result) + self.serialize_op(newop) def serialize_op_malloc(self, op): index = self.structtypedesc_position(op.args[0].value) @@ -979,7 +994,7 @@ # virtualizable access read PTRTYPE = args[0].concretetype if PTRTYPE.TO._hints.get('virtualizable', False): - XXX + assert op.args[1].value != 'vable_access' # non virtual case index = self.serialize_oparg("red", args[0]) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sat Mar 1 22:39:32 2008 @@ -1,7 +1,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import we_are_translated, CDefinedIntSymbolic -from pypy.jit.timeshifter import rtimeshift, rcontainer +from pypy.jit.timeshifter import rtimeshift, rcontainer, rvalue from pypy.jit.timeshifter.greenkey import empty_key, GreenKey, newgreendict from pypy.rpython.lltypesystem import lltype, llmemory Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py Sat Mar 1 22:39:32 2008 @@ -1,5 +1,5 @@ from pypy.jit.hintannotator.model import originalconcretetype -from pypy.jit.timeshifter import rvalue +from pypy.jit.timeshifter import rvalue, rcontainer from pypy.objspace.flow import model as flowmodel from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable @@ -315,7 +315,6 @@ def gettypedesc(self): if self.typedesc is None: - hrtyper = self.hrtyper T = self.original_concretetype.TO self.typedesc = rcontainer.StructTypeDesc(self.RGenOp, T) return self.typedesc @@ -325,7 +324,7 @@ redirected_fielddescs = unrolling_iterable( typedesc.redirected_fielddescs) TYPE = self.original_concretetype - kind = self.hrtyper.RGenOp.kindToken(TYPE) + kind = self.RGenOp.kindToken(TYPE) def make_arg_redbox(jitstate, inputargs_gv, i): box = typedesc.factory() @@ -348,7 +347,6 @@ def residual_argtypes(self): argtypes = [self.original_concretetype] - getredrepr = self.hrtyper.getredrepr typedesc = self.gettypedesc() for fielddesc, _ in typedesc.redirected_fielddescs: FIELDTYPE = fielddesc.RESTYPE Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Mar 1 22:39:32 2008 @@ -109,21 +109,7 @@ jitcode = writer.make_bytecode(graph2) # the bytecode writer can ask for llhelpers about lists and dicts rtyper.specialize_more_blocks() - argcolors = [] - - # make residual functype - RESTYPE = originalconcretetype(hannotator.binding(graph2.getreturnvar())) - ARGS = [] - for var in graph2.getargs(): - # XXX ignoring virtualizables for now - binding = hannotator.binding(var) - if not binding.is_green(): - ARGS.append(originalconcretetype(binding)) - self.RESIDUAL_FUNCTYPE = lltype.FuncType(ARGS, RESTYPE) - for i, ll_val in enumerate(values): - color = writer.varcolor(graph2.startblock.inputargs[i]) - argcolors.append(color) # rewire the original portal @@ -136,10 +122,10 @@ rewriter.rewrite(origportalgraph=origportalgraph, portalgraph=portalgraph, view = conftest.option.view and self.small) + self.RESIDUAL_FUNCTYPE = rewriter.RESIDUAL_FUNCTYPE self.writer = writer self.jitcode = jitcode - self.argcolors = argcolors def serialize(self, func, values, policy=None, @@ -153,22 +139,25 @@ else: self.__dict__.update(cache) assert argtypes == getargtypes(self.rtyper.annotator, values) - return self.writer, self.jitcode, self.argcolors + return self.writer, self.jitcode if len(self._cache_order) >= 3: del self._cache[self._cache_order.pop(0)] self._serialize(func, values, policy, inline, backendoptimize, portal) cache = self.__dict__.copy() self._cache[key] = cache, getargtypes(self.rtyper.annotator, values) self._cache_order.append(key) - return self.writer, self.jitcode, self.argcolors + return self.writer, self.jitcode def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): if hasattr(ll_function, 'convert_arguments'): assert len(ll_function.convert_arguments) == len(values) values = [decoder(value) for decoder, value in zip( ll_function.convert_arguments, values)] - writer, jitcode, argcolors = self.serialize(ll_function, values, - **kwds) + writer, jitcode= self.serialize(ll_function, values, **kwds) + argcolors = [] + for i, ll_val in enumerate(values): + color = writer.varcolor(self.graph.startblock.inputargs[i]) + argcolors.append(color) rgenop = writer.interpreter.rgenop sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py Sat Mar 1 22:39:32 2008 @@ -2,9 +2,10 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.annlowlevel import cachedtype, cast_base_ptr_to_instance from pypy.rpython.annlowlevel import base_ptr_lltype, cast_instance_to_base_ptr -from pypy.jit.timeshifter import rvalue +from pypy.rpython.annlowlevel import llhelper +from pypy.jit.timeshifter import rvalue, rvirtualizable from pypy.rlib.unroll import unrolling_iterable -from pypy.jit.timeshifter import rvirtualizable +from pypy.rlib.objectmodel import we_are_translated from pypy.annotation import model as annmodel @@ -211,8 +212,7 @@ """.split() def __init__(self, RGenOp, TYPE): - XXX - StructTypeDesc.__init__(self, hrtyper, TYPE) + StructTypeDesc.__init__(self, RGenOp, TYPE) ACCESS = self.TYPE.ACCESS redirected_fields = ACCESS.redirected_fields self.redirected_fielddescs = [] @@ -227,6 +227,7 @@ self.rti_desc = self.getfielddesc('vable_rti') self.access_desc = self.getfielddesc('vable_access') TOPPTR = self.access_desc.PTRTYPE + self.STRUCTTYPE = TOPPTR self.s_structtype = annmodel.lltype_to_annotation(TOPPTR) self.my_redirected_getsetters_untouched = {} @@ -238,7 +239,7 @@ if fielddesc.PTRTYPE != self.PTRTYPE: continue my_redirected_names.append(fielddesc.fieldname) - self._define_getset_field_ptr(hrtyper, fielddesc, j) + self._define_getset_field_ptr(RGenOp, fielddesc, j) access_untouched = lltype.malloc(ACCESS, immortal=True) @@ -253,32 +254,28 @@ self._define_collect_residual_args() - self._define_access_is_null(hrtyper) + self._define_access_is_null(RGenOp) def _define_virtual_desc(self): pass - def _define_getset_field_ptr(self, hrtyper, fielddesc, j): - XXX - annhelper = hrtyper.annhelper - s_lltype = annmodel.lltype_to_annotation(fielddesc.RESTYPE) - + def _define_getset_field_ptr(self, RGenOp, fielddesc, j): untouched = self.my_redirected_getsetters_untouched touched = self.my_redirected_getsetters_touched - mkptr = annhelper.delayedfunction - fnpairs = rvirtualizable.define_getset_field_ptrs(fielddesc, j) name = fielddesc.fieldname for getsetters, (get_field, set_field) in zip((untouched, touched), fnpairs): - get_field_ptr = mkptr(get_field, [self.s_structtype], s_lltype, - needtype = True) - set_field_ptr = mkptr(set_field, [self.s_structtype, s_lltype], - annmodel.s_None, needtype=True) + TYPE = lltype.Ptr(lltype.FuncType([self.STRUCTTYPE], + fielddesc.RESTYPE)) + get_field_ptr = llhelper(TYPE, get_field) + TYPE = lltype.Ptr(lltype.FuncType( + [self.STRUCTTYPE, fielddesc.RESTYPE], lltype.Void)) + set_field_ptr = llhelper(TYPE, set_field) getsetters[name] = get_field_ptr, set_field_ptr @@ -322,16 +319,11 @@ self.collect_residual_args = collect_residual_args - def _define_access_is_null(self, hrtyper): - XXX - RGenOp = hrtyper.RGenOp - annhelper = hrtyper.annhelper + def _define_access_is_null(self, RGenOp): def access_is_null(struc): assert not struc.vable_access - access_is_null_ptr = annhelper.delayedfunction(access_is_null, - [self.s_structtype], - annmodel.s_None, - needtype = True) + TYPE = lltype.Ptr(lltype.FuncType([self.STRUCTTYPE], lltype.Void)) + access_is_null_ptr = llhelper(TYPE, access_is_null) self.gv_access_is_null_ptr = RGenOp.constPrebuiltGlobal( access_is_null_ptr) self.access_is_null_token = RGenOp.sigToken( @@ -924,7 +916,11 @@ base_desc = typedesc.base_desc base_token = base_desc.fieldtoken builder.genop_setfield(base_token, gv_outside, gv_base) - vable_rti_ptr = cast_instance_to_base_ptr(vable_rti) + if we_are_translated(): + vable_rti_ptr = cast_instance_to_base_ptr(vable_rti) + else: + vable_rti_ptr = vable_rti + gv_vable_rti = builder.rgenop.genconst(vable_rti_ptr) rti_token = typedesc.rti_desc.fieldtoken builder.genop_setfield(rti_token, gv_outside, gv_vable_rti) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rvirtualizable.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rvirtualizable.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rvirtualizable.py Sat Mar 1 22:39:32 2008 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory, lloperation -from pypy.rpython.annlowlevel import cachedtype +from pypy.rpython.annlowlevel import cachedtype, base_ptr_lltype from pypy.rpython.annlowlevel import cast_base_ptr_to_instance from pypy.rpython.annlowlevel import cast_instance_to_base_ptr +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable debug_print = lloperation.llop.debug_print @@ -13,7 +14,8 @@ def touch_update(strucref): struc = lltype.cast_opaque_ptr(TOPPTR, strucref) vable_rti = struc.vable_rti - vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) + if we_are_translated(): + vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) vable_rti.touch(struc.vable_base) vable_base = struc.vable_base @@ -37,7 +39,8 @@ T = fielddesc.RESTYPE if fielddesc.canbevirtual and fielddesc.gcref: vable_rti = struc.vable_rti - vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) + if we_are_translated(): + vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) vable_rti.touched_ptr_field(struc.vable_base, j) struc = lltype.cast_pointer(fielddesc.PTRTYPE, struc) setattr(struc, fielddesc.fieldname, value) @@ -47,7 +50,8 @@ tgt = lltype.cast_pointer(fielddesc.PTRTYPE, struc) if fielddesc.canbevirtual and fielddesc.gcref: vable_rti = struc.vable_rti - vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) + if we_are_translated(): + vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) vable_base = struc.vable_base if vable_rti.is_field_virtual(vable_base, j): # this will force @@ -58,13 +62,15 @@ def set_field_untouched(struc, value): vable_rti = struc.vable_rti - vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) + if we_are_translated(): + vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) vable_rti.touch_update(lltype.cast_opaque_ptr(llmemory.GCREF, struc)) set_field_touched(struc, value) def get_field_untouched(struc): vable_rti = struc.vable_rti - vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) + if we_are_translated(): + vable_rti = cast_base_ptr_to_instance(VirtualizableRTI, vable_rti) return vable_rti.read_field(fielddesc, struc.vable_base, j) return ((get_field_untouched, set_field_untouched), @@ -94,6 +100,9 @@ return vrti._get_forced(vablerti, fielddesc, base) _read_field._annspecialcase_ = "specialize:arg(2)" + # hack for testing: make the llinterpreter believe this is a Ptr to base + # instance + _TYPE = base_ptr_lltype() class VirtualizableRTI(RTI): _attrs_ = "frameinfo touch_update shape_place".split() From cfbolz at codespeak.net Sat Mar 1 22:43:03 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Mar 2008 22:43:03 +0100 (CET) Subject: [pypy-svn] r52022 - pypy/branch/jit-refactoring/pypy/jit/timeshifter/test Message-ID: <20080301214303.7C0CB168506@codespeak.net> Author: cfbolz Date: Sat Mar 1 22:43:02 2008 New Revision: 52022 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/support.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_rcontainer.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_rvalue.py Log: fix those tests Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/support.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/support.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/support.py Sat Mar 1 22:43:02 2008 @@ -57,11 +57,6 @@ raise AttributeError, name -class FakeHRTyper(object): - RGenOp = FakeRGenOp - -fakehrtyper = FakeHRTyper() - class FakeGenVar(GenVar): def __init__(self, count=0): self.count=count @@ -87,7 +82,7 @@ def vmalloc(TYPE, *boxes): jitstate = FakeJITState() assert isinstance(TYPE, lltype.Struct) # for now - structdesc = rcontainer.StructTypeDesc(fakehrtyper, TYPE) + structdesc = rcontainer.StructTypeDesc(FakeRGenOp, TYPE) box = structdesc.factory() for fielddesc, valuebox in zip(structdesc.fielddescs, boxes): if valuebox is None: @@ -104,5 +99,5 @@ def getfielddesc(STRUCT, name): assert isinstance(STRUCT, lltype.Struct) - structdesc = rcontainer.StructTypeDesc(fakehrtyper, STRUCT) + structdesc = rcontainer.StructTypeDesc(FakeRGenOp, STRUCT) return structdesc.fielddesc_by_name[name] Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_rcontainer.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_rcontainer.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_rcontainer.py Sat Mar 1 22:43:02 2008 @@ -2,7 +2,7 @@ from pypy.jit.timeshifter import rvalue, rcontainer from pypy.jit.timeshifter.test.support import FakeJITState, FakeGenVar from pypy.jit.timeshifter.test.support import FakeGenConst -from pypy.jit.timeshifter.test.support import fakehrtyper, signed_kind +from pypy.jit.timeshifter.test.support import signed_kind from pypy.jit.timeshifter.test.support import vmalloc, makebox from pypy.jit.timeshifter.test.support import getfielddesc Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_rvalue.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_rvalue.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_rvalue.py Sat Mar 1 22:43:02 2008 @@ -60,7 +60,7 @@ V0 = FakeGenVar() box = rvalue.PtrRedBox("dummy pointer", V0) STRUCT = lltype.Struct("dummy", ("foo", lltype.Signed)) - desc = rcontainer.StructFieldDesc(FakeHRTyper(), lltype.Ptr(STRUCT), "foo", 0) + desc = rcontainer.StructFieldDesc(FakeRGenOp, lltype.Ptr(STRUCT), "foo", 0) box2 = box.op_getfield(jitstate, desc) V1 = box2.genvar assert box.known_nonzero From jared.grubb at codespeak.net Sun Mar 2 02:15:51 2008 From: jared.grubb at codespeak.net (jared.grubb at codespeak.net) Date: Sun, 2 Mar 2008 02:15:51 +0100 (CET) Subject: [pypy-svn] r52024 - in pypy/dist/pypy: interpreter/pyparser rlib/parsing rlib/parsing/test Message-ID: <20080302011551.F07C6168506@codespeak.net> Author: jared.grubb Date: Sun Mar 2 02:15:50 2008 New Revision: 52024 Modified: pypy/dist/pypy/interpreter/pyparser/parsestring.py pypy/dist/pypy/rlib/parsing/regexparse.py pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py Log: parsestring: simplify the octal parsing a bit regexparse: simplify the unescape() function test_pcre_regtest: preprocesses the testoutput1 file and runs most of the tests Modified: pypy/dist/pypy/interpreter/pyparser/parsestring.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/parsestring.py (original) +++ pypy/dist/pypy/interpreter/pyparser/parsestring.py Sun Mar 2 02:15:50 2008 @@ -155,15 +155,13 @@ lis.append('\013') # VT elif ch == 'a': lis.append('\007') # BEL, not classic C - elif '0' <= ch <= '7': - c = ord(s[ps - 1]) - ord('0') - if ps < end and '0' <= s[ps] <= '7': - c = (c << 3) + ord(s[ps]) - ord('0') - ps += 1 - if ps < end and '0' <= s[ps] <= '7': - c = (c << 3) + ord(s[ps]) - ord('0') - ps += 1 - lis.append(chr(c)) + elif ch in '01234567': + # Look for up to two more octal digits + span = ps + span += (span < end) and (s[span] in '01234567') + span += (span < end) and (s[span] in '01234567') + lis.append(chr(int(s[ps - 1 : span], 8))) + ps = span elif ch == 'x': if ps+2 <= end and isxdigit(s[ps]) and isxdigit(s[ps + 1]): lis.append(chr(int(s[ps : ps + 2], 16))) Modified: pypy/dist/pypy/rlib/parsing/regexparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/regexparse.py (original) +++ pypy/dist/pypy/rlib/parsing/regexparse.py Sun Mar 2 02:15:50 2008 @@ -9,72 +9,59 @@ set = py.builtin.set ESCAPES = { - "\\a": "\a", - "\\b": "\b", - "\\e": "\x1b", - "\\f": "\f", - "\\n": "\n", - "\\r": "\r", - "\\t": "\t", - "\\v": "\v", - "\\": "\\" + "a": "\a", + "b": "\b", + "e": "\x1b", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", } for i in range(256): - if chr(i) not in 'x01234567sSwWdD': - # 'x' and numbers are reserved for hexadecimal/octal escapes - escaped = "\\" + chr(i) - if escaped not in ESCAPES: - ESCAPES[escaped] = chr(i) - - # Three digit octals - escaped = "\\%03o" % i - if escaped not in ESCAPES: - ESCAPES[escaped] = chr(i) - - if 0 <= i <= 077: - # Two digit octal digs are ok too - escaped = "\\%02o" % i - if escaped not in ESCAPES: - ESCAPES[escaped] = chr(i) - # Add the ctrl-x types: # Rule, according to PCRE: # if x is a lower case letter, it is converted to upper case. # Then bit 6 of the character (hex 40) is inverted. - # Thus, \cz => 0x1A, but \c{ => 0x3B, while \c; => 0x7B. - escaped = "\\c%s" % chr(i) - if escaped not in ESCAPES: - ESCAPES[escaped] = chr(ord(chr(i).upper()) ^ 0x40) + # Thus, \cz => 0x1A, \c{ => 0x3B, \c; => 0x7B. + escaped = "c%s" % chr(i) + ESCAPES[escaped] = chr(ord(chr(i).upper()) ^ 0x40) + +def unescape_muncher(string): + """Return a tuple, representing the first character of the string + (appropriately unescaped) and the rest of the string that wasn't + handled.""" + if string[0] != '\\': + # Not an escape character + return string[0], string[1:] + if string[1] == 'x': + # Hex char, must have two hex digits + char = chr(int(string[2:4], 16)) + return char, string[4:] + if string[1] in '01234567': + # Octal number, up to three digits long + span = 2 + span += (span < len(string)) and (string[span] in '01234567') + span += (span < len(string)) and (string[span] in '01234567') + char = chr(int(string[1:span], 8)) + return char, string[span:] + if string[1] == 'c': + # Special \cx types + return ESCAPES['c'+string[2]], string[3:] + if string[1] in ESCAPES: + # Special escapes are in ESCAPE + return ESCAPES[string[1]], string[2:] + # Otherwise, it's just the character it's meant to be (e.g., '\.') + return string[1], string[2:] - -for a in "0123456789ABCDEFabcdef": - for b in "0123456789ABCDEFabcdef": - escaped = "\\x%s%s" % (a, b) - if escaped not in ESCAPES: - ESCAPES[escaped] = chr(int("%s%s" % (a, b), 16)) - + def unescape(s): + """Unescape a whole string.""" result = [] - i = 0 - while i < len(s): - if s[i] != "\\": - result.append(s[i]) - i += 1 - continue - if s[i + 1] == "x": - escaped = s[i: i + 4] - i += 4 - elif s[i + 1] in "01234567": - escaped = s[i: i + 4] - i += 4 - else: - escaped = s[i: i + 2] - i += 2 - if escaped not in ESCAPES: - raise ValueError("escape %r unknown" % (escaped, )) - else: - result.append(ESCAPES[escaped]) + while s: + char, s = unescape_muncher(s) + result.append(char) return "".join(result) syntax = r""" @@ -184,20 +171,15 @@ charclass: '\' 'd' return { set([chr(c) for c in range(ord('0'), ord('9')+1)]) } - | '\' - 's' + | '\' 's' return { set(['\t', '\n', '\f', '\r', ' ']) } - | '\' - 'w' + | '\' 'w' return { set([chr(c) for c in range(ord('a'), ord('z')+1)] + [chr(c) for c in range(ord('A'), ord('Z')+1)] + [chr(c) for c in range(ord('0'), ord('9')+1)] + ['_']) } - | '\' - 'D' + | '\' 'D' return { set([chr(c) for c in range(256)]) - set([chr(c) for c in range(ord('0'), ord('9')+1)]) } - | '\' - 'S' + | '\' 'S' return { set([chr(c) for c in range(256)]) - set(['\t', '\n', '\f', '\r', ' ']) } - | '\' - 'W' + | '\' 'W' return { set([chr(c) for c in range(256)]) - set([chr(c) for c in range(ord('a'), ord('z')+1)] + [chr(c) for c in range(ord('A'), ord('Z')+1)] + [chr(c) for c in range(ord('0'), ord('9')+1)] + ['_'])}; NUM: @@ -1862,6 +1844,9 @@ + + + def test_generate(): f = py.magic.autopath() oldcontent = f.read() Modified: pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py Sun Mar 2 02:15:50 2008 @@ -12,143 +12,121 @@ py.test.skip("In Progress...") -def get_simult_lines(tests, results, test_line_num=0): - """Returns a line from the input/output, ensuring that - we are sync'd up between the two.""" - test = tests.pop(0) - result = results.pop(0) +def read_file(file): + lines = [line for line in file.readlines()] - test_line_num += 1 - - if test != result: - raise Exception("Lost sync between files at input line %d.\n INPUT: %s\n OUTPUT: %s" % (test_line_num, test, result)) - - return test - -def create_regex_iterator(tests, results): - """Gets a test definition line, formatted per the PCRE spec. This is a - generator that returns each regex test.""" - while tests: + # Look for things to skip... + no_escape = r'(^|[^\\])(\\\\)*' # Make sure there's no escaping \ + greedy_ops = re.compile(no_escape + r'[*?+}\(]\?') # Look for *? +? }? (? + back_refs = re.compile(no_escape + r'\(.*' + no_escape + r'\\1') # find a \1 + + # suite = [ + # [regex, flags, [(test,result),(test,result),...]] + # [regex, flags, [(test,result),(test,result),...]] + # ] + suite = [] + while lines: delim = None regex = '' - # A line is marked by a start-delimeter and an end-delimeter. # The delimeter is non-alphanumeric # If a backslash follows the delimiter, then the backslash should # be appended to the end. (Otherwise, \ + delim would not be a # delim anymore!) while 1: - regex += get_simult_lines(tests, results) - - if delim is None: - delim = regex[0] + regex += lines.pop(0) + if not delim: + if not regex.strip(): # Suppress blank lanes before delim + regex = '' + continue + delim = regex.strip()[0] assert delim in (set(string.printable) - set(string.letters) - set(string.digits)) - test_re = re.compile(r'%(delim)s(([^%(delim)s]|\\%(delim)s)*([^\\]))%(delim)s(\\?)(.*)' % {'delim': delim}) + test_re = re.compile(r'%(delim)s(([^%(delim)s]|\\%(delim)s)*([^\\]))%(delim)s(\\?)([^\n\r]*)' % {'delim': delim}) # last two groups are an optional backslash and optional flags matches = test_re.findall(regex) if matches: break - assert len(matches)==1 + assert len(matches)==1 # check to make sure we matched right regex = matches[0][0] regex += matches[0][-2] # Add the backslash, if we gotta flags = matches[0][-1] # Get the flags for the regex - yield regex, flags + tests = [] -def create_result_iterator(tests, results): - """Gets the expected return sets for each regular expression.""" - # Second line is the test to run against the regex - # ' TEXT' - while 1: - test = get_simult_lines(tests, results) - if not test: - raise StopIteration - if not test.startswith(' '): - raise Exception("Input & output match, but I don't understand. (Got %r)" % test) - if test.endswith('\\'): # Tests that end in \ expect the \ to be chopped off - assert not test.endswith('\\\\') # make sure there are no \\ at end - test = test[:-1] - test = unescape(test[4:]) - - # Third line in the OUTPUT is the result, either: - # ' 0: ...' for a match (but this is ONLY escaped by \x__ types) - # 'No match' for no match - result = results.pop(0) - result = re.sub(r'\\x([0-9a-fA-F]{2})', lambda m: chr(int(m.group(1),16)), result) - if result == 'No match': + if greedy_ops.search(regex) or back_refs.search(regex): + # Suppress complex features we can't do + pass + elif flags: + # Suppress any test that requires PCRE flags pass - elif result.startswith(' 0:'): - # Now we need to eat any further lines like: - # ' 1: ....' a subgroup match - while results[0]: - if results[0][2] == ':': - results.pop(0) - else: - break else: - raise Exception("Lost sync in output.") - yield test, result - -class SkipException(Exception): - pass - + # In any other case, we're going to add the test + # All the above test fall through and DONT get appended + suite.append([regex, flags, tests]) + + # Now find the test and expected result + while lines: + test = lines.pop(0).strip() + if not test: + break # blank line ends the set + if test.endswith('\\'): # Tests that end in \ expect the \ to be chopped off + assert not test.endswith('\\\\\\') # Make sure not three \'s. otherwise this check will get ridiculous + if not test.endswith('\\\\'): # Two \'s means a real \ + test = test[:-1] + test = unescape(test) + + # Third line in the OUTPUT is the result, either: + # ' 0: ...' for a match (but this is ONLY escaped by \x__ types) + # 'No match' for no match + match = lines.pop(0).rstrip('\r\n') + match = re.sub(r'\\x([0-9a-fA-F]{2})', lambda m: chr(int(m.group(1),16)), match) + if match.startswith('No match'): + pass + elif match.startswith(' 0:'): + # Now we need to eat any further lines like: + # ' 1: ....' a subgroup match + while lines[0].strip(): + # ' 0+ ...' is also possible here + if lines[0][2] in [':','+']: + lines.pop(0) + else: + break + else: + print " *** %r ***" % match + raise Exception("Lost sync in output.") + tests.append((test,match)) + return suite + def test_file(): """Open the PCRE tests and run them.""" - tests = [line.rstrip() for line in open('testinput1','r').readlines()] - results = [line.rstrip() for line in open('testoutput1','r').readlines()] - - regex_flag_mapping = { '': lambda s: s, - 'i': lambda s: s.upper() - } - - regex_set = create_regex_iterator(tests, results) - import pdb - for regex, regex_flags in regex_set: - try: - print '%r' % regex - - # Create an iterator to grab the test/results for this regex - result_set = create_result_iterator(tests, results) - - # Handle the flags: - if regex_flags in regex_flag_mapping: - text_prepare = regex_flag_mapping[regex_flags] - elif 'x' in regex_flags: - raise SkipException("Cant do extended PRCE expressions") - else: - print "UNKNOWN FLAGS: %s" % regex_flags - continue - - skipped = any([op in regex for op in ['*?', '??', '+?', '}?', '(?']]) - if skipped: - raise SkipException("Cant do non-greedy operators or '(?' constructions)") - - regex_to_use = text_prepare(regex) + suite = read_file(open('testoutput1','r')) - anchor_left = regex_to_use.startswith('^') - anchor_right = regex_to_use.endswith('$') and not regex_to_use.endswith('\\$') - if anchor_left: - regex_to_use = regex_to_use[1:] # chop the ^ if it's there - if anchor_right: - regex_to_use = regex_to_use[:-1] # chop the $ if it's there - - if not regex_to_use: - raise SkipException("Cant do blank regex") - except SkipException, e: - print " SKIPPED (%s)" % e.message - # now burn all the tests for this regex - for _ in result_set: - pass + import pdb + while suite: + regex, flags, tests = suite.pop(0) + print '/%r/%s' % (regex, flags) + + regex_to_use = regex + + anchor_left = regex_to_use.startswith('^') + anchor_right = regex_to_use.endswith('$') and not regex_to_use.endswith('\\$') + if anchor_left: + regex_to_use = regex_to_use[1:] # chop the ^ if it's there + if anchor_right: + regex_to_use = regex_to_use[:-1] # chop the $ if it's there + + if not regex_to_use: + print " SKIPPED (Cant do blank regex)" continue # Finally, we make the pypy regex runner runner = make_runner(regex_to_use) # Now run the test expressions against the Regex - for test, result in result_set: + for test, match in tests: # Create possible subsequences that we should test if anchor_left: start_range = [0] @@ -163,21 +141,23 @@ # Search the possibilities for a match... for start, end in subseq_gen: - attempt = text_prepare(test[start:end]) + attempt = test[start:end] matched = runner.recognize(attempt) if matched: break # Did we get what we expected? - if result == 'No match': + if match == 'No match': if matched: print " FALSE MATCH: regex==%r test==%r" % (regex, test) else: - print " pass: regex==%r test==%r" % (regex, test) - elif result.startswith(' 0: '): + pass + #print " pass: regex==%r test==%r" % (regex, test) + elif match.startswith(' 0: '): if not matched: print " MISSED: regex==%r test==%r" % (regex, test) - elif not attempt==text_prepare(result[4:]): - print " BAD MATCH: regex==%r test==%r found==%r expect==%r" % (regex, test, attempt, result[4:]) + elif not attempt==match[4:]: + print " BAD MATCH: regex==%r test==%r found==%r expect==%r" % (regex, test, attempt, match[4:]) else: - print " pass: regex==%r test==%r" % (regex, test) + pass + #print " pass: regex==%r test==%r" % (regex, test) From pypy-svn at codespeak.net Sun Mar 2 10:04:29 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Sun, 2 Mar 2008 10:04:29 +0100 (CET) Subject: [pypy-svn] Best Sales 2008! Message-ID: 20080302130352.113855.qmail@lan-84-240-8-65.vln.skynet.lt An HTML attachment was scrubbed... URL: http://codespeak.net/pipermail/pypy-svn/attachments/20080302/7daf45db/attachment.htm From cfbolz at codespeak.net Sun Mar 2 15:53:21 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Mar 2008 15:53:21 +0100 (CET) Subject: [pypy-svn] r52040 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080302145321.A52E516855E@codespeak.net> Author: cfbolz Date: Sun Mar 2 15:53:21 2008 New Revision: 52040 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: support for metacalls Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sun Mar 2 15:53:21 2008 @@ -69,6 +69,53 @@ def _freeze_(self): return True + +class CallDesc: + __metaclass__ = cachedtype + + def __init__(self, RGenOp, rtyper, FUNCTYPE, voidargs=()): + self.sigtoken = RGenOp.sigToken(FUNCTYPE.TO) + self.result_kind = RGenOp.kindToken(FUNCTYPE.TO.RESULT) + # xxx what if the result is virtualizable? + self.redboxbuilder = rvalue.ll_redboxbuilder(FUNCTYPE.TO.RESULT) + whatever_return_value = FUNCTYPE.TO.RESULT._defl() + numargs = len(FUNCTYPE.TO.ARGS) + argiter = unrolling_iterable(FUNCTYPE.TO.ARGS) + def green_call(interpreter, fnptr_gv, greenargs): + fnptr = fnptr_gv.revealconst(FUNCTYPE) + assert len(greenargs) + len(voidargs) == numargs + args = () + j = 0 + k = 0 + for ARG in argiter: + if ARG == lltype.Void: + arg = voidargs[k] + # XXX terrible hack + if not we_are_translated(): + arg._TYPE = lltype.Void + args += (arg, ) + k += 1 + else: + genconst = greenargs[j] + arg = genconst.revealconst(ARG) + args += (arg, ) + j += 1 + rgenop = interpreter.jitstate.curbuilder.rgenop + try: + result = rgenop.genconst(fnptr(*args)) + except Exception, e: + if not we_are_translated(): + residual_exception_nontranslated(interpreter.jitstate, e, rtyper) + else: + interpreter.jitstate.residual_exception(e) + result = rgenop.genconst(whatever_return_value) + interpreter.green_result(result) + self.green_call = green_call + + def _freeze_(self): + return True + + class IndirectCallsetDesc(object): __metaclass__ = cachedtype @@ -143,6 +190,7 @@ self.graph_color = self.graph_calling_color(graph) self.calldescs = [] self.indirectcalldescs = [] + self.metacalldescs = [] self.is_portal = is_portal # mapping constant -> index in constants self.const_positions = {} @@ -176,6 +224,8 @@ self.graph_positions = {} # mapping fnobjs to index self.calldesc_positions = {} + # mapping class to index + self.metacalldesc_positions = {} # mapping fnobjs to index self.indirectcalldesc_positions = {} @@ -205,6 +255,7 @@ self.num_local_mergepoints, self.graph_color, self.calldescs, + self.metacalldescs, self.indirectcalldescs, self.is_portal) bytecode._source = self.assembler @@ -608,6 +659,29 @@ self.calldesc_positions[key] = result return result + def metacalldesc_position(self, op): + key = op + if key in self.metacalldesc_positions: + return self.metacalldesc_positions[key] + result = len(self.metacalldescs) + metadesc = op.args[1].value(self) + ARGS = [arg.concretetype for arg in op.args[2:]] + argiter = unrolling_iterable(ARGS) + def func(interpreter, redargs): + args = () + j = 0 + for ARG in argiter: + if ARG == lltype.Void: + args += (None, ) + else: + box = redargs[j] + args += (box, ) + j += 1 + return metadesc.metafunc(interpreter.jitstate, *args) + self.metacalldescs.append(func) + self.metacalldesc_positions[key] = result + return result + def indirectcalldesc_position(self, graph2code): key = graph2code.items() key.sort() @@ -753,6 +827,15 @@ print op, kind, withexc return handler(op, withexc) + def serialize_op_ts_metacall(self, op): + emitted_args = [] + for v in op.args[2:]: + if v.concretetype != lltype.Void: + emitted_args.append(self.serialize_oparg("red", v)) + metaindex = self.metacalldesc_position(op) + self.emit("metacall", metaindex, len(emitted_args), *emitted_args) + self.register_redvar(op.result) + def serialize_op_indirect_call(self, op): kind, withexc = self.guess_call_kind(op) if kind == "green": Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py Sun Mar 2 15:53:21 2008 @@ -132,6 +132,10 @@ index = src.load_2byte() function = jitcode.calldescs[index] args.append(function) + elif argspec == "metacalldesc": + index = src.load_2byte() + function = jitcode.metacalldescs[index] + args.append(function) elif argspec == "indirectcalldesc": index = src.load_2byte() function = jitcode.indirectcalldescs[index] Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sun Mar 2 15:53:21 2008 @@ -23,7 +23,8 @@ keydescs, structtypedescs, fielddescs, arrayfielddescs, interiordescs, exceptioninstances, oopspecdescs, promotiondescs, called_bytecodes, num_mergepoints, - graph_color, calldescs, indirectcalldescs, is_portal): + graph_color, calldescs, metacalldescs, + indirectcalldescs, is_portal): self.name = name self.code = code self.constants = constants @@ -41,6 +42,7 @@ self.num_mergepoints = num_mergepoints self.graph_color = graph_color self.calldescs = calldescs + self.metacalldescs = metacalldescs self.indirectcalldescs = indirectcalldescs self.is_portal = is_portal @@ -121,6 +123,10 @@ index = self.load_2byte() function = self.frame.bytecode.calldescs[index] args += (function, ) + elif argspec == "metacalldesc": + index = self.load_2byte() + function = self.frame.bytecode.metacalldescs[index] + args += (function, ) elif argspec == "indirectcalldesc": index = self.load_2byte() function = self.frame.bytecode.indirectcalldescs[ind