From cfbolz at codespeak.net Fri Feb 1 13:47:42 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 13:47:42 +0100 (CET) Subject: [pypy-svn] r51174 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080201124742.44FB8168441@codespeak.net> Author: cfbolz Date: Fri Feb 1 13:47:40 2008 New Revision: 51174 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (contents, props changed) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: the first test where the rainbow interpreter actually runs passes. required a very messy test runner and some whacking. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Fri Feb 1 13:47:40 2008 @@ -1,3 +1,4 @@ +from pypy.rlib.rarithmetic import intmask from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype @@ -24,15 +25,40 @@ def _freeze_(self): return True +SIGN_EXTEND2 = 1 << 15 + +STOP = object() + class JitInterpreter(object): def __init__(self): self.opcode_implementations = [] self.opcode_descs = [] self.opname_to_index = {} self.jitstate = None + self.queue = None self.bytecode = None + self.pc = -1 self._add_implemented_opcodes() + def run(self, jitstate, bytecode, greenargs, redargs): + self.jitstate = jitstate + self.queue = rtimeshift.ensure_queue(jitstate, + rtimeshift.BaseDispatchQueue) + rtimeshift.enter_frame(self.jitstate, self.queue) + self.frame = self.jitstate.frame + self.frame.pc = 0 + self.frame.bytecode = bytecode + self.frame.local_boxes = redargs + self.frame.local_green = greenargs + self.bytecode_loop() + + def bytecode_loop(self): + while 1: + bytecode = self.load_2byte() + result = self.opcode_implementations[bytecode](self) + if result is STOP: + return + # construction-time interface def _add_implemented_opcodes(self): @@ -44,7 +70,6 @@ self.opcode_implementations.append(getattr(self, name).im_func) self.opcode_descs.append(None) - def find_opcode(self, name): return self.opname_to_index.get(name, -1) @@ -54,7 +79,9 @@ def implementation(self): args = () for i in numargs: - args.append(self.get_greenarg()) + genconst = self.get_greenarg() + arg = self.jitstate.curbuilder.revealconst(opdesc.ARGS[i]) + args += (arg, ) result = opdesc.llop(*args) self.green_result(result) elif color == "red": @@ -65,12 +92,9 @@ else: XXX def implementation(self): - XXX - # the following is nonsense: the green arguments are - # GenConsts, so there are revealconsts missing - args = (self.jitstate, ) + args = (opdesc, self.jitstate, ) for i in numargs: - args.append(self.get_redarg()) + args += (self.get_redarg(), ) result = impl(*args) self.red_result(result) else: @@ -83,12 +107,46 @@ return index - # operation implemetations + # operation helper functions + + def load_2byte(self): + pc = self.frame.pc + result = ((ord(self.frame.bytecode.code[pc]) << 8) | + ord(self.frame.bytecode.code[pc + 1])) + self.frame.pc = pc + 2 + return intmask((result ^ SIGN_EXTEND2) - SIGN_EXTEND2) + + def load_4byte(self): + pc = self.frame.pc + result = ((ord(self.frame.bytecode.code[pc + 0]) << 24) | + (ord(self.frame.bytecode.code[pc + 1]) << 16) | + (ord(self.frame.bytecode.code[pc + 2]) << 8) | + (ord(self.frame.bytecode.code[pc + 3]) << 0)) + self.frame.pc = pc + 4 + return intmask(result) + + def get_greenarg(self): + i = self.load_2byte() + if i < 0: + return self.frame.bytecode.constants[~i] + return self.frame.local_green[i] + + def get_redarg(self): + return self.frame.local_boxes[self.load_2byte()] + + def red_result(self, box): + self.frame.local_boxes.append(box) + + def green_result(self, gv): + self.frame.local_green.append(gv) + + # operation implementations def opimpl_make_redbox(self): XXX def opimpl_goto(self): - XXX + target = self.load_4byte() + self.frame.pc = target def opimpl_green_goto_iftrue(self): XXX @@ -97,20 +155,35 @@ XXX def opimpl_red_return(self): - XXX + rtimeshift.save_return(self.jitstate) + # XXX for now + newstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) + self.jitstate = newstate + return STOP def opimpl_green_return(self): XXX + return STOP # XXX wrong, of course def opimpl_make_new_redvars(self): # an opcode with a variable number of args # num_args arg_old_1 arg_new_1 ... - XXX + num = self.load_2byte() + newlocalboxes = [] + for i in range(num): + newlocalboxes.append(self.get_redarg()) + self.frame.local_boxes = newlocalboxes def opimpl_make_new_greenvars(self): # an opcode with a variable number of args # num_args arg_old_1 arg_new_1 ... - XXX + num = self.load_2byte() + newgreens = [] + for i in range(num): + newgreens.append(self.get_greenarg()) + self.frame.local_green = newgreens + + class BytecodeWriter(object): @@ -268,7 +341,6 @@ return color def register_redvar(self, arg): - print "register_redvar", arg assert arg not in self.redvar_positions self.redvar_positions[arg] = result = self.free_red[self.current_block] self.free_red[self.current_block] += 1 @@ -288,7 +360,7 @@ def green_position(self, arg): if isinstance(arg, flowmodel.Variable): return self.greenvar_positions[arg] - return -self.const_position(arg) - 1 + return ~self.const_position(arg) def const_position(self, const): if const in self.const_positions: @@ -374,15 +446,15 @@ -# XXX too lazy to fix the interface of make_opdesc +# XXX too lazy to fix the interface of make_opdesc, ExceptionDesc class PseudoHOP(object): def __init__(self, op, args_s, s_result, RGenOp): self.spaceop = op self.args_s = args_s self.s_result = s_result - self.rtyper = PseudoHRTyper(RGenOp) + self.rtyper = PseudoHRTyper(RGenOp=RGenOp) class PseudoHRTyper(object): - def __init__(self, RGenOp): - self.RGenOp = RGenOp + def __init__(self, **args): + self.__dict__.update(**args) Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 1 13:47:40 2008 @@ -0,0 +1,140 @@ +from pypy.translator.translator import TranslationContext, graphof +from pypy.jit.hintannotator.annotator import HintAnnotator +from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy +from pypy.jit.hintannotator.model import SomeLLAbstractConstant, OriginFlags +from pypy.jit.hintannotator.model import originalconcretetype +from pypy.jit.rainbow.bytecode import BytecodeWriter, label, tlabel, assemble +from pypy.jit.codegen.llgraph.rgenop import RGenOp +from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest +from pypy.jit.rainbow import bytecode +from pypy.jit.timeshifter import rtimeshift, exception, rvalue +from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython.llinterp import LLInterpreter +from pypy.annotation import model as annmodel +from pypy import conftest + +def getargtypes(annotator, values): + return [annotation(annotator, x) for x in values] + +def annotation(a, x): + T = lltype.typeOf(x) + if T == lltype.Ptr(rstr.STR): + t = str + else: + t = annmodel.lltype_to_annotation(T) + return a.typeannotation(t) + +P_OOPSPEC_NOVIRTUAL = HintAnnotatorPolicy(oopspec=True, + novirtualcontainer=True, + entrypoint_returns_red=False) + +class AbstractInterpretationTest(object): + + def serialize(self, func, values, backendoptimize=False): + # build the normal ll graphs for ll_function + t = TranslationContext() + a = t.buildannotator() + argtypes = getargtypes(a, values) + a.build_types(func, argtypes) + rtyper = t.buildrtyper(type_system = self.type_system) + rtyper.specialize() + self.rtyper = rtyper + if backendoptimize: + from pypy.translator.backendopt.all import backend_optimizations + backend_optimizations(t) + graph1 = graphof(t, func) + + # build hint annotator types + hannotator = HintAnnotator(base_translator=t, policy=P_OOPSPEC_NOVIRTUAL) + hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, + {OriginFlags(): True}) + for v in graph1.getargs()]) + hannotator.simplify() + t = hannotator.translator + self.hannotator = hannotator + if conftest.option.view: + t.view() + graph2 = graphof(t, func) + self.graph = graph2 + writer = BytecodeWriter(t, hannotator, RGenOp) + jitcode = writer.make_bytecode(graph2) + argcolors = [] + for i, ll_val in enumerate(values): + color = writer.varcolor(graph2.startblock.inputargs[i]) + argcolors.append(color) + return writer, jitcode, argcolors + + def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): + # XXX clean this mess up + writer, jitcode, argcolors = self.serialize(ll_function, values) + hrtyper = bytecode.PseudoHRTyper(RGenOp=writer.RGenOp, + annotator=writer.translator.annotator, + rtyper=writer.translator.annotator.base_translator.rtyper) + edesc = exception.ExceptionDesc(hrtyper, False) + rgenop = writer.RGenOp() + # make residual functype + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + ha = self.hannotator + RESTYPE = originalconcretetype(self.hannotator.binding(self.graph.getreturnvar())) + ARGS = [] + for var in self.graph.getargs(): + # XXX ignoring virtualizables for now + ARGS.append(originalconcretetype(self.hannotator.binding(var))) + FUNC = lltype.FuncType(ARGS, RESTYPE) + sigtoken = rgenop.sigToken(FUNC) + builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") + print builder, builder.rgenop, rgenop + builder.start_writing() + jitstate = rtimeshift.JITState(builder, None, + edesc.null_exc_type_box, + edesc.null_exc_value_box) + def ll_finish_jitstate(jitstate, exceptiondesc, graphsigtoken): + returnbox = rtimeshift.getreturnbox(jitstate) + gv_ret = returnbox.getgenvar(jitstate) + builder = jitstate.curbuilder + for virtualizable_box in jitstate.virtualizables: + assert isinstance(virtualizable_box, rvalue.PtrRedBox) + content = virtualizable_box.content + assert isinstance(content, rcontainer.VirtualizableStruct) + content.store_back(jitstate) + exceptiondesc.store_global_excdata(jitstate) + jitstate.curbuilder.finish_and_return(graphsigtoken, gv_ret) + # build arguments + greenargs = [] + redargs = [] + residualargs = [] + i = 0 + for color, ll_val in zip(argcolors, values): + if color == "green": + greenargs.append(writer.RGenOp.constPrebuiltGlobal(const.value)) + else: + TYPE = lltype.typeOf(ll_val) + kind = rgenop.kindToken(TYPE) + boxcls = rvalue.ll_redboxcls(TYPE) + redargs.append(boxcls(kind, inputargs_gv[i])) + residualargs.append(ll_val) + i += 1 + writer.interpreter.run(jitstate, jitcode, greenargs, redargs) + ll_finish_jitstate(writer.interpreter.jitstate, edesc, sigtoken) + builder.end() + generated = gv_generated.revealconst(lltype.Ptr(FUNC)) + graph = generated._obj.graph + if conftest.option.view: + graph.show() + llinterp = LLInterpreter(self.rtyper) + res = llinterp.eval_graph(graph, residualargs) + return res + + def Xtest_return_green(self): + def f(): + return 1 + self.interpret(f, []) + + def test_very_simple(self): + def f(x, y): + return x + y + res = self.interpret(f, [1, 2]) + assert res == 3 + +class TestLLType(AbstractInterpretationTest): + type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Fri Feb 1 13:47:40 2008 @@ -78,7 +78,7 @@ def ll_gen1(opdesc, jitstate, argbox): ARG0 = opdesc.ARG0 RESULT = opdesc.RESULT - opname = opdesc.name + opname = opdesc.opname if opdesc.tryfold and argbox.is_constant(): arg = rvalue.ll_getvalue(argbox, ARG0) if not opdesc.canraise: @@ -106,7 +106,7 @@ ARG0 = opdesc.ARG0 ARG1 = opdesc.ARG1 RESULT = opdesc.RESULT - opname = opdesc.name + opname = opdesc.opname if opdesc.tryfold and argbox0.is_constant() and argbox1.is_constant(): # const propagate arg0 = rvalue.ll_getvalue(argbox0, ARG0) @@ -991,6 +991,7 @@ self.backframe = backframe self.dispatchqueue = dispatchqueue #self.local_boxes = ... set by callers + #self.local_green = ... set by callers def enter_block(self, incoming, memo): for box in self.local_boxes: From cfbolz at codespeak.net Fri Feb 1 13:50:35 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 13:50:35 +0100 (CET) Subject: [pypy-svn] r51175 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080201125035.3B261168471@codespeak.net> Author: cfbolz Date: Fri Feb 1 13:50:34 2008 New Revision: 51175 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Log: reshuffle some metho Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Fri Feb 1 13:50:34 2008 @@ -59,53 +59,6 @@ if result is STOP: return - # construction-time interface - - def _add_implemented_opcodes(self): - for name in dir(self): - if not name.startswith("opimpl_"): - continue - opname = name[len("opimpl_"):] - self.opname_to_index[opname] = len(self.opcode_implementations) - self.opcode_implementations.append(getattr(self, name).im_func) - self.opcode_descs.append(None) - - def find_opcode(self, name): - return self.opname_to_index.get(name, -1) - - def make_opcode_implementation(self, color, opdesc): - numargs = unrolling_iterable(range(opdesc.nb_args)) - if color == "green": - def implementation(self): - args = () - for i in numargs: - genconst = self.get_greenarg() - arg = self.jitstate.curbuilder.revealconst(opdesc.ARGS[i]) - args += (arg, ) - result = opdesc.llop(*args) - self.green_result(result) - elif color == "red": - if opdesc.nb_args == 1: - impl = rtimeshift.ll_gen1 - elif opdesc.nb_args == 2: - impl = rtimeshift.ll_gen2 - else: - XXX - def implementation(self): - args = (opdesc, self.jitstate, ) - for i in numargs: - args += (self.get_redarg(), ) - result = impl(*args) - self.red_result(result) - else: - assert 0, "unknown color" - implementation.func_name = "opimpl_%s_%s" % (color, opdesc.opname) - opname = "%s_%s" % (color, opdesc.opname) - index = self.opname_to_index[opname] = len(self.opcode_implementations) - self.opcode_implementations.append(implementation) - self.opcode_descs.append(opdesc) - return index - # operation helper functions @@ -183,7 +136,53 @@ newgreens.append(self.get_greenarg()) self.frame.local_green = newgreens + # construction-time interface + + def _add_implemented_opcodes(self): + for name in dir(self): + if not name.startswith("opimpl_"): + continue + opname = name[len("opimpl_"):] + self.opname_to_index[opname] = len(self.opcode_implementations) + self.opcode_implementations.append(getattr(self, name).im_func) + self.opcode_descs.append(None) + def find_opcode(self, name): + return self.opname_to_index.get(name, -1) + + def make_opcode_implementation(self, color, opdesc): + numargs = unrolling_iterable(range(opdesc.nb_args)) + if color == "green": + def implementation(self): + args = () + for i in numargs: + genconst = self.get_greenarg() + arg = self.jitstate.curbuilder.revealconst(opdesc.ARGS[i]) + args += (arg, ) + result = opdesc.llop(*args) + self.green_result(result) + elif color == "red": + if opdesc.nb_args == 1: + impl = rtimeshift.ll_gen1 + elif opdesc.nb_args == 2: + impl = rtimeshift.ll_gen2 + else: + XXX + def implementation(self): + args = (opdesc, self.jitstate, ) + for i in numargs: + args += (self.get_redarg(), ) + result = impl(*args) + self.red_result(result) + else: + assert 0, "unknown color" + implementation.func_name = "opimpl_%s_%s" % (color, opdesc.opname) + opname = "%s_%s" % (color, opdesc.opname) + index = self.opname_to_index[opname] = len(self.opcode_implementations) + self.opcode_implementations.append(implementation) + self.opcode_descs.append(opdesc) + return index + class BytecodeWriter(object): From cfbolz at codespeak.net Fri Feb 1 14:11:25 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 14:11:25 +0100 (CET) Subject: [pypy-svn] r51177 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080201131125.35A03168477@codespeak.net> Author: cfbolz Date: Fri Feb 1 14:11:24 2008 New Revision: 51177 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: first test with a green switch passes Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Fri Feb 1 14:11:24 2008 @@ -102,7 +102,11 @@ self.frame.pc = target def opimpl_green_goto_iftrue(self): - XXX + genconst = self.get_greenarg() + target = self.load_4byte() + arg = genconst.revealconst(lltype.Bool) + if arg: + self.frame.pc = target def opimpl_red_goto_iftrue(self): XXX @@ -115,7 +119,10 @@ return STOP def opimpl_green_return(self): - XXX + rtimeshift.save_return(self.jitstate) + newstate = rtimeshift.leave_graph_yellow(self.queue) + self.jitstate = newstate + return STOP return STOP # XXX wrong, of course def opimpl_make_new_redvars(self): @@ -154,12 +161,13 @@ numargs = unrolling_iterable(range(opdesc.nb_args)) if color == "green": def implementation(self): - args = () + args = (opdesc.RESULT, ) for i in numargs: genconst = self.get_greenarg() - arg = self.jitstate.curbuilder.revealconst(opdesc.ARGS[i]) + arg = genconst.revealconst(opdesc.ARGS[i]) args += (arg, ) - result = opdesc.llop(*args) + rgenop = self.jitstate.curbuilder.rgenop + result = rgenop.genconst(opdesc.llop(*args)) self.green_result(result) elif color == "red": if opdesc.nb_args == 1: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 1 14:11:24 2008 @@ -11,6 +11,7 @@ from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.llinterp import LLInterpreter from pypy.annotation import model as annmodel +from pypy.rlib.jit import hint from pypy import conftest def getargtypes(annotator, values): @@ -79,7 +80,9 @@ ARGS = [] for var in self.graph.getargs(): # XXX ignoring virtualizables for now - ARGS.append(originalconcretetype(self.hannotator.binding(var))) + binding = self.hannotator.binding(var) + if not binding.is_green(): + ARGS.append(originalconcretetype(binding)) FUNC = lltype.FuncType(ARGS, RESTYPE) sigtoken = rgenop.sigToken(FUNC) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") @@ -106,7 +109,7 @@ i = 0 for color, ll_val in zip(argcolors, values): if color == "green": - greenargs.append(writer.RGenOp.constPrebuiltGlobal(const.value)) + greenargs.append(writer.RGenOp.constPrebuiltGlobal(ll_val)) else: TYPE = lltype.typeOf(ll_val) kind = rgenop.kindToken(TYPE) @@ -136,5 +139,16 @@ res = self.interpret(f, [1, 2]) assert res == 3 + def test_green_switch(self): + def f(green, x, y): + green = hint(green, concrete=True) + if green: + return x + y + return x - y + res = self.interpret(f, [1, 1, 2]) + assert res == 3 + res = self.interpret(f, [0, 1, 2]) + assert res == -1 + class TestLLType(AbstractInterpretationTest): type_system = "lltype" From antocuni at codespeak.net Fri Feb 1 14:31:26 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 1 Feb 2008 14:31:26 +0100 (CET) Subject: [pypy-svn] r51178 - pypy/dist/pypy/jit/codegen/dump/test Message-ID: <20080201133126.65C8D168471@codespeak.net> Author: antocuni Date: Fri Feb 1 14:31:24 2008 New Revision: 51178 Modified: pypy/dist/pypy/jit/codegen/dump/test/test_rgenop.py Log: fix test Modified: pypy/dist/pypy/jit/codegen/dump/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/dump/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/dump/test/test_rgenop.py Fri Feb 1 14:31:24 2008 @@ -2,8 +2,7 @@ from pypy.jit.codegen.dump.rgenop import RDumpGenOp from pypy.jit.timeshifter.test.test_timeshift import Whatever from pypy.rpython.lltypesystem import lltype -from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTests, FUNC, FUNC2 -from ctypes import cast, c_int, c_void_p, CFUNCTYPE +from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTests class TestRDumpGenop(AbstractRGenOpTests): RGenOp = RDumpGenOp From antocuni at codespeak.net Fri Feb 1 14:41:58 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 1 Feb 2008 14:41:58 +0100 (CET) Subject: [pypy-svn] r51179 - in pypy/dist/pypy/translator/cli: . src test Message-ID: <20080201134158.93CF4168052@codespeak.net> Author: antocuni Date: Fri Feb 1 14:41:58 2008 New Revision: 51179 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/query.py pypy/dist/pypy/translator/cli/src/pypylib.cs pypy/dist/pypy/translator/cli/src/query.cs pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: fix typeof of types defined in external assemblies Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 1 14:41:58 2008 @@ -495,8 +495,7 @@ def typeof(cliClass): TYPE = cliClass._INSTANCE - name = '%s.%s' % (TYPE._namespace, TYPE._classname) - return PythonNet.System.Type.GetType(name) + return PythonNet.System.Type.GetType(TYPE._assembly_qualified_name) class Entry(ExtRegistryEntry): _about_ = typeof Modified: pypy/dist/pypy/translator/cli/query.py ============================================================================== --- pypy/dist/pypy/translator/cli/query.py (original) +++ pypy/dist/pypy/translator/cli/query.py Fri Feb 1 14:41:58 2008 @@ -33,7 +33,8 @@ except ImportError: pass else: - Assembly.LoadFrom(dll) + ass = Assembly.LoadFrom(dll) + assert ass is not None clr.AddReference(pypylib) load_assembly(pypylib) @@ -96,6 +97,7 @@ desc = ClassDesc() desc.Assembly = mscorlib desc.FullName = name + desc.AssemblyQualifiedName = name # XXX desc.BaseType = 'System.Object' desc.IsArray = True desc.ElementType = 'System.Object' # not really true, but we need something @@ -105,6 +107,7 @@ desc = ClassDesc() desc.Assembly = mscorlib desc.FullName = name + desc.AssemblyQualifiedName = name # XXX desc.BaseType = 'System.Array' desc.ElementType = itemdesc.FullName desc.IsArray = True @@ -154,6 +157,7 @@ # no superclass for now, will add it later TYPE = NativeInstance(assembly, namespace, name, None, {}, {}) TYPE._is_value_type = self.IsValueType + TYPE._assembly_qualified_name = self.AssemblyQualifiedName Class = CliClass(TYPE, {}, {}) self._cliclass = Class # we need to check also for System.Array to prevent a circular recursion Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Fri Feb 1 14:41:58 2008 @@ -53,6 +53,7 @@ namespace pypy.runtime { + public delegate int DelegateType_int__int(int a); public delegate int DelegateType_int__int_int(int a, int b); public class Utils Modified: pypy/dist/pypy/translator/cli/src/query.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/query.cs (original) +++ pypy/dist/pypy/translator/cli/src/query.cs Fri Feb 1 14:41:58 2008 @@ -58,6 +58,7 @@ outfile.WriteLine("desc = ClassDesc()"); outfile.WriteLine("desc.Assembly = '{0}'", t.Assembly.FullName); outfile.WriteLine("desc.FullName = '{0}'", t.FullName); + outfile.WriteLine("desc.AssemblyQualifiedName = '{0}'", t.AssemblyQualifiedName); outfile.WriteLine("desc.BaseType = '{0}'", GetBaseType(t)); outfile.WriteLine("desc.IsArray = {0}", t.IsArray); outfile.WriteLine("desc.IsValueType = {0}", t.IsValueType); Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 1 14:41:58 2008 @@ -355,6 +355,13 @@ res = self.interpret(fn, []) assert res is True + def test_typeof_pypylib(self): + DelegateType = CLR.pypy.runtime.DelegateType_int__int_int + def fn(): + return typeof(DelegateType) is not None + res = self.interpret(fn, []) + assert res is True + def test_mix_None_and_instance(self): def g(x): return x From antocuni at codespeak.net Fri Feb 1 15:15:54 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 1 Feb 2008 15:15:54 +0100 (CET) Subject: [pypy-svn] r51182 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080201141554.D6499168440@codespeak.net> Author: antocuni Date: Fri Feb 1 15:15:52 2008 New Revision: 51182 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: add an helper to allow downcasts of cli objects Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 1 15:15:52 2008 @@ -528,6 +528,26 @@ c_methodname = hop.inputconst(ootype.Void, methodname) return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype) + +def clidowncast(cliClass, obj): + return obj + +class Entry(ExtRegistryEntry): + _about_ = clidowncast + + def compute_result_annotation(self, s_type, s_value): + assert s_type.is_constant() + cliClass = s_type.const + TYPE = cliClass._INSTANCE + assert ootype.isSubclass(TYPE, s_value.ootype) + return SomeOOInstance(TYPE) + + def specialize_call(self, hop): + assert isinstance(hop.args_s[0].const, CliClass) + assert isinstance(hop.args_s[1], annmodel.SomeOOInstance) + v_inst = hop.inputarg(hop.args_r[1], arg=1) + return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) + from pypy.translator.cli.query import CliNamespace CLR = CliNamespace(None) CLR._buildtree() Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 1 15:15:52 2008 @@ -7,7 +7,7 @@ from pypy.translator.cli.test.runtest import CliTest from pypy.translator.cli.dotnet import SomeCliClass, SomeCliStaticMethod,\ NativeInstance, CLR, box, unbox, OverloadingResolver, NativeException,\ - native_exc, new_array, init_array, typeof, eventhandler + native_exc, new_array, init_array, typeof, eventhandler, clidowncast System = CLR.System ArrayList = CLR.System.Collections.ArrayList @@ -362,6 +362,18 @@ res = self.interpret(fn, []) assert res is True + def test_clidowncast(self): + def fn(): + a = ArrayList() + b = ArrayList() + a.Add(b) + c = a.get_Item(0) # type of c is Object + c = clidowncast(ArrayList, c) + c.Add(None) + return c.get_Item(0) + res = self.interpret(fn, []) + assert res is None + def test_mix_None_and_instance(self): def g(x): return x From antocuni at codespeak.net Fri Feb 1 15:17:05 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 1 Feb 2008 15:17:05 +0100 (CET) Subject: [pypy-svn] r51183 - pypy/dist/pypy/translator/cli/test Message-ID: <20080201141705.1C44916845F@codespeak.net> Author: antocuni Date: Fri Feb 1 15:17:05 2008 New Revision: 51183 Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: fix test to work also when interpreted Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 1 15:17:05 2008 @@ -469,9 +469,7 @@ def test_dynamic_method(self): from pypy.rpython.ootypesystem import ootype - self._skip_pythonnet("does not work") DelegateType = CLR.pypy.runtime.DelegateType_int__int_int - DELEGATETYPE = DelegateType._INSTANCE Utils = CLR.pypy.runtime.Utils def fn(): tInt = typeof(System.Int32) @@ -483,7 +481,7 @@ il.Emit(OpCodes.Add) il.Emit(OpCodes.Ret) myfunc = meth.CreateDelegate(typeof(DelegateType)) - myfunc = ootype.oodowncast(DELEGATETYPE, myfunc) # XXX messy + myfunc = clidowncast(DelegateType, myfunc) return myfunc.Invoke(30, 12) res = self.interpret(fn, []) assert res == 42 From cfbolz at codespeak.net Fri Feb 1 15:20:44 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 15:20:44 +0100 (CET) Subject: [pypy-svn] r51184 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080201142044.299FE168464@codespeak.net> Author: cfbolz Date: Fri Feb 1 15:20:43 2008 New Revision: 51184 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: converting green things into red things works Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Fri Feb 1 15:20:43 2008 @@ -18,16 +18,20 @@ green consts are negative indexes """ - def __init__(self, code, constants): + def __init__(self, code, constants, typekinds, redboxclasses): self.code = code self.constants = constants + self.typekinds = typekinds + self.redboxclasses = redboxclasses def _freeze_(self): return True SIGN_EXTEND2 = 1 << 15 -STOP = object() +class STOP(object): + pass +STOP = STOP() class JitInterpreter(object): def __init__(self): @@ -58,6 +62,8 @@ result = self.opcode_implementations[bytecode](self) if result is STOP: return + else: + assert result is None # operation helper functions @@ -95,7 +101,11 @@ # operation implementations def opimpl_make_redbox(self): - XXX + genconst = self.get_greenarg() + typeindex = self.load_2byte() + kind = self.frame.bytecode.typekinds[typeindex] + redboxcls = self.frame.bytecode.redboxclasses[typeindex] + self.red_result(redboxcls(kind, genconst)) def opimpl_goto(self): target = self.load_4byte() @@ -123,7 +133,6 @@ newstate = rtimeshift.leave_graph_yellow(self.queue) self.jitstate = newstate return STOP - return STOP # XXX wrong, of course def opimpl_make_new_redvars(self): # an opcode with a variable number of args @@ -206,18 +215,29 @@ self.seen_blocks = {} self.assembler = [] self.constants = [] + self.typekinds = [] + self.redboxclasses = [] + # mapping constant -> index in constants self.const_positions = {} + # mapping blocks to True self.seen_blocks = {} - self.additional_positions = {} self.redvar_positions = {} + # mapping block to the free red index self.free_red = {} self.greenvar_positions = {} + # mapping block to the free green index self.free_green = {} + # mapping TYPE to index + self.type_positions = {} + self.graph = graph self.entrymap = flowmodel.mkentrymap(graph) self.make_bytecode_block(graph.startblock) assert self.current_block is None - return JitCode(assemble(self.interpreter, *self.assembler), self.constants) + return JitCode(assemble(self.interpreter, *self.assembler), + self.constants, + self.typekinds, + self.redboxclasses) def make_bytecode_block(self, block, insert_goto=False): if block in self.seen_blocks: @@ -330,6 +350,7 @@ resultindex = self.register_redvar((arg, block)) argindex = self.green_position(arg) self.emit(argindex) + self.emit(self.type_position(arg.concretetype)) return resultindex def opcolor(self, op): @@ -377,6 +398,15 @@ self.constants.append(const) self.const_positions[const] = result return result + + def type_position(self, TYPE): + if TYPE in self.type_positions: + return self.type_positions[TYPE] + self.typekinds.append(self.RGenOp.kindToken(TYPE)) + self.redboxclasses.append(rvalue.ll_redboxcls(TYPE)) + result = len(self.type_positions) + self.type_positions[TYPE] = result + return result def emit(self, stuff): assert stuff is not None Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 1 15:20:43 2008 @@ -139,6 +139,12 @@ res = self.interpret(f, [1, 2]) assert res == 3 + def test_convert_const_to_red(self): + def f(x): + return x + 1 + res = self.interpret(f, [2]) + assert res == 3 + def test_green_switch(self): def f(green, x, y): green = hint(green, concrete=True) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Fri Feb 1 15:20:43 2008 @@ -63,18 +63,21 @@ "make_new_greenvars", 0, "red_return", 0) assert len(jitcode.constants) == 0 + assert len(jitcode.typekinds) == 0 def test_constant(self): def f(x): return x + 1 writer, jitcode = self.serialize(f, [int]) assert jitcode.code == assemble(writer.interpreter, - "make_redbox", -1, + "make_redbox", -1, 0, "red_int_add", 0, 1, "make_new_redvars", 1, 2, "make_new_greenvars", 0, "red_return", 0) assert len(jitcode.constants) == 1 + assert len(jitcode.typekinds) == 1 + assert len(jitcode.redboxclasses) == 1 def test_green_switch(self): def f(x, y, z): @@ -98,6 +101,7 @@ ) assert jitcode.code == expected assert len(jitcode.constants) == 0 + assert len(jitcode.typekinds) == 0 def test_green_switch2(self): def f(x, y, z): @@ -129,7 +133,7 @@ ) assert jitcode.code == expected assert len(jitcode.constants) == 0 - + assert len(jitcode.typekinds) == 0 class TestLLType(AbstractSerializationTest): From cfbolz at codespeak.net Fri Feb 1 15:27:02 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 15:27:02 +0100 (CET) Subject: [pypy-svn] r51185 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080201142702.85514168465@codespeak.net> Author: cfbolz Date: Fri Feb 1 15:27:01 2008 New Revision: 51185 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: a bit bigger test that contains mostly green vars Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 1 15:27:01 2008 @@ -156,5 +156,23 @@ res = self.interpret(f, [0, 1, 2]) assert res == -1 + def test_arith_plus_minus(self): + def ll_plus_minus(encoded_insn, nb_insn, x, y): + acc = x + pc = 0 + hint(nb_insn, concrete=True) + while pc < nb_insn: + op = (encoded_insn >> (pc*4)) & 0xF + op = hint(op, concrete=True) + if op == 0xA: + acc += y + elif op == 0x5: + acc -= y + pc += 1 + return acc + assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 + res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10]) + assert res == 42 + class TestLLType(AbstractInterpretationTest): type_system = "lltype" From exarkun at codespeak.net Fri Feb 1 16:02:11 2008 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 1 Feb 2008 16:02:11 +0100 (CET) Subject: [pypy-svn] r51187 - pypy/build/buildbot Message-ID: <20080201150211.A44A7168465@codespeak.net> Author: exarkun Date: Fri Feb 1 16:02:10 2008 New Revision: 51187 Modified: pypy/build/buildbot/master.cfg Log: re-enable pypy tests on 64bit slave Modified: pypy/build/buildbot/master.cfg ============================================================================== --- pypy/build/buildbot/master.cfg (original) +++ pypy/build/buildbot/master.cfg Fri Feb 1 16:02:10 2008 @@ -55,7 +55,7 @@ {"name": "pypy-c-allworkingmodules-faassen-64", "slavenames": ["linux-dvs0"], "builddir": "pypy-c-allworkingmodules-faassen-64", - "factory": PyPyBuildFactory(None, + "factory": PyPyBuildFactory(["--boxed"], [], ["--allworkingmodules", "--faassen"])}, From cfbolz at codespeak.net Sat Feb 2 02:44:12 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 02:44:12 +0100 (CET) Subject: [pypy-svn] r51196 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080202014412.83C671683EE@codespeak.net> Author: cfbolz Date: Sat Feb 2 02:44:11 2008 New Revision: 51196 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: implement red switches Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sat Feb 2 02:44:11 2008 @@ -55,6 +55,7 @@ self.frame.local_boxes = redargs self.frame.local_green = greenargs self.bytecode_loop() + return self.jitstate def bytecode_loop(self): while 1: @@ -70,6 +71,7 @@ def load_2byte(self): pc = self.frame.pc + assert pc >= 0 result = ((ord(self.frame.bytecode.code[pc]) << 8) | ord(self.frame.bytecode.code[pc + 1])) self.frame.pc = pc + 2 @@ -77,6 +79,7 @@ def load_4byte(self): pc = self.frame.pc + assert pc >= 0 result = ((ord(self.frame.bytecode.code[pc + 0]) << 24) | (ord(self.frame.bytecode.code[pc + 1]) << 16) | (ord(self.frame.bytecode.code[pc + 2]) << 8) | @@ -99,6 +102,13 @@ def green_result(self, gv): self.frame.local_green.append(gv) + def newjitstate(self, newjitstate): + self.jitstate = newjitstate + if newjitstate is not None: + self.frame = newjitstate.frame + else: + self.frame = None + # operation implementations def opimpl_make_redbox(self): genconst = self.get_greenarg() @@ -119,14 +129,24 @@ self.frame.pc = target def opimpl_red_goto_iftrue(self): - XXX + switchbox = self.get_redarg() + target = self.load_4byte() + # XXX not sure about passing no green vars + descision = rtimeshift.split(self.jitstate, switchbox, self.frame.pc) + if descision: + self.frame.pc = target def opimpl_red_return(self): rtimeshift.save_return(self.jitstate) - # XXX for now - newstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) - self.jitstate = newstate - return STOP + newjitstate = rtimeshift.dispatch_next(self.queue) + resumepoint = rtimeshift.getresumepoint(newjitstate) + if resumepoint == -1: + # XXX for now + newjitstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) + self.newjitstate(newjitstate) + return STOP + self.newjitstate(newjitstate) + self.frame.pc = resumepoint def opimpl_green_return(self): rtimeshift.save_return(self.jitstate) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 02:44:11 2008 @@ -117,8 +117,9 @@ redargs.append(boxcls(kind, inputargs_gv[i])) residualargs.append(ll_val) i += 1 - writer.interpreter.run(jitstate, jitcode, greenargs, redargs) - ll_finish_jitstate(writer.interpreter.jitstate, edesc, sigtoken) + jitstate = writer.interpreter.run(jitstate, jitcode, greenargs, redargs) + if jitstate is not None: + ll_finish_jitstate(jitstate, edesc, sigtoken) builder.end() generated = gv_generated.revealconst(lltype.Ptr(FUNC)) graph = generated._obj.graph @@ -174,5 +175,13 @@ res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10]) assert res == 42 + def test_red_switch(self): + def f(x, y): + if x: + return x + return y + res = self.interpret(f, [1, 2]) + + class TestLLType(AbstractInterpretationTest): type_system = "lltype" From xoraxax at codespeak.net Sat Feb 2 12:57:38 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 2 Feb 2008 12:57:38 +0100 (CET) Subject: [pypy-svn] r51201 - pypy/dist/pypy/tool Message-ID: <20080202115738.964D9168411@codespeak.net> Author: xoraxax Date: Sat Feb 2 12:57:36 2008 New Revision: 51201 Modified: pypy/dist/pypy/tool/cache.py Log: Add a repr for the Cache class. Modified: pypy/dist/pypy/tool/cache.py ============================================================================== --- pypy/dist/pypy/tool/cache.py (original) +++ pypy/dist/pypy/tool/cache.py Sat Feb 2 12:57:36 2008 @@ -58,6 +58,9 @@ if lock: lock.release() getorbuild._annspecialcase_ = "specialize:memo" + def __repr__(self): + return "" % (self.__class__.__name__, len(self.content)) + def _ready(self, result): pass From xoraxax at codespeak.net Sat Feb 2 12:58:44 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 2 Feb 2008 12:58:44 +0100 (CET) Subject: [pypy-svn] r51202 - pypy/dist/pypy/rpython Message-ID: <20080202115844.A3D17168411@codespeak.net> Author: xoraxax Date: Sat Feb 2 12:58:44 2008 New Revision: 51202 Modified: pypy/dist/pypy/rpython/rpbc.py Log: Remove confusing getattr test (which does not really make any sense IMHO) in the rpbc code that generates the Desc warning. Modified: pypy/dist/pypy/rpython/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/rpbc.py (original) +++ pypy/dist/pypy/rpython/rpbc.py Sat Feb 2 12:58:44 2008 @@ -14,6 +14,7 @@ from pypy.rpython import callparse + def small_cand(rtyper, s_pbc): if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets and \ hasattr(rtyper.type_system.rpbc, 'SmallFunctionSetPBCRepr'): @@ -482,10 +483,7 @@ try: thisattrvalue = frozendesc.attrcache[attr] except KeyError: - # don't issue warning if this attribute can be read, but - # is not used - if not hasattr(frozendesc.pyobj, attr): - warning("Desc %r has no attribute %r" % (frozendesc, attr)) + warning("Desc %r has no attribute %r" % (frozendesc, attr)) continue llvalue = r_value.convert_const(thisattrvalue) setattr(result, mangled_name, llvalue) From cfbolz at codespeak.net Sat Feb 2 13:06:15 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 13:06:15 +0100 (CET) Subject: [pypy-svn] r51203 - pypy/branch/jit-refactoring/pypy/jit/timeshifter Message-ID: <20080202120615.97747168437@codespeak.net> Author: cfbolz Date: Sat Feb 2 13:06:14 2008 New Revision: 51203 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: oops, forgot to check this in yesterday Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Sat Feb 2 13:06:14 2008 @@ -992,6 +992,8 @@ self.dispatchqueue = dispatchqueue #self.local_boxes = ... set by callers #self.local_green = ... set by callers + #self.pc = ... set by callers + #self.bytecode = ... set by callers def enter_block(self, incoming, memo): for box in self.local_boxes: @@ -1014,6 +1016,9 @@ newbackframe = self.backframe.copy(memo) result = VirtualFrame(newbackframe, self.dispatchqueue) result.local_boxes = [box.copy(memo) for box in self.local_boxes] + result.pc = self.pc + result.bytecode = self.bytecode + result.local_green = self.local_green[:] return result def replace(self, memo): From cfbolz at codespeak.net Sat Feb 2 13:11:27 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 13:11:27 +0100 (CET) Subject: [pypy-svn] r51204 - in pypy/branch/jit-refactoring/pypy/jit/timeshifter: . test Message-ID: <20080202121127.DBDE8168436@codespeak.net> Author: cfbolz Date: Sat Feb 2 13:11:27 2008 New Revision: 51204 Added: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (contents, props changed) pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py (contents, props changed) Log: add a key wrapper class to be able to use lists of genconsts in the states_dicts as keys Added: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Sat Feb 2 13:11:27 2008 @@ -0,0 +1,72 @@ +from pypy.rpython.annlowlevel import cachedtype +from pypy.rlib.rarithmetic import intmask +from pypy.rlib.unroll import unrolling_iterable +from pypy.rpython.lltypesystem import lltype + +class KeyDesc(object): + __metaclass__ = cachedtype + + def __init__(self, RGenOp, *TYPES): + self.RGenOp = RGenOp + self.TYPES = TYPES + TARGETTYPES = [] + for TYPE in TYPES: + # XXX more cases? + TARGET = lltype.Signed + if TYPE == lltype.Float: + TARGET = TYPE + TARGETTYPES.append(TARGET) + + iterator = unrolling_iterable(enumerate(TARGETTYPES)) + length = len(TYPES) + def greenhash(self, rgenop): + retval = 0x345678 + mult = 1000003 + for i, TARGET in iterator: + genconst = self.values[i] + item = genconst.revealconst(TARGET) + retval = intmask((retval ^ hash(item)) * intmask(mult)) + mult = mult + 82520 + 2*length + return retval + self.hash = greenhash + def greencompare(self, other, rgenop): + for i, TARGET in iterator: + genconst = self.values[i] + item_self = genconst.revealconst(TARGET) + genconst = other.values[i] + item_other = genconst.revealconst(TARGET) + if item_self != item_other: + return False + return True + self.compare = greencompare + + def _freeze_(self): + return True + + +class GreenKey(object): + def __init__(self, values, desc, rgenop): + self.desc = desc + self.values = values + self.rgenop = rgenop + + def __eq__(self, other): + raise TypeError("don't store GreenKeys in a normal dict") + + def __ne__(self, other): + raise TypeError("don't store GreenKeys in a normal dict") + + def __hash__(self): + raise TypeError("not hashable") + +def greenkey_eq(self, other): + assert self.rgenop is other.rgenop + if self is other: + return True + if self.desc is not other.desc: + return False + return self.desc.compare(self, other, self.rgenop) + +def greenkey_hash(self): + return self.desc.hash(self, self.rgenop) + Added: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Sat Feb 2 13:11:27 2008 @@ -0,0 +1,30 @@ +from pypy.jit.timeshifter.greenkey import GreenKey, KeyDesc +from pypy.jit.timeshifter.greenkey import greenkey_hash, greenkey_eq +from pypy.jit.codegen.llgraph.rgenop import RGenOp +from pypy.rlib.objectmodel import r_dict +from pypy.rpython.lltypesystem import lltype + +rgenop = RGenOp() + +class TestGreenKeys(object): + def newdict(self): + return r_dict(greenkey_eq, greenkey_hash) + + def newkey(self, *values): + desc = KeyDesc(RGenOp, *[lltype.typeOf(val) for val in values]) + return GreenKey([rgenop.genconst(val) for val in values], desc, rgenop) + + def test_simple(self): + d = self.newdict() + d[self.newkey(1, 2)] = 1 + assert d[self.newkey(1, 2)] == 1 + + def test_check_types(self): + d = self.newdict() + d[self.newkey(1, 2)] = 1 + assert self.newkey(True, 2) not in d + + def test_check_lengths(self): + d = self.newdict() + d[self.newkey(1, 2)] = 1 + assert self.newkey(1, 2, 0) not in d From xoraxax at codespeak.net Sat Feb 2 13:16:33 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 2 Feb 2008 13:16:33 +0100 (CET) Subject: [pypy-svn] r51205 - in pypy/dist/pypy: annotation tool/algo tool/algo/test Message-ID: <20080202121633.650771683EE@codespeak.net> Author: xoraxax Date: Sat Feb 2 13:16:31 2008 New Revision: 51205 Added: pypy/dist/pypy/tool/algo/test/test_unionfind.py Modified: pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/tool/algo/unionfind.py Log: Fix the mysterious RTyper warnings. Before, the memo code was registering a callback on the finish method of every ever created MemoTable instance. But the union find code was merging these tables together. Now the callback is ignored if it is called for a dominated MemoTable, i.e. one that was merged into another one. The accompanied unittest tests the new behaviour of the unionfind code, I do not know how to test the real issue reproducibly. The result of this checkin is that translation runs show only a single RTyper warning of this kind: [rtyper:WARNING] Desc > has no attribute '$memofield_getorbuild_0' Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Sat Feb 2 13:16:31 2008 @@ -96,11 +96,19 @@ self.funcdesc = funcdesc self.table = {args: value} self.graph = None + self.do_not_process = False + + def register_finish(self): + bookkeeper = self.funcdesc.bookkeeper + bookkeeper.pending_specializations.append(self.finish) def update(self, other): self.table.update(other.table) self.graph = None # just in case + def cleanup(self): + self.do_not_process = True + fieldnamecounter = 0 def getuniquefieldname(self): @@ -110,6 +118,8 @@ return fieldname def finish(self): + if self.do_not_process: + return from pypy.annotation.model import unionof assert self.graph is None, "MemoTable already finished" # list of which argument positions can take more than one value @@ -273,7 +283,7 @@ def compute_one_result(args): value = func(*args) memotable = MemoTable(funcdesc, args, value) - bookkeeper.pending_specializations.append(memotable.finish) + memotable.register_finish() return memotable memotables = UnionFind(compute_one_result) Added: pypy/dist/pypy/tool/algo/test/test_unionfind.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/algo/test/test_unionfind.py Sat Feb 2 13:16:31 2008 @@ -0,0 +1,26 @@ +from pypy.tool.algo.unionfind import UnionFind + + +def test_cleanup(): + state = [] + class ReferencedByExternalState(object): + def __init__(self, obj): + state.append(self) + self.obj = obj + + def update(self, other): + pass + + def cleanup(self): + state.remove(self) + + uf = UnionFind(ReferencedByExternalState) + uf.find(1) + for i in xrange(1, 10, 2): + uf.union(i, 1) + uf.find(2) + for i in xrange(2, 20, 2): + uf.union(i, 2) + assert len(state) < 3 + + Modified: pypy/dist/pypy/tool/algo/unionfind.py ============================================================================== --- pypy/dist/pypy/tool/algo/unionfind.py (original) +++ pypy/dist/pypy/tool/algo/unionfind.py Sat Feb 2 13:16:31 2008 @@ -87,6 +87,8 @@ self.link_to_parent[rep2] = rep1 del self.weight[rep2] + if hasattr(info2, "cleanup"): + info2.cleanup() del self.root_info[rep2] self.weight[rep1] = w From cfbolz at codespeak.net Sat Feb 2 13:27:18 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 13:27:18 +0100 (CET) Subject: [pypy-svn] r51206 - in pypy/branch/jit-refactoring/pypy/jit/timeshifter: . test Message-ID: <20080202122718.1D432168439@codespeak.net> Author: cfbolz Date: Sat Feb 2 13:27:17 2008 New Revision: 51206 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Log: put a newgreendict function in the main file Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Sat Feb 2 13:27:17 2008 @@ -1,6 +1,7 @@ from pypy.rpython.annlowlevel import cachedtype from pypy.rlib.rarithmetic import intmask from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.objectmodel import r_dict from pypy.rpython.lltypesystem import lltype class KeyDesc(object): @@ -70,3 +71,7 @@ def greenkey_hash(self): return self.desc.hash(self, self.rgenop) +def newgreendict(): + return r_dict(greenkey_eq, greenkey_hash) + + Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Sat Feb 2 13:27:17 2008 @@ -1,30 +1,26 @@ -from pypy.jit.timeshifter.greenkey import GreenKey, KeyDesc +from pypy.jit.timeshifter.greenkey import GreenKey, KeyDesc, newgreendict from pypy.jit.timeshifter.greenkey import greenkey_hash, greenkey_eq from pypy.jit.codegen.llgraph.rgenop import RGenOp -from pypy.rlib.objectmodel import r_dict from pypy.rpython.lltypesystem import lltype rgenop = RGenOp() class TestGreenKeys(object): - def newdict(self): - return r_dict(greenkey_eq, greenkey_hash) - def newkey(self, *values): desc = KeyDesc(RGenOp, *[lltype.typeOf(val) for val in values]) return GreenKey([rgenop.genconst(val) for val in values], desc, rgenop) def test_simple(self): - d = self.newdict() + d = newgreendict() d[self.newkey(1, 2)] = 1 assert d[self.newkey(1, 2)] == 1 def test_check_types(self): - d = self.newdict() + d = newgreendict() d[self.newkey(1, 2)] = 1 assert self.newkey(True, 2) not in d def test_check_lengths(self): - d = self.newdict() + d = newgreendict() d[self.newkey(1, 2)] = 1 assert self.newkey(1, 2, 0) not in d From cfbolz at codespeak.net Sat Feb 2 13:34:40 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 13:34:40 +0100 (CET) Subject: [pypy-svn] r51207 - in pypy/branch/jit-refactoring/pypy/jit/timeshifter: . test Message-ID: <20080202123440.0E1AA1683FB@codespeak.net> Author: cfbolz Date: Sat Feb 2 13:34:40 2008 New Revision: 51207 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Log: some cleanups, introduce a special empty key Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Sat Feb 2 13:34:40 2008 @@ -7,10 +7,16 @@ class KeyDesc(object): __metaclass__ = cachedtype - def __init__(self, RGenOp, *TYPES): + def __init__(self, RGenOp=None, *TYPES): self.RGenOp = RGenOp self.TYPES = TYPES TARGETTYPES = [] + + if RGenOp is None: + assert len(TYPES) == 0 + self.hash = lambda self: 0 + self.compare = lambda self, other: True + for TYPE in TYPES: # XXX more cases? TARGET = lltype.Signed @@ -20,7 +26,7 @@ iterator = unrolling_iterable(enumerate(TARGETTYPES)) length = len(TYPES) - def greenhash(self, rgenop): + def greenhash(self): retval = 0x345678 mult = 1000003 for i, TARGET in iterator: @@ -30,7 +36,7 @@ mult = mult + 82520 + 2*length return retval self.hash = greenhash - def greencompare(self, other, rgenop): + def greencompare(self, other): for i, TARGET in iterator: genconst = self.values[i] item_self = genconst.revealconst(TARGET) @@ -46,10 +52,9 @@ class GreenKey(object): - def __init__(self, values, desc, rgenop): + def __init__(self, values, desc): self.desc = desc self.values = values - self.rgenop = rgenop def __eq__(self, other): raise TypeError("don't store GreenKeys in a normal dict") @@ -61,17 +66,16 @@ raise TypeError("not hashable") def greenkey_eq(self, other): - assert self.rgenop is other.rgenop if self is other: return True if self.desc is not other.desc: return False - return self.desc.compare(self, other, self.rgenop) + return self.desc.compare(self, other) def greenkey_hash(self): - return self.desc.hash(self, self.rgenop) + return self.desc.hash(self) def newgreendict(): return r_dict(greenkey_eq, greenkey_hash) - +empty_key = GreenKey([], KeyDesc()) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Sat Feb 2 13:34:40 2008 @@ -1,5 +1,5 @@ from pypy.jit.timeshifter.greenkey import GreenKey, KeyDesc, newgreendict -from pypy.jit.timeshifter.greenkey import greenkey_hash, greenkey_eq +from pypy.jit.timeshifter.greenkey import greenkey_hash, greenkey_eq, empty_key from pypy.jit.codegen.llgraph.rgenop import RGenOp from pypy.rpython.lltypesystem import lltype @@ -8,7 +8,7 @@ class TestGreenKeys(object): def newkey(self, *values): desc = KeyDesc(RGenOp, *[lltype.typeOf(val) for val in values]) - return GreenKey([rgenop.genconst(val) for val in values], desc, rgenop) + return GreenKey([rgenop.genconst(val) for val in values], desc) def test_simple(self): d = newgreendict() @@ -24,3 +24,10 @@ d = newgreendict() d[self.newkey(1, 2)] = 1 assert self.newkey(1, 2, 0) not in d + + def test_empty_key(self): + d = newgreendict() + assert empty_key not in d + d[empty_key] = 1 + assert d[empty_key] == 1 + assert self.newkey(1, 2, 0) not in d From xoraxax at codespeak.net Sat Feb 2 13:37:35 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 2 Feb 2008 13:37:35 +0100 (CET) Subject: [pypy-svn] r51208 - pypy/dist/pypy/tool/algo/test Message-ID: <20080202123735.0FFE61683FB@codespeak.net> Author: xoraxax Date: Sat Feb 2 13:37:34 2008 New Revision: 51208 Modified: pypy/dist/pypy/tool/algo/test/test_unionfind.py Log: Make the assert in the union find test stricter. Modified: pypy/dist/pypy/tool/algo/test/test_unionfind.py ============================================================================== --- pypy/dist/pypy/tool/algo/test/test_unionfind.py (original) +++ pypy/dist/pypy/tool/algo/test/test_unionfind.py Sat Feb 2 13:37:34 2008 @@ -21,6 +21,6 @@ uf.find(2) for i in xrange(2, 20, 2): uf.union(i, 2) - assert len(state) < 3 + assert len(state) == 2 # we have exactly 2 partitions From cfbolz at codespeak.net Sat Feb 2 16:29:42 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 16:29:42 +0100 (CET) Subject: [pypy-svn] r51211 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080202152942.E1CB116845B@codespeak.net> Author: cfbolz Date: Sat Feb 2 16:29:41 2008 New Revision: 51211 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: Implement (local) merge points in the rainbow interpreter. This breaks many (most?) existing jit tests because some details in rtimeshift.py changed. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sat Feb 2 16:29:41 2008 @@ -1,9 +1,10 @@ from pypy.rlib.rarithmetic import intmask +from pypy.rlib.unroll import unrolling_iterable from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.timeshifter import rtimeshift, rvalue -from pypy.rlib.unroll import unrolling_iterable +from pypy.jit.timeshifter.greenkey import KeyDesc, empty_key, GreenKey class JitCode(object): """ @@ -18,11 +19,14 @@ green consts are negative indexes """ - def __init__(self, code, constants, typekinds, redboxclasses): + def __init__(self, code, constants, typekinds, redboxclasses, keydescs, + num_mergepoints): self.code = code self.constants = constants self.typekinds = typekinds self.redboxclasses = redboxclasses + self.keydescs = keydescs + self.num_mergepoints = num_mergepoints def _freeze_(self): return True @@ -46,8 +50,7 @@ def run(self, jitstate, bytecode, greenargs, redargs): self.jitstate = jitstate - self.queue = rtimeshift.ensure_queue(jitstate, - rtimeshift.BaseDispatchQueue) + self.queue = rtimeshift.DispatchQueue(bytecode.num_mergepoints) rtimeshift.enter_frame(self.jitstate, self.queue) self.frame = self.jitstate.frame self.frame.pc = 0 @@ -66,6 +69,17 @@ else: assert result is None + def dispatch(self): + newjitstate = rtimeshift.dispatch_next(self.queue) + resumepoint = rtimeshift.getresumepoint(newjitstate) + self.newjitstate(newjitstate) + if resumepoint == -1: + # XXX what about green returns? + newjitstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) + self.newjitstate(newjitstate) + return STOP + else: + self.frame.pc = resumepoint # operation helper functions @@ -138,21 +152,10 @@ def opimpl_red_return(self): rtimeshift.save_return(self.jitstate) - newjitstate = rtimeshift.dispatch_next(self.queue) - resumepoint = rtimeshift.getresumepoint(newjitstate) - if resumepoint == -1: - # XXX for now - newjitstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) - self.newjitstate(newjitstate) - return STOP - self.newjitstate(newjitstate) - self.frame.pc = resumepoint + return self.dispatch() def opimpl_green_return(self): - rtimeshift.save_return(self.jitstate) - newstate = rtimeshift.leave_graph_yellow(self.queue) - self.jitstate = newstate - return STOP + XXX def opimpl_make_new_redvars(self): # an opcode with a variable number of args @@ -167,11 +170,28 @@ # an opcode with a variable number of args # num_args arg_old_1 arg_new_1 ... num = self.load_2byte() + if num == 0 and len(self.frame.local_green) == 0: + # fast (very common) case + return newgreens = [] for i in range(num): newgreens.append(self.get_greenarg()) self.frame.local_green = newgreens + def opimpl_merge(self): + mergepointnum = self.load_2byte() + keydescnum = self.load_2byte() + if keydescnum == -1: + key = empty_key + else: + keydesc = self.frame.bytecode.keydescs[keydescnum] + key = GreenKey(self.frame.local_green[:keydesc.nb_vals], keydesc) + states_dic = self.queue.local_caches[mergepointnum] + done = rtimeshift.retrieve_jitstate_for_merge(states_dic, self.jitstate, + key, None) + if done: + return self.dispatch() + # construction-time interface def _add_implemented_opcodes(self): @@ -219,7 +239,8 @@ self.opcode_implementations.append(implementation) self.opcode_descs.append(opdesc) return index - + + class BytecodeWriter(object): @@ -237,6 +258,7 @@ self.constants = [] self.typekinds = [] self.redboxclasses = [] + self.keydescs = [] # mapping constant -> index in constants self.const_positions = {} # mapping blocks to True @@ -249,6 +271,10 @@ self.free_green = {} # mapping TYPE to index self.type_positions = {} + # mapping tuple of green TYPES to index + self.keydesc_positions = {} + + self.num_mergepoints = 0 self.graph = graph self.entrymap = flowmodel.mkentrymap(graph) @@ -257,7 +283,9 @@ return JitCode(assemble(self.interpreter, *self.assembler), self.constants, self.typekinds, - self.redboxclasses) + self.redboxclasses, + self.keydescs, + self.num_mergepoints) def make_bytecode_block(self, block, insert_goto=False): if block in self.seen_blocks: @@ -271,16 +299,16 @@ self.free_green[block] = 0 self.free_red[block] = 0 self.current_block = block + self.emit(label(block)) reds, greens = self.sort_by_color(block.inputargs) for arg in reds: self.register_redvar(arg) for arg in greens: self.register_greenvar(arg) - #self.insert_merges(block) + self.insert_merges(block) for op in block.operations: self.serialize_op(op) - #self.insert_splits(block) self.insert_exits(block) self.current_block = oldblock @@ -288,6 +316,7 @@ if block.exits == (): returnvar, = block.inputargs color = self.varcolor(returnvar) + assert color == "red" # XXX green return values not supported yet index = self.serialize_oparg(color, returnvar) self.emit("%s_return" % color) self.emit(index) @@ -312,6 +341,30 @@ else: XXX + def insert_merges(self, block): + if block is self.graph.returnblock: + return + if len(self.entrymap[block]) <= 1: + return + num = self.num_mergepoints + self.num_mergepoints += 1 + # make keydesc + key = () + for arg in self.sort_by_color(block.inputargs)[1]: + TYPE = arg.concretetype + key += (TYPE, ) + if not key: + keyindex = -1 # use prebuilt empty_key + elif key not in self.keydesc_positions: + keyindex = len(self.keydesc_positions) + self.keydesc_positions[key] = keyindex + self.keydescs.append(KeyDesc(self.RGenOp, *key)) + else: + keyindex = self.keydesc_positions[key] + self.emit("merge") + self.emit(num) + self.emit(keyindex) + def insert_renaming(self, args): reds, greens = self.sort_by_color(args) for color, args in [("red", reds), ("green", greens)]: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 16:29:41 2008 @@ -181,7 +181,17 @@ return x return y res = self.interpret(f, [1, 2]) + assert res == 1 + def test_merge(self): + def f(x, y, z): + if x: + a = y - z + else: + a = y + z + return 1 + a + res = self.interpret(f, [1, 2, 3]) + assert res == 0 class TestLLType(AbstractInterpretationTest): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Sat Feb 2 16:29:41 2008 @@ -135,6 +135,40 @@ assert len(jitcode.constants) == 0 assert len(jitcode.typekinds) == 0 + def test_merge(self): + def f(x, y, z): + if x: + a = y - z + else: + a = y + z + return 1 + a + writer, jitcode = self.serialize(f, [int, int, int]) + expected = assemble(writer.interpreter, + "red_int_is_true", 0, + "red_goto_iftrue", 3, tlabel("add"), + "make_new_redvars", 2, 1, 2, + "make_new_greenvars", 0, + "red_int_add", 0, 1, + "make_new_redvars", 1, 2, + "make_new_greenvars", 0, + label("after"), + "merge", 0, -1, + "make_redbox", -1, 0, + "red_int_add", 1, 0, + "make_new_redvars", 1, 2, + "make_new_greenvars", 0, + "red_return", 0, + label("add"), + "make_new_redvars", 2, 1, 2, + "make_new_greenvars", 0, + "red_int_sub", 0, 1, + "make_new_redvars", 1, 2, + "make_new_greenvars", 0, + "goto", tlabel("after"), + ) + assert jitcode.code == expected + assert len(jitcode.constants) == 1 + assert len(jitcode.typekinds) == 1 class TestLLType(AbstractSerializationTest): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Sat Feb 2 16:29:41 2008 @@ -10,13 +10,16 @@ def __init__(self, RGenOp=None, *TYPES): self.RGenOp = RGenOp self.TYPES = TYPES - TARGETTYPES = [] + self.nb_vals = len(TYPES) + if not TYPES: + assert RGenOp is None if RGenOp is None: assert len(TYPES) == 0 self.hash = lambda self: 0 self.compare = lambda self, other: True + TARGETTYPES = [] for TYPE in TYPES: # XXX more cases? TARGET = lltype.Signed @@ -53,6 +56,7 @@ class GreenKey(object): def __init__(self, values, desc): + assert len(values) == desc.nb_vals self.desc = desc self.values = values Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py Sat Feb 2 16:29:41 2008 @@ -1269,6 +1269,7 @@ attrname = hop.args_v[1].value DispatchQueueSubclass = self.get_dispatch_subclass(mpfamily) + py.test.skip("broken due to different key handling") if global_resumer is not None: states_dic = {} def merge_point(jitstate, *key): Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Sat Feb 2 16:29:41 2008 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, lloperation, llmemory from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.timeshifter import rvalue, rcontainer, rvirtualizable +from pypy.jit.timeshifter.greenkey import newgreendict, empty_key from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.annlowlevel import cachedtype, base_ptr_lltype from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -878,31 +879,26 @@ # ____________________________________________________________ -class BaseDispatchQueue(object): +class DispatchQueue(object): resuming = None - def __init__(self): + def __init__(self, num_local_caches=0): self.split_chain = None self.global_merge_chain = None self.return_chain = None + self.num_local_caches = num_local_caches self.clearlocalcaches() def clearlocalcaches(self): self.mergecounter = 0 + self.local_caches = [newgreendict() + for i in range(self.num_local_caches)] def clear(self): - self.__init__() + self.__init__(self.num_local_caches) def build_dispatch_subclass(attrnames): - if len(attrnames) == 0: - return BaseDispatchQueue - attrnames = unrolling_iterable(attrnames) - class DispatchQueue(BaseDispatchQueue): - def clearlocalcaches(self): - BaseDispatchQueue.clearlocalcaches(self) - for name in attrnames: - setattr(self, name, {}) # the new dicts have various types! - return DispatchQueue + py.test.skip("no longer exists") class FrozenVirtualFrame(object): @@ -1252,10 +1248,6 @@ return jitstate -def ensure_queue(jitstate, DispatchQueueClass): - return DispatchQueueClass() -ensure_queue._annspecialcase_ = 'specialize:arg(1)' - def replayable_ensure_queue(jitstate, DispatchQueueClass): if jitstate.frame is None: # common case return DispatchQueueClass() @@ -1291,14 +1283,14 @@ def merge_returning_jitstates(dispatchqueue, force_merge=False): return_chain = dispatchqueue.return_chain - return_cache = {} + return_cache = newgreendict() still_pending = None opened = None while return_chain is not None: jitstate = return_chain return_chain = return_chain.next opened = start_writing(jitstate, opened) - res = retrieve_jitstate_for_merge(return_cache, jitstate, (), + res = retrieve_jitstate_for_merge(return_cache, jitstate, empty_key, return_marker, force_merge=force_merge) if res is False: # not finished @@ -1311,13 +1303,13 @@ # more general one. return_chain = still_pending if return_chain is not None: - return_cache = {} + return_cache = newgreendict() still_pending = None while return_chain is not None: jitstate = return_chain return_chain = return_chain.next opened = start_writing(jitstate, opened) - res = retrieve_jitstate_for_merge(return_cache, jitstate, (), + res = retrieve_jitstate_for_merge(return_cache, jitstate, empty_key, return_marker, force_merge=force_merge) if res is False: # not finished From cfbolz at codespeak.net Sat Feb 2 17:01:36 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 17:01:36 +0100 (CET) Subject: [pypy-svn] r51214 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080202160136.893EF168477@codespeak.net> Author: cfbolz Date: Sat Feb 2 17:01:35 2008 New Revision: 51214 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: two tests about loops and a small bug fix Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sat Feb 2 17:01:35 2008 @@ -322,7 +322,7 @@ self.emit(index) elif len(block.exits) == 1: link, = block.exits - self.insert_renaming(link.args) + self.insert_renaming(link) self.make_bytecode_block(link.target, insert_goto=True) elif len(block.exits) == 2: linkfalse, linktrue = block.exits @@ -333,10 +333,10 @@ self.emit("%s_goto_iftrue" % color) self.emit(index) self.emit(tlabel(linktrue)) - self.insert_renaming(linkfalse.args) + self.insert_renaming(linkfalse) self.make_bytecode_block(linkfalse.target, insert_goto=True) self.emit(label(linktrue)) - self.insert_renaming(linktrue.args) + self.insert_renaming(linktrue) self.make_bytecode_block(linktrue.target, insert_goto=True) else: XXX @@ -365,8 +365,8 @@ self.emit(num) self.emit(keyindex) - def insert_renaming(self, args): - reds, greens = self.sort_by_color(args) + def insert_renaming(self, link): + reds, greens = self.sort_by_color(link.args, link.target.inputargs) for color, args in [("red", reds), ("green", greens)]: result = [] for v in args: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 17:01:35 2008 @@ -193,5 +193,16 @@ res = self.interpret(f, [1, 2, 3]) assert res == 0 + def test_loop_merging(self): + def ll_function(x, y): + tot = 0 + while x: + tot += y + x -= 1 + return tot + res = self.interpret(ll_function, [7, 2]) + assert res == 14 + + class TestLLType(AbstractInterpretationTest): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Sat Feb 2 17:01:35 2008 @@ -170,5 +170,34 @@ assert len(jitcode.constants) == 1 assert len(jitcode.typekinds) == 1 + def test_loop(self): + def f(x): + r = 0 + while x: + r += x + x -= 1 + return r + writer, jitcode = self.serialize(f, [int]) + expected = assemble(writer.interpreter, + "make_redbox", -1, 0, + "make_new_redvars", 2, 0, 1, + "make_new_greenvars", 0, + label("while"), + "merge", 0, -1, + "red_int_is_true", 0, + "red_goto_iftrue", 2, tlabel("body"), + "make_new_redvars", 1, 0, + "make_new_greenvars", 0, + "red_return", 0, + label("body"), + "make_new_redvars", 2, 0, 1, + "make_new_greenvars", 0, + "red_int_add", 1, 0, + "make_redbox", -2, 0, + "red_int_sub", 0, 3, + "make_new_redvars", 2, 2, 4, + "goto", tlabel("while")) + + class TestLLType(AbstractSerializationTest): type_system = "lltype" From cfbolz at codespeak.net Sat Feb 2 17:23:36 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 17:23:36 +0100 (CET) Subject: [pypy-svn] r51215 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080202162336.2689C1684D3@codespeak.net> Author: cfbolz Date: Sat Feb 2 17:23:36 2008 New Revision: 51215 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: a slightly more complex test Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 17:23:36 2008 @@ -203,6 +203,18 @@ res = self.interpret(ll_function, [7, 2]) assert res == 14 + def test_loop_merging2(self): + def ll_function(x, y): + tot = 0 + while x: + if tot < 3: + tot *= y + else: + tot += y + x -= 1 + return tot + res = self.interpret(ll_function, [7, 2]) + assert res == 0 class TestLLType(AbstractInterpretationTest): type_system = "lltype" From cfbolz at codespeak.net Sat Feb 2 17:27:01 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 17:27:01 +0100 (CET) Subject: [pypy-svn] r51216 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080202162701.A48E91684D3@codespeak.net> Author: cfbolz Date: Sat Feb 2 17:27:01 2008 New Revision: 51216 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: add ability to check for the correct amount of generated instructions Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 17:27:01 2008 @@ -11,6 +11,7 @@ from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.llinterp import LLInterpreter from pypy.annotation import model as annmodel +from pypy.objspace.flow.model import summary from pypy.rlib.jit import hint from pypy import conftest @@ -123,12 +124,20 @@ builder.end() generated = gv_generated.revealconst(lltype.Ptr(FUNC)) graph = generated._obj.graph + self.residual_graph = graph if conftest.option.view: graph.show() llinterp = LLInterpreter(self.rtyper) res = llinterp.eval_graph(graph, residualargs) return res + def check_insns(self, expected=None, **counts): + self.insns = summary(self.residual_graph) + if expected is not None: + assert self.insns == expected + for opname, count in counts.items(): + assert self.insns.get(opname, 0) == count + def Xtest_return_green(self): def f(): return 1 @@ -154,8 +163,10 @@ return x - y res = self.interpret(f, [1, 1, 2]) assert res == 3 + self.check_insns({"int_add": 1}) res = self.interpret(f, [0, 1, 2]) assert res == -1 + self.check_insns({"int_sub": 1}) def test_arith_plus_minus(self): def ll_plus_minus(encoded_insn, nb_insn, x, y): @@ -174,6 +185,7 @@ assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10]) assert res == 42 + self.check_insns({'int_add': 2, 'int_sub': 1}) def test_red_switch(self): def f(x, y): From cfbolz at codespeak.net Sat Feb 2 18:52:04 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 18:52:04 +0100 (CET) Subject: [pypy-svn] r51219 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080202175204.23BD8168457@codespeak.net> Author: cfbolz Date: Sat Feb 2 18:52:03 2008 New Revision: 51219 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: cache the bytecode for a function Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 18:52:03 2008 @@ -4,7 +4,7 @@ from pypy.jit.hintannotator.model import SomeLLAbstractConstant, OriginFlags from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.rainbow.bytecode import BytecodeWriter, label, tlabel, assemble -from pypy.jit.codegen.llgraph.rgenop import RGenOp +from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest from pypy.jit.rainbow import bytecode from pypy.jit.timeshifter import rtimeshift, exception, rvalue @@ -32,7 +32,31 @@ class AbstractInterpretationTest(object): + RGenOp = LLRGenOp + + def setup_class(cls): + from pypy.jit.timeshifter.test.conftest import option + cls.on_llgraph = cls.RGenOp is LLRGenOp + cls._cache = {} + cls._cache_order = [] + + def teardown_class(cls): + del cls._cache + del cls._cache_order + def serialize(self, func, values, backendoptimize=False): + key = func, backendoptimize + try: + cache, argtypes = self._cache[key] + except KeyError: + pass + else: + self.__dict__.update(cache) + assert argtypes == getargtypes(self.rtyper.annotator, values) + return self.writer, self.jitcode, self.argcolors + + if len(self._cache_order) >= 3: + del self._cache[self._cache_order.pop(0)] # build the normal ll graphs for ll_function t = TranslationContext() a = t.buildannotator() @@ -58,12 +82,31 @@ t.view() graph2 = graphof(t, func) self.graph = graph2 - writer = BytecodeWriter(t, hannotator, RGenOp) + writer = BytecodeWriter(t, hannotator, self.RGenOp) jitcode = writer.make_bytecode(graph2) argcolors = [] + + # make residual functype + ha = self.hannotator + RESTYPE = originalconcretetype(hannotator.binding(graph2.getreturnvar())) + ARGS = [] + for var in graph2.getargs(): + # XXX ignoring virtualizables for now + binding = hannotator.binding(var) + if not binding.is_green(): + ARGS.append(originalconcretetype(binding)) + self.RESIDUAL_FUNCTYPE = lltype.FuncType(ARGS, RESTYPE) + for i, ll_val in enumerate(values): color = writer.varcolor(graph2.startblock.inputargs[i]) argcolors.append(color) + self.writer = writer + self.jitcode = jitcode + self.argcolors = argcolors + + cache = self.__dict__.copy() + self._cache[key] = cache, getargtypes(rtyper.annotator, values) + self._cache_order.append(key) return writer, jitcode, argcolors def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): @@ -74,20 +117,8 @@ rtyper=writer.translator.annotator.base_translator.rtyper) edesc = exception.ExceptionDesc(hrtyper, False) rgenop = writer.RGenOp() - # make residual functype - FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) - ha = self.hannotator - RESTYPE = originalconcretetype(self.hannotator.binding(self.graph.getreturnvar())) - ARGS = [] - for var in self.graph.getargs(): - # XXX ignoring virtualizables for now - binding = self.hannotator.binding(var) - if not binding.is_green(): - ARGS.append(originalconcretetype(binding)) - FUNC = lltype.FuncType(ARGS, RESTYPE) - sigtoken = rgenop.sigToken(FUNC) + sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") - print builder, builder.rgenop, rgenop builder.start_writing() jitstate = rtimeshift.JITState(builder, None, edesc.null_exc_type_box, @@ -122,7 +153,7 @@ if jitstate is not None: ll_finish_jitstate(jitstate, edesc, sigtoken) builder.end() - generated = gv_generated.revealconst(lltype.Ptr(FUNC)) + generated = gv_generated.revealconst(lltype.Ptr(self.RESIDUAL_FUNCTYPE)) graph = generated._obj.graph self.residual_graph = graph if conftest.option.view: From cfbolz at codespeak.net Sat Feb 2 19:25:13 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 19:25:13 +0100 (CET) Subject: [pypy-svn] r51220 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter/test Message-ID: <20080202182513.5B1B2168425@codespeak.net> Author: cfbolz Date: Sat Feb 2 19:25:11 2008 New Revision: 51220 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Log: start moving some of the test_timeshift tests over to run on the rainbow interpreter. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 19:25:11 2008 @@ -1,3 +1,4 @@ +import py from pypy.translator.translator import TranslationContext, graphof from pypy.jit.hintannotator.annotator import HintAnnotator from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy @@ -138,17 +139,21 @@ greenargs = [] redargs = [] residualargs = [] - i = 0 - for color, ll_val in zip(argcolors, values): + red_i = 0 + for i, (color, ll_val) in enumerate(zip(argcolors, values)): if color == "green": greenargs.append(writer.RGenOp.constPrebuiltGlobal(ll_val)) else: TYPE = lltype.typeOf(ll_val) kind = rgenop.kindToken(TYPE) boxcls = rvalue.ll_redboxcls(TYPE) - redargs.append(boxcls(kind, inputargs_gv[i])) + if i in opt_consts: + gv_arg = rgenop.genconst(ll_val) + else: + gv_arg = inputargs_gv[red_i] + redargs.append(boxcls(kind, gv_arg)) residualargs.append(ll_val) - i += 1 + red_i += 1 jitstate = writer.interpreter.run(jitstate, jitcode, greenargs, redargs) if jitstate is not None: ll_finish_jitstate(jitstate, edesc, sigtoken) @@ -169,22 +174,40 @@ for opname, count in counts.items(): assert self.insns.get(opname, 0) == count - def Xtest_return_green(self): - def f(): - return 1 - self.interpret(f, []) +class SimpleTests(AbstractInterpretationTest): + def test_simple_fixed(self): + py.test.skip("green return not working") + def ll_function(x, y): + return hint(x + y, concrete=True) + res = self.interpret(ll_function, [5, 7]) + assert res == 12 + self.check_insns({}) def test_very_simple(self): def f(x, y): return x + y res = self.interpret(f, [1, 2]) assert res == 3 + self.check_insns({"int_add": 1}) def test_convert_const_to_red(self): def f(x): return x + 1 res = self.interpret(f, [2]) assert res == 3 + self.check_insns({"int_add": 1}) + + def test_loop_convert_const_to_redbox(self): + def ll_function(x, y): + x = hint(x, concrete=True) + tot = 0 + while x: # conversion from green '0' to red 'tot' + tot += y + x -= 1 + return tot + res = self.interpret(ll_function, [7, 2]) + assert res == 14 + self.check_insns({'int_add': 7}) def test_green_switch(self): def f(green, x, y): @@ -199,24 +222,31 @@ assert res == -1 self.check_insns({"int_sub": 1}) - def test_arith_plus_minus(self): - def ll_plus_minus(encoded_insn, nb_insn, x, y): - acc = x - pc = 0 - hint(nb_insn, concrete=True) - while pc < nb_insn: - op = (encoded_insn >> (pc*4)) & 0xF - op = hint(op, concrete=True) - if op == 0xA: - acc += y - elif op == 0x5: - acc -= y - pc += 1 - return acc - assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 - res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10]) - assert res == 42 - self.check_insns({'int_add': 2, 'int_sub': 1}) + def test_simple_opt_const_propagation2(self): + def ll_function(x, y): + return x + y + res = self.interpret(ll_function, [5, 7], [0, 1]) + assert res == 12 + self.check_insns({}) + + def test_simple_opt_const_propagation1(self): + def ll_function(x): + return -x + res = self.interpret(ll_function, [5], [0]) + assert res == -5 + self.check_insns({}) + + def test_loop_folding(self): + def ll_function(x, y): + tot = 0 + x = hint(x, concrete=True) + while x: + tot += y + x -= 1 + return tot + res = self.interpret(ll_function, [7, 2], [0, 1]) + assert res == 14 + self.check_insns({}) def test_red_switch(self): def f(x, y): @@ -243,8 +273,25 @@ tot += y x -= 1 return tot - res = self.interpret(ll_function, [7, 2]) + res = self.interpret(ll_function, [7, 2], []) + assert res == 14 + self.check_insns(int_add = 2, + int_is_true = 2) + + res = self.interpret(ll_function, [7, 2], [0]) + assert res == 14 + self.check_insns(int_add = 2, + int_is_true = 1) + + res = self.interpret(ll_function, [7, 2], [1]) + assert res == 14 + self.check_insns(int_add = 1, + int_is_true = 2) + + res = self.interpret(ll_function, [7, 2], [0, 1]) assert res == 14 + self.check_insns(int_add = 1, + int_is_true = 1) def test_loop_merging2(self): def ll_function(x, y): @@ -259,5 +306,106 @@ res = self.interpret(ll_function, [7, 2]) assert res == 0 -class TestLLType(AbstractInterpretationTest): + def test_two_loops_merging(self): + def ll_function(x, y): + tot = 0 + while x: + tot += y + x -= 1 + while y: + tot += y + y -= 1 + return tot + res = self.interpret(ll_function, [7, 3], []) + assert res == 27 + self.check_insns(int_add = 3, + int_is_true = 3) + + def test_convert_greenvar_to_redvar(self): + def ll_function(x, y): + hint(x, concrete=True) + return x - y + res = self.interpret(ll_function, [70, 4], [0]) + assert res == 66 + self.check_insns(int_sub = 1) + res = self.interpret(ll_function, [70, 4], [0, 1]) + assert res == 66 + self.check_insns({}) + + def test_green_across_split(self): + def ll_function(x, y): + hint(x, concrete=True) + if y > 2: + z = x - y + else: + z = x + y + return z + res = self.interpret(ll_function, [70, 4], [0]) + assert res == 66 + self.check_insns(int_add = 1, + int_sub = 1) + + def test_merge_const_before_return(self): + def ll_function(x): + if x > 0: + y = 17 + else: + y = 22 + x -= 1 + y += 1 + return y+x + res = self.interpret(ll_function, [-70], []) + assert res == 23-71 + self.check_insns({'int_gt': 1, 'int_add': 2, 'int_sub': 2}) + + def test_merge_3_redconsts_before_return(self): + py.test.skip("XXX hint(variable=True) not implemented yet") + def ll_function(x): + if x > 2: + y = hint(54, variable=True) + elif x > 0: + y = hint(17, variable=True) + else: + y = hint(22, variable=True) + x -= 1 + y += 1 + return y+x + res = self.interpret(ll_function, [-70], []) + assert res == ll_function(-70) + res = self.interpret(ll_function, [1], []) + assert res == ll_function(1) + res = self.interpret(ll_function, [-70], []) + assert res == ll_function(-70) + + def test_merge_const_at_return(self): + py.test.skip("green return") + def ll_function(x): + if x > 0: + return 17 + else: + return 22 + res = self.interpret(ll_function, [-70], []) + assert res == 22 + self.check_insns({'int_gt': 1}) + + def test_arith_plus_minus(self): + def ll_plus_minus(encoded_insn, nb_insn, x, y): + acc = x + pc = 0 + while pc < nb_insn: + op = (encoded_insn >> (pc*4)) & 0xF + op = hint(op, concrete=True) + if op == 0xA: + acc += y + elif op == 0x5: + acc -= y + pc += 1 + return acc + assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 + res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10], [0, 1]) + assert res == 42 + self.check_insns({'int_add': 2, 'int_sub': 1}) + + +class TestLLType(SimpleTests): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Sat Feb 2 19:25:11 2008 @@ -382,184 +382,6 @@ class BaseTestTimeshift(TimeshiftingTests): - def test_simple_fixed(self): - py.test.skip("green return not working") - def ll_function(x, y): - return hint(x + y, concrete=True) - res = self.timeshift(ll_function, [5, 7]) - assert res == 12 - self.check_insns({}) - - def test_very_simple(self): - def ll_function(x, y): - return x + y - res = self.timeshift(ll_function, [5, 7]) - assert res == 12 - self.check_insns({'int_add': 1}) - - def test_convert_const_to_redbox(self): - def ll_function(x, y): - x = hint(x, concrete=True) - tot = 0 - while x: # conversion from green '0' to red 'tot' - tot += y - x -= 1 - return tot - res = self.timeshift(ll_function, [7, 2]) - assert res == 14 - self.check_insns({'int_add': 7}) - - def test_simple_opt_const_propagation2(self): - def ll_function(x, y): - return x + y - res = self.timeshift(ll_function, [5, 7], [0, 1]) - assert res == 12 - self.check_insns({}) - - def test_simple_opt_const_propagation1(self): - def ll_function(x): - return -x - res = self.timeshift(ll_function, [5], [0]) - assert res == -5 - self.check_insns({}) - - def test_loop_folding(self): - def ll_function(x, y): - tot = 0 - x = hint(x, concrete=True) - while x: - tot += y - x -= 1 - return tot - res = self.timeshift(ll_function, [7, 2], [0, 1]) - assert res == 14 - self.check_insns({}) - - def test_loop_merging(self): - def ll_function(x, y): - tot = 0 - while x: - tot += y - x -= 1 - return tot - res = self.timeshift(ll_function, [7, 2], []) - assert res == 14 - self.check_insns(int_add = 2, - int_is_true = 2) - - res = self.timeshift(ll_function, [7, 2], [0]) - assert res == 14 - self.check_insns(int_add = 2, - int_is_true = 1) - - res = self.timeshift(ll_function, [7, 2], [1]) - assert res == 14 - self.check_insns(int_add = 1, - int_is_true = 2) - - res = self.timeshift(ll_function, [7, 2], [0, 1]) - assert res == 14 - self.check_insns(int_add = 1, - int_is_true = 1) - - def test_two_loops_merging(self): - def ll_function(x, y): - tot = 0 - while x: - tot += y - x -= 1 - while y: - tot += y - y -= 1 - return tot - res = self.timeshift(ll_function, [7, 3], []) - assert res == 27 - self.check_insns(int_add = 3, - int_is_true = 3) - - def test_convert_greenvar_to_redvar(self): - def ll_function(x, y): - hint(x, concrete=True) - return x - y - res = self.timeshift(ll_function, [70, 4], [0]) - assert res == 66 - self.check_insns(int_sub = 1) - res = self.timeshift(ll_function, [70, 4], [0, 1]) - assert res == 66 - self.check_insns({}) - - def test_green_across_split(self): - def ll_function(x, y): - hint(x, concrete=True) - if y > 2: - z = x - y - else: - z = x + y - return z - res = self.timeshift(ll_function, [70, 4], [0]) - assert res == 66 - self.check_insns(int_add = 1, - int_sub = 1) - - def test_merge_const_before_return(self): - def ll_function(x): - if x > 0: - y = 17 - else: - y = 22 - x -= 1 - y += 1 - return y+x - res = self.timeshift(ll_function, [-70], []) - assert res == 23-71 - self.check_insns({'int_gt': 1, 'int_add': 2, 'int_sub': 2}) - - def test_merge_3_redconsts_before_return(self): - def ll_function(x): - if x > 2: - y = hint(54, variable=True) - elif x > 0: - y = hint(17, variable=True) - else: - y = hint(22, variable=True) - x -= 1 - y += 1 - return y+x - res = self.timeshift(ll_function, [-70], []) - assert res == ll_function(-70) - res = self.timeshift(ll_function, [1], []) - assert res == ll_function(1) - res = self.timeshift(ll_function, [-70], []) - assert res == ll_function(-70) - - def test_merge_const_at_return(self): - py.test.skip("green return") - def ll_function(x): - if x > 0: - return 17 - else: - return 22 - res = self.timeshift(ll_function, [-70], []) - assert res == 22 - self.check_insns({'int_gt': 1}) - - def test_arith_plus_minus(self): - def ll_plus_minus(encoded_insn, nb_insn, x, y): - acc = x - pc = 0 - while pc < nb_insn: - op = (encoded_insn >> (pc*4)) & 0xF - op = hint(op, concrete=True) - if op == 0xA: - acc += y - elif op == 0x5: - acc -= y - pc += 1 - return acc - assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 - res = self.timeshift(ll_plus_minus, [0xA5A, 3, 32, 10], [0, 1]) - assert res == 42 - self.check_insns({'int_add': 2, 'int_sub': 1}) def test_simple_struct(self): S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), From cfbolz at codespeak.net Sat Feb 2 20:13:42 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 20:13:42 +0100 (CET) Subject: [pypy-svn] r51221 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080202191342.8273F168436@codespeak.net> Author: cfbolz Date: Sat Feb 2 20:13:40 2008 New Revision: 51221 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: support hint(variable=True). if only supporting hint(promote=True) were as easy Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sat Feb 2 20:13:40 2008 @@ -418,7 +418,7 @@ block = self.current_block if (arg, block) in self.redvar_positions: # already converted - return self.redvar_positions[arg] + return self.redvar_positions[arg, block] self.emit("make_redbox") resultindex = self.register_redvar((arg, block)) argindex = self.green_position(arg) @@ -441,11 +441,13 @@ color = "red" return color - def register_redvar(self, arg): + def register_redvar(self, arg, where=-1): assert arg not in self.redvar_positions - self.redvar_positions[arg] = result = self.free_red[self.current_block] - self.free_red[self.current_block] += 1 - return result + if where == -1: + where = self.free_red[self.current_block] + self.free_red[self.current_block] += 1 + self.redvar_positions[arg] = where + return where def redvar_position(self, arg): return self.redvar_positions[arg] @@ -510,6 +512,14 @@ assert self.hannotator.binding(result).is_green() self.register_greenvar(result, self.green_position(arg)) return + if "variable" in hints: + assert not self.hannotator.binding(result).is_green() + if self.hannotator.binding(arg).is_green(): + resultindex = self.convert_to_red(arg) + self.register_redvar(result, resultindex) + else: + self.register_redvar(result, self.redvar_position(arg)) + return XXX Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 20:13:40 2008 @@ -176,7 +176,7 @@ class SimpleTests(AbstractInterpretationTest): def test_simple_fixed(self): - py.test.skip("green return not working") + py.test.skip("green return") def ll_function(x, y): return hint(x + y, concrete=True) res = self.interpret(ll_function, [5, 7]) @@ -359,7 +359,6 @@ self.check_insns({'int_gt': 1, 'int_add': 2, 'int_sub': 2}) def test_merge_3_redconsts_before_return(self): - py.test.skip("XXX hint(variable=True) not implemented yet") def ll_function(x): if x > 2: y = hint(54, variable=True) From xoraxax at codespeak.net Sun Feb 3 11:59:57 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 3 Feb 2008 11:59:57 +0100 (CET) Subject: [pypy-svn] r51224 - in pypy/dist/pypy: annotation jit/hintannotator tool/algo tool/algo/test translator/backendopt Message-ID: <20080203105957.8A537168407@codespeak.net> Author: xoraxax Date: Sun Feb 3 11:59:55 2008 New Revision: 51224 Modified: pypy/dist/pypy/annotation/description.py pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/jit/hintannotator/bookkeeper.py pypy/dist/pypy/tool/algo/test/test_unionfind.py pypy/dist/pypy/tool/algo/unionfind.py pypy/dist/pypy/translator/backendopt/malloc.py Log: Refactor UnionFind API. It calls absorb instead of update now. Modified: pypy/dist/pypy/annotation/description.py ============================================================================== --- pypy/dist/pypy/annotation/description.py (original) +++ pypy/dist/pypy/annotation/description.py Sun Feb 3 11:59:55 2008 @@ -26,6 +26,7 @@ for shape, table in other.calltables.items(): for row in table: self.calltable_add_row(shape, row) + absorb = update # UnionFind API def calltable_lookup_row(self, callshape, row): # this code looks up a table of which graph to @@ -63,6 +64,7 @@ self.descs.update(other.descs) self.read_locations.update(other.read_locations) self.attrs.update(other.attrs) + absorb = update # UnionFind API def get_s_value(self, attrname): try: @@ -100,6 +102,7 @@ self.descs.update(other.descs) self.read_locations.update(other.read_locations) self.s_value = unionof(self.s_value, other.s_value) + absorb = update # UnionFind API def get_s_value(self, attrname): return self.s_value Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Sun Feb 3 11:59:55 2008 @@ -102,12 +102,10 @@ bookkeeper = self.funcdesc.bookkeeper bookkeeper.pending_specializations.append(self.finish) - def update(self, other): + def absorb(self, other): self.table.update(other.table) self.graph = None # just in case - - def cleanup(self): - self.do_not_process = True + other.do_not_process = True fieldnamecounter = 0 Modified: pypy/dist/pypy/jit/hintannotator/bookkeeper.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/bookkeeper.py (original) +++ pypy/dist/pypy/jit/hintannotator/bookkeeper.py Sun Feb 3 11:59:55 2008 @@ -110,6 +110,7 @@ def update(self, other): self.tsgraphs.update(other.tsgraphs) + absorb = update # UnionFind API class ImpurityAnalyzer(graphanalyze.GraphAnalyzer): Modified: pypy/dist/pypy/tool/algo/test/test_unionfind.py ============================================================================== --- pypy/dist/pypy/tool/algo/test/test_unionfind.py (original) +++ pypy/dist/pypy/tool/algo/test/test_unionfind.py Sun Feb 3 11:59:55 2008 @@ -8,11 +8,8 @@ state.append(self) self.obj = obj - def update(self, other): - pass - - def cleanup(self): - state.remove(self) + def absorb(self, other): + state.remove(other) uf = UnionFind(ReferencedByExternalState) uf.find(1) Modified: pypy/dist/pypy/tool/algo/unionfind.py ============================================================================== --- pypy/dist/pypy/tool/algo/unionfind.py (original) +++ pypy/dist/pypy/tool/algo/unionfind.py Sun Feb 3 11:59:55 2008 @@ -82,13 +82,11 @@ rep1, rep2, info1, info2, = rep2, rep1, info2, info1 if info1 is not None: - info1.update(info2) + info1.absorb(info2) self.link_to_parent[rep2] = rep1 del self.weight[rep2] - if hasattr(info2, "cleanup"): - info2.cleanup() del self.root_info[rep2] self.weight[rep1] = w Modified: pypy/dist/pypy/translator/backendopt/malloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/malloc.py (original) +++ pypy/dist/pypy/translator/backendopt/malloc.py Sun Feb 3 11:59:55 2008 @@ -15,11 +15,12 @@ self.creationpoints = {} # set of ("type of creation point", ...) self.usepoints = {} # set of ("type of use point", ...) - def update(self, other): + def absorb(self, other): self.variables.update(other.variables) self.creationpoints.update(other.creationpoints) self.usepoints.update(other.usepoints) + class BaseMallocRemover(object): IDENTITY_OPS = ('same_as',) From cfbolz at codespeak.net Sun Feb 3 14:39:32 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 3 Feb 2008 14:39:32 +0100 (CET) Subject: [pypy-svn] r51225 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter timeshifter/test Message-ID: <20080203133932.3DB3A168414@codespeak.net> Author: cfbolz Date: Sun Feb 3 14:39:31 2008 New Revision: 51225 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Log: some support for direct red calls in the jit (nothing fancy, no exceptions). Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sun Feb 3 14:39:31 2008 @@ -3,6 +3,7 @@ from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype +from pypy.jit.hintannotator import model as hintmodel from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.jit.timeshifter.greenkey import KeyDesc, empty_key, GreenKey @@ -19,14 +20,17 @@ green consts are negative indexes """ - def __init__(self, code, constants, typekinds, redboxclasses, keydescs, - num_mergepoints): + def __init__(self, name, code, constants, typekinds, redboxclasses, + keydescs, called_bytecodes, num_mergepoints, is_portal): + self.name = name self.code = code self.constants = constants self.typekinds = typekinds self.redboxclasses = redboxclasses self.keydescs = keydescs + self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints + self.is_portal = is_portal def _freeze_(self): return True @@ -44,11 +48,10 @@ self.opname_to_index = {} self.jitstate = None self.queue = None - self.bytecode = None - self.pc = -1 self._add_implemented_opcodes() - def run(self, jitstate, bytecode, greenargs, redargs): + def run(self, jitstate, bytecode, greenargs, redargs, + start_bytecode_loop=True): self.jitstate = jitstate self.queue = rtimeshift.DispatchQueue(bytecode.num_mergepoints) rtimeshift.enter_frame(self.jitstate, self.queue) @@ -57,7 +60,8 @@ self.frame.bytecode = bytecode self.frame.local_boxes = redargs self.frame.local_green = greenargs - self.bytecode_loop() + if start_bytecode_loop: + self.bytecode_loop() return self.jitstate def bytecode_loop(self): @@ -70,14 +74,16 @@ assert result is None def dispatch(self): + is_portal = self.frame.bytecode.is_portal newjitstate = rtimeshift.dispatch_next(self.queue) resumepoint = rtimeshift.getresumepoint(newjitstate) self.newjitstate(newjitstate) if resumepoint == -1: - # XXX what about green returns? - newjitstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) + newjitstate = rtimeshift.leave_graph_red( + self.queue, is_portal) self.newjitstate(newjitstate) - return STOP + if newjitstate is None or is_portal: + return STOP else: self.frame.pc = resumepoint @@ -192,6 +198,30 @@ if done: return self.dispatch() + def opimpl_red_direct_call(self): + greenargs = [] + num = self.load_2byte() + for i in range(num): + greenargs.append(self.get_greenarg()) + redargs = [] + num = self.load_2byte() + for i in range(num): + redargs.append(self.get_redarg()) + bytecodenum = self.load_2byte() + targetbytecode = self.frame.bytecode.called_bytecodes[bytecodenum] + self.run(self.jitstate, targetbytecode, greenargs, redargs, + start_bytecode_loop=False) + # this frame will be resumed later in the next bytecode, which is + # red_after_direct_call + + def opimpl_red_after_direct_call(self): + newjitstate = rtimeshift.collect_split( + self.jitstate, self.frame.pc, + self.frame.local_green) + assert newjitstate is self.jitstate + + + # ____________________________________________________________ # construction-time interface def _add_implemented_opcodes(self): @@ -244,21 +274,32 @@ class BytecodeWriter(object): - def __init__(self, t, hintannotator, RGenOp): + def __init__(self, t, hannotator, RGenOp): self.translator = t self.annotator = t.annotator - self.hannotator = hintannotator + self.hannotator = hannotator self.interpreter = JitInterpreter() self.RGenOp = RGenOp self.current_block = None - - def make_bytecode(self, graph): + self.raise_analyzer = hannotator.exceptiontransformer.raise_analyzer + self.all_graphs = {} # mapping graph to bytecode + self.unfinished_graphs = [] + + def can_raise(self, op): + return self.raise_analyzer.analyze(op) + + def make_bytecode(self, graph, is_portal=True): + if is_portal: + self.all_graphs[graph] = JitCode.__new__(JitCode) self.seen_blocks = {} self.assembler = [] self.constants = [] self.typekinds = [] self.redboxclasses = [] self.keydescs = [] + self.called_bytecodes = [] + self.num_mergepoints = 0 + self.is_portal = is_portal # mapping constant -> index in constants self.const_positions = {} # mapping blocks to True @@ -273,19 +314,31 @@ self.type_positions = {} # mapping tuple of green TYPES to index self.keydesc_positions = {} - - self.num_mergepoints = 0 + # mapping graphs to index + self.graph_positions = {}