From ale at codespeak.net Tue May 1 08:55:46 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 08:55:46 +0200 (CEST) Subject: [pypy-svn] r42512 - pypy/dist/pypy/lib/pyontology/test Message-ID: <20070501065546.5EAA18076@code0.codespeak.net> Author: ale Date: Tue May 1 08:55:44 2007 New Revision: 42512 Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py Log: Changed and added test for SPARQL. Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_sparql.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_sparql.py Tue May 1 08:55:44 2007 @@ -69,7 +69,7 @@ assert len(res[0]) == len(res[1]) == 4 assert res[0][1] in ['http://example.org/ns#p', 'http://example.org/ns#q'] assert res[1][1] in ['http://example.org/ns#p', 'http://example.org/ns#q'] - assert result[3][0] == 'x<2' + assert result[3].formula == 'query_x<2' # There are 8 ways of having the triples in the query, if predicate is not a builtin owl predicate # @@ -113,6 +113,7 @@ O.attach_fd() O.finish() res = O.sparql(query) + assert len(res) == 1 assert res[0]['x'] == u'http://example.org/ns#sub' def test_case_2(): @@ -166,7 +167,6 @@ res = O.sparql(query) assert res[0]['x'] == u'http://example.org/ns#sub' assert res[0]['y'] == 123 - assert res[0]['x'] == u'http://example.org/ns#sub' def test_case_6(): """ return the values of p """ @@ -202,7 +202,88 @@ O.finish() res = O.sparql(query) assert res[0]['y'] == 123 + +query_x = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?x + WHERE { + ?x rdf:type ltw:Active_FundingOrganisation . + ?x ltw:organisationNameAbbreviation 'BMBF' . + }""" + +def test_queryx(): + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query_x) + assert len(res) == 1 + assert res[0]['x'] == u'http://www.lt-world.org/ltw.owl#obj_61128' + +query_y = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + PREFIX xsd: + SELECT ?project + WHERE { + ?project ltw:dateStart ?date_begin . + ?project ltw:dateEnd ?date_end . + FILTER ( ?date_begin < "2007-01-01T00:00:00Z"^^xsd:dateTime + && ?date_end >= "2006-01-01T00:00:00Z"^^xsd:dateTime) . + }""" + + +def test_queryy(): + py.test.skip("Needs special care for dateTime type") + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query_y) + assert len(res) == 1 + assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_61128' + + +query1_a = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?person ?activity + WHERE { + ?activity rdf:type ltw:Active_Project . + ?person rdf:type ltw:Active_Person . + } + ORDER BY ?person""" +query1_b = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?person ?activity + WHERE { + ?activity ltw:hasParticipant ?person_obj . + ?person_obj ltw:personName ?person . + } + ORDER BY ?person""" +def test_query1_a(): + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query1_a) + assert len(res) == 1 + +def test_query1_b(): + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query1_b) + assert len(res) == 1 + query1 = """ PREFIX ltw : PREFIX owl : @@ -219,19 +300,22 @@ query2 = """ PREFIX ltw : PREFIX owl : - SELECT ?project ?date_begin + PREFIX rdf : + SELECT ?project ?date_begin ?x WHERE { - ?project ltw:supportedBy ltw:BMBF . + ?project ltw:supportedBy ?x . + ?x rdf:type ltw:Active_FundingOrganisation . + ?x ltw:organisationNameAbbreviation 'BMBF' . ?project ltw:dateStart ?date_begin . ?project ltw:dateEnd ?date_end . FILTER ( ?date_begin < 2007 && ?date_end >= 2006) . }""" #which project is funded in a technological area (i.e. Semantic web), -query3 = """ +query_3 = """ PREFIX ltw : PREFIX owl : PREFIX rdf : - SELECT ?project + SELECT ?project ?x ?y WHERE { ?project rdf:type ltw:Active_Project . ?project ltw:lt_technologicalMethod ?y . @@ -239,7 +323,47 @@ ?project ltw:supportedBy ?x . }""" -def test_query1(): +query3a = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?project ?x ?y + WHERE { + ?project rdf:type ltw:Active_Project . + ?project ltw:lt_technologicalMethod ?y . + }""" + +query3b = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?project ?x ?y + WHERE { + ?y rdf:type ltw:Semantic_Web . + ?project ltw:supportedBy ?x . + }""" + +def test_query3a(): + py.test.skip("WIP") + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query3a) + assert len(res) == 1 + assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59773' + +def test_query3b(): + py.test.skip("WIP") + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query3a) + assert len(res) == 1 + assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59773' + +def test_query_1(): O = Ontology() O.add_file(datapath("testont2.rdf"), 'xml') O.attach_fd() @@ -249,27 +373,26 @@ assert res[0]['activity'] == u'http://www.lt-world.org/ltw.owl#obj_59754' assert res[0]['person'] == u'\nKlara Vicsi' -def test_query2(): -# py.test.skip("Doesn't work yet") +def test_query_2(): + py.test.skip("Needs special care for dateTime type") O = Ontology() O.add_file(datapath("testont2.rdf"), 'xml') O.attach_fd() res = O.sparql(query2) - assert len(res) == 2 + assert len(res) == 1 assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59754' assert res[0]['date_begin'] == datetime.date(1998,9,1) -def test_query3(): - #py.test.skip("Doesn't work yet") +def test_query_3(): + py.test.skip("WIP") O = Ontology() O.add_file(datapath("testont2.rdf"), 'xml') O.attach_fd() - res = O.sparql(query3) + res = O.sparql(query_3) assert len(res) == 1 - assert res[0]['activity'] == u'http://www.lt-world.org/ltw.owl#obj_59754' - assert res[0]['person'] == u'\nKlara Vicsi' + assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59773' import xmlrpclib, socket, os, signal From ale at codespeak.net Tue May 1 08:57:34 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 08:57:34 +0200 (CEST) Subject: [pypy-svn] r42513 - pypy/dist/pypy/lib/pyontology/test Message-ID: <20070501065734.2E1958076@code0.codespeak.net> Author: ale Date: Tue May 1 08:57:33 2007 New Revision: 42513 Modified: pypy/dist/pypy/lib/pyontology/test/testont2.rdf Log: Added types needed for the tests. Added a section for 'Semantci Web' Modified: pypy/dist/pypy/lib/pyontology/test/testont2.rdf ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/testont2.rdf (original) +++ pypy/dist/pypy/lib/pyontology/test/testont2.rdf Tue May 1 08:57:33 2007 @@ -1,5 +1,33 @@ + + + +Semantic Web + +Semantic Web + +Semantic Web + +The Semantic Web is a W3C-based initiative for representing knowledge on the World Wide Web in a machine-readable fashion, such that it can be understood and used by machines for intelligent applications. + + + +EUROPA!Europaeische Union!European Comission!Eurpean Commission!Eurpean Comission!European Commission - Directorate General + +EC + +EC + +European Commission + +European Commission + + + +Federal Ministry of Education and Research +BMBF + @@ -8,7 +36,8 @@ - + + From ale at codespeak.net Tue May 1 09:01:05 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 09:01:05 +0200 (CEST) Subject: [pypy-svn] r42514 - pypy/dist/pypy/lib/pyontology Message-ID: <20070501070105.BC1CA8076@code0.codespeak.net> Author: ale Date: Tue May 1 09:01:05 2007 New Revision: 42514 Modified: pypy/dist/pypy/lib/pyontology/sparql_grammar.py Log: Removed extra quoting of strings. Support for 'and' and 'or'. Construction of constraints from FILTER expression Modified: pypy/dist/pypy/lib/pyontology/sparql_grammar.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/sparql_grammar.py (original) +++ pypy/dist/pypy/lib/pyontology/sparql_grammar.py Tue May 1 09:01:05 2007 @@ -28,11 +28,28 @@ def replace_string(s, loc, toks): - return [rdfliteral((toks[0]), datatype="http://www.w3.org/2001/XMLSchema#String")] + return [rdfliteral((toks[0][1:-1]), datatype="http://www.w3.org/2001/XMLSchema#String")] def replace_float(s, loc, toks): return [rdfliteral(float(toks[0]), datatype="http://www.w3.org/2001/XMLSchema#float")] +def replace_RDFLiteral(s, loc, toks): + + if toks.IRIref: + lit = toks[0] + lit.datatype = toks.IRIref + ret = [lit] + else: + ret = [rdfliteral(toks[0], datatype="http://www.w3.org/2001/XMLSchema#String")] + return ret + +def construct_constraint(s, loc, toks): + varlist = [] + for x in toks.Var: + varlist.append(x[0]) + varlist.append(toks[0]) + return [varlist] + class SPARQLGrammar(object): # All productions are declared Forward(). This is primarily @@ -75,8 +92,8 @@ lne = punctuation('ne') bnode = punctuation('_:') comma = punctuation(',') - lor = punctuation('||') - land = punctuation('&&') + lor = punctuation('||').setParseAction(lambda x,y,z: [' or ']) + land = punctuation('&&').setParseAction(lambda x,y,z: [' and ']) # keywords @@ -138,7 +155,7 @@ OptionalGraphPattern = production('OptionalGraphPattern') GraphGraphPattern = production('GraphGraphPattern') GroupOrUnionGraphPattern = production('GroupOrUnionGraphPattern') - Constraint = production('Constraint') + Constraint = production('Constraint').setParseAction(construct_constraint) ConstructTemplate = production('ConstructTemplate') Triples = production('Triples') Triples1 = production('Triples1') @@ -174,7 +191,7 @@ PrimaryExpression = production('PrimaryExpression') RDFTerm = production('RDFTerm') NumericLiteral = production('NumericLiteral') - RDFLiteral = production('RDFLiteral') + RDFLiteral = production('RDFLiteral').setParseAction(replace_RDFLiteral) BooleanLiteral = production('BooleanLiteral') String = production('String').setParseAction(replace_string) IRIref = production('IRIref') From ale at codespeak.net Tue May 1 09:03:54 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 09:03:54 +0200 (CEST) Subject: [pypy-svn] r42515 - pypy/dist/pypy/lib/pyontology Message-ID: <20070501070354.AE2B98076@code0.codespeak.net> Author: ale Date: Tue May 1 09:03:54 2007 New Revision: 42515 Modified: pypy/dist/pypy/lib/pyontology/constraint_classes.py Log: Corrected the constraints working on restrictions. Changed the solver mechanics to be able to solve the problem with respect to only some of the variables. Modified: pypy/dist/pypy/lib/pyontology/constraint_classes.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/constraint_classes.py (original) +++ pypy/dist/pypy/lib/pyontology/constraint_classes.py Tue May 1 09:03:54 2007 @@ -31,6 +31,10 @@ print repository try: foundSolution = repository.consistency(verbose) +# This ties the repository to a special distributor - not nice + if not foundSolution: + dom = repository.getDomains() + foundSolution = [x for x in self._distributor.vars if dom[x].size() == 1] except ConsistencyFailure, exc: if verbose: print strftime('%H:%M:%S'), exc @@ -88,10 +92,16 @@ class MyDistributor(SplitDistributor): - def __init__(self): + def __init__(self, vars_to_distribute): SplitDistributor.__init__(self,2) + self.vars = vars_to_distribute self.to_split = None + def findSmallestDomain(self, domains): + dom_sizes = [(domains[v].size(),v) for v in self.vars if domains[v].size()>1] + dom_sizes.sort() + return dom_sizes[0][1] + def nb_subdomains(self, domains): """See AbstractDistributor""" self.to_split = self.findSmallestDomain(domains) @@ -258,7 +268,7 @@ class PropertyConstrain3(AbstractConstraint): cost = 1 def __init__(self, prop, variable, cls_or_restriction): - AbstractConstraint.__init__(self, [ prop]) + AbstractConstraint.__init__(self, [variable]) self.object = cls_or_restriction self.variable = variable self.prop = prop @@ -266,23 +276,24 @@ def narrow(self, domains): # Narrow the domains of object and variable to those values # that are connected by self.prop +# import pdb +# pdb.set_trace() dom = domains[self.prop] sub = domains[self.variable] obj = domains[self.object] - vals_dict = dom._dict + sub_= [x[0] for x in dom.getValues()] + obj_= [x[1] for x in dom.getValues()] - keep = set() sub_rem = [] + obj_rem = [] for v in sub.getValues(): - if not v in vals_dict: + if not v in sub_: sub_rem.append(v) - #sub.removeValue(v) - else: - for o in dom.getValuesPrKey(v): - keep.add(o) - remove = [x for x in obj.getValues() if not x in keep] + for v in obj.getValues(): + if not v in obj_: + obj_rem.append(v) sub.removeValues(sub_rem) - obj.removeValues(remove) + obj.removeValues(obj_rem) class MemberConstraint(AbstractConstraint): cost = 1 @@ -585,14 +596,25 @@ assert len(inter) > 0 cls = domains[self.variable].setValues(inter) -class SomeValueConstraint(OneofPropertyConstraint): +class SomeValueConstraint(AbstractConstraint): cost = 100 - + + def __init__(self, variable, property, list_of_vals): + AbstractConstraint.__init__(self, [variable ]) + self.variable = variable + self.property = property + self.List = list_of_vals + cost = 100 + + def estimateCost(self, domains): + return self.cost + + def narrow(self, domains): val = set(domains[self.List].getValues()) dom = domains[self.variable] - property = dom.property + property = self.property indi = dom.getValues() prop = Linkeddict(list(domains[property].getValues())) for v in indi: @@ -606,7 +628,7 @@ else: dom.removeValue(v) -class AllValueConstraint(OneofPropertyConstraint): +class AllValueConstraint(SomeValueConstraint): """ AllValuesfrom property restriction is used to define the class of individuals for which the values for the property (defined by the onProperty triple) all comes from the class description @@ -621,7 +643,7 @@ if not val: return dom = domains[self.variable] - property = dom.property + property = self.property indi = dom.getValues() prop = domains[property]._dict remove = [] @@ -635,14 +657,14 @@ remove.append(v) dom.removeValues(remove) -class HasvalueConstraint(OneofPropertyConstraint): +class HasvalueConstraint(SomeValueConstraint): cost = 100 def narrow(self, domains): val = self.List dom = domains[self.variable] - property = dom.property + property = self.property indi = dom.getValues() prop = Linkeddict(domains[property].getValues()) remove = [] From ale at codespeak.net Tue May 1 09:05:39 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 09:05:39 +0200 (CEST) Subject: [pypy-svn] r42516 - pypy/dist/pypy/lib/pyontology Message-ID: <20070501070539.27C858076@code0.codespeak.net> Author: ale Date: Tue May 1 09:05:38 2007 New Revision: 42516 Modified: pypy/dist/pypy/lib/pyontology/pyontology.py Log: Remember to add Literal to the domains. Lots of fixes and tweaks in SPARQL implementation Modified: pypy/dist/pypy/lib/pyontology/pyontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/pyontology.py (original) +++ pypy/dist/pypy/lib/pyontology/pyontology.py Tue May 1 09:05:38 2007 @@ -145,7 +145,7 @@ variables[self.name].setValues([v[0] for v in prop.getValues()]) elif ('owl_Thing' in variables.keys() and isinstance(self, ClassDomain) and self.size() == 0): - variables[self.name].setValues(list(variables['owl_Thing'].getValues())) + variables[self.name].setValues(list(variables['owl_Thing'].getValues())+list(variables['owl_Literal'])) variables.update(self.domains) glob_constraints.extend(self.in_constraint) # assert self.size() != 0 @@ -533,6 +533,9 @@ continue self.variables[key].finish(self.variables, self.constraints) else: +# In case of an inconsistent ontology remove the comments to get more +# than one error + # try: constraint.narrow(self.variables) # except ConsistencyFailure, e: @@ -553,7 +556,7 @@ triples = where.GroupGraphPattern[0].Triples new = [] - vars = [] + vars = set() for trip in triples: case = 0 inc = 1 @@ -572,7 +575,7 @@ elif item.VAR1: var_uri = URIRef('query_'+item.VAR1[0][0]) newtrip.append(var_uri) - vars.append(var_uri) + vars.add(var_uri) case += trip_.index(item) + inc if inc == 2: inc = 1 @@ -583,6 +586,12 @@ newtrip.append(case) new.append(newtrip) constrain = where.GroupGraphPattern[0].Constraint + if constrain: + constrain = constrain[0] + expr = constrain.pop(-1) + for x in constrain: + expr = expr.replace(x, URIRef('query_'+x)) + constrain = Expression([URIRef('query_'+x) for x in constrain], expr) return new, prefixes, resvars, constrain, vars # There are 8 ways of having the triples in the query, if predicate is not a builtin owl predicate @@ -615,12 +624,22 @@ elif case == 1: # Add a HasValue constraint ns,pred = trip[1].split("#") - if ns in namespaces.values(): + if 0: #ns in namespaces.values(): + import pdb + pdb.set_trace() self.consider_triple(trip) else: var = self.make_var(Restriction, URIRef(trip[0])) - self.onProperty(var, URIRef(trip[1])) - self.hasValue(var, trip[2]) + prop = URIRef(trip[1]) + prop_name = self.mangle_name(prop) + self.onProperty(var, prop) + query_dom[prop_name] = self.variables[prop_name] + if trip[2] in self.variables['owl_Literal'] : + val = trip[2] + else: + val = self.variables[self.mangle_name(trip[2])] + query_constr.append(HasvalueConstraint(var, prop_name, val)) + #self.hasValue(var, trip[2]) elif case == 2: # for all p's return p if p[0]==s and p[1]==o @@ -672,18 +691,9 @@ prop = self.make_var(Property, URIRef(trip[1])) query_dom[prop] = self.variables[prop] p_vals = list(self.variables[prop].getValues()) - sub = self.make_var(Thing, trip[0]) - vals = set([v[0] for v in p_vals]) - if self.variables[sub].size(): - vals &= set(self.variables[sub].getValues()) - self.variables[sub].setValues(vals) - obj = self.make_var(Thing, trip[2]) - vals = set([v[1] for v in p_vals]) - if self.variables[obj].size(): - vals &= set(self.variables[obj].getValues()) - self.variables[obj].setValues(vals) + sub = self.make_var(ClassDomain, trip[0]) + obj = self.make_var(ClassDomain, trip[2]) con = PropertyConstrain3(prop, sub, obj) -# con = Expression([sub,prop,obj], "%s == (%s, %s)" %(prop, sub, obj)) query_constr.append(con) elif case == 6: @@ -709,13 +719,15 @@ _dom, _ = self.variables[self.mangle_name(v)].finish(self.variables, query_constr) #query_dom, query_constr) query_dom.update(_dom) # Build a repository with the variables in the query - dom = dict([(self.mangle_name(v),self.variables[self.mangle_name(v)]) - for v in vars]) - + solve_vars = [self.mangle_name(v) for v in vars] + dom = dict([(v, self.variables[v]) + for v in solve_vars]) dom.update(query_dom) +# import pdb +# pdb.set_trace() # solve the repository and return the solution - rep = Repository(dom.keys(), dom, query_constr) - res_s = Solver(MyDistributor()).solve(rep, verbose=0) + rep = Repository(solve_vars, dom, query_constr) + res_s = Solver(MyDistributor(solve_vars)).solve(rep, verbose=0) res = [] query_vars = dict([('query_%s_'%name,name) for name in resvars]) for d in res_s: @@ -1059,7 +1071,11 @@ Individuals that have a property with the value of var. To make an assertion we need to know for which class the restriction applies""" sub = self.make_var(Restriction, s) - cons = HasvalueConstraint(sub, var) + try: + prop = self.variables[sub].property + except KeyError: + prop = None + cons = HasvalueConstraint(sub, prop, var) self.constraints.append(cons) def allValuesFrom(self, s, var): From cfbolz at codespeak.net Tue May 1 13:43:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 13:43:39 +0200 (CEST) Subject: [pypy-svn] r42520 - in pypy/extradoc/talk/badhonnef2007: . image Message-ID: <20070501114339.328A18069@code0.codespeak.net> Author: cfbolz Date: Tue May 1 13:43:38 2007 New Revision: 42520 Added: pypy/extradoc/talk/badhonnef2007/ pypy/extradoc/talk/badhonnef2007/image/ pypy/extradoc/talk/badhonnef2007/image/arch.pdf (contents, props changed) pypy/extradoc/talk/badhonnef2007/image/bench.png (contents, props changed) pypy/extradoc/talk/badhonnef2007/image/py-web.png (contents, props changed) pypy/extradoc/talk/badhonnef2007/talk.pdf (contents, props changed) pypy/extradoc/talk/badhonnef2007/talk.tex Log: talk about pypy and prolog, heavily stolen from the dls talk Added: pypy/extradoc/talk/badhonnef2007/image/arch.pdf ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/badhonnef2007/image/bench.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/badhonnef2007/image/py-web.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/badhonnef2007/talk.pdf ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/badhonnef2007/talk.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/badhonnef2007/talk.tex Tue May 1 13:43:38 2007 @@ -0,0 +1,465 @@ +\documentclass[utf8]{beamer} + +% This file is a solution template for: + +% - Talk at a conference/colloquium. +% - Talk length is about 20min. +% - Style is ornate. + +\mode +{ + \usetheme{Warsaw} + % or ... + + \setbeamercovered{transparent} + % or whatever (possibly just delete it) +} + + +\usepackage[english]{babel} + +\usepackage[utf8]{inputenc} +% or whatever + +\usepackage{times} +\usepackage[T1]{fontenc} +% Or whatever. Note that the encoding and the font should match. If T1 +% does not look nice, try deleting the line with the fontenc. + + +\title{A flexible Prolog interpreter in Python} + +\author{Carl Friedrich Bolz} +% - Give the names in the same order as the appear in the paper. +% - Use the \inst{?} command only if the authors have different +% affiliation. + +\institute[Heinrich-Heine-Universit?t D?sseldorf] +{ + Institut f?r Informatik\\ + Heinrich-Heine-Universit?t D?sseldorf +} + +\date{24. Workshop der GI-Fachgruppe Programmiersprachen und Rechenkonzepte, 4. Mai 2007} +% - Either use conference name or its abbreviation. +% - Not really informative to the audience, more for people (including +% yourself) who are reading the slides online + + +% If you have a file called "university-logo-filename.xxx", where xxx +% is a graphic format that can be processed by latex or pdflatex, +% resp., then you can add a logo as follows: + +\pgfdeclareimage[height=0.5cm]{pypy-logo}{image/py-web.png} +\logo{\pgfuseimage{pypy-logo}} + + + +% Delete this, if you do not want the table of contents to pop up at +% the beginning of each subsection: +%\AtBeginSubsection[] +%{ +% \begin{frame} +% \frametitle{Outline} +% \tableofcontents[currentsection,currentsubsection] +% \end{frame} +%} + + +% If you wish to uncover everything in a step-wise fashion, uncomment +% the following command: + +%\beamerdefaultoverlayspecification{<+->} + + +\begin{document} + +\begin{frame} + \titlepage +\end{frame} + +\begin{frame} + \frametitle{Outline} + \tableofcontents + % You might wish to add the option [pausesections] +\end{frame} + + +% Structuring a talk is a difficult task and the following structure +% may not be suitable. Here are some rules that apply for this +% solution: + +% - Exactly two or three sections (other than the summary). +% - At *most* three subsections per section. +% - Talk about 30s to 2min per frame. So there should be between about +% 15 and 30 frames, all told. + +% - A conference audience is likely to know very little of what you +% are going to talk about. So *simplify*! +% - In a 20min talk, getting the main ideas across is hard +% enough. Leave out details, even if it means being less precise than +% you think necessary. +% - If you omit details that are vital to the proof/implementation, +% just say so once. Everybody will be happy with that. + +\section{What is Pyrolog?} + +\begin{frame} + \frametitle{Pyrolog} + + \begin{itemize} + \item + Pyrolog is a Prolog interpreter written in RPython + \item + RPython is a subset of Python translatable to other languages + \item + translation part done with the help of the PyPy project + \end{itemize} +\end{frame} + +\section{The PyPy Approach to VM Construction} +\subsection{Overview} +\begin{frame} + \frametitle{What is PyPy?} + \begin{itemize} + \item + started as a Python VM implementation + in RPython (a well-chosen subset of Python) + \item + includes a translation tool-chain + \item + is becoming a general environment for writing interpreters (JavaScript, Prolog started) + \item + Open source project (MIT license) + \item + received EU funding for 2.5 years + \end{itemize} + +\end{frame} + +\subsection{Motivation} +\begin{frame} + \frametitle{VMs are still hard} + It is hard to achieve: + + \begin{itemize} + \item + flexibility + \item + maintainability + \item + performance (needs dynamic compilation techniques) + \end{itemize} + Especially with limited resources (like Open Source projects, research projects) +\end{frame} + + +\begin{frame} + \frametitle{The Python case (i)} + CPython (the reference implementation) is a straightforward, portable VM. + + \begin{itemize} + \item + Pervasive decisions: reference counting, single global lock ... + \item + No dynamic compilation +% \pause + \item + Extensions: + \begin{itemize} + \item + \alert{Stackless} (unlimited recursion, coroutines, serializable continuations) + \item + \alert{Psyco} (run-time specializer) + \item + \alert{Jython}, \alert{IronPython} + \end{itemize} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{The Python case (ii)} + \begin{itemize} + \item + Extensions have problems + \begin{itemize} + \item + need to keep track of CPython + \item + are hard to maintain + \item + Psyco very hard to port to other hardware architectures + \end{itemize} + \item + The community wants Python to run everywhere: + Jython (Java), IronPython (.NET). + Lots of effort and duplication. + + \item + At various points various incompatibilities between + the implementations + + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{The Prolog case} + \begin{itemize} + \item + problem mitigated by the fact that Prolog the language does not change + \item + a lot of implementations out there + \item + well-tuned mature C implementations (Sictsus, XSB, SWI, GNU-Prolog) + \begin{itemize} + \item + have sometimes incompatible extensions to core Prolog + \item + interfacing with libraries is tedious + \item + changing the language to experiment is hard + \item + fixed implementation decisions (GC, how to generate code, etc.) + \end{itemize} + \item + on CLR (P\#) and JVM (Prolog Caf?, tuProlog) + \begin{itemize} + \item + interfacing with libraries of the platform mostly easy + \item + no extensions to core Prolog (like tabling, coroutines) + \item + slow, compared to good C implementations + \end{itemize} + \end{itemize} +\end{frame} + + +\subsection{Approach} +\begin{frame} + \frametitle{PyPy's Approach} + \alert{Goal:} generate VMs from a single high-level description of the + language, in a retargettable way. + \begin{itemize} + \item + Write an interpreter for a dynamic language (Python, Prolog, JavaScript, + whatever) in a high-level language (Python) + \item + Leave out low-level details + \item + Favour simplicity and flexibility + \item + Define a mapping to low-level targets + \item + Generate VMs from the interpreter + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Mapping to low-level targets} + \begin{itemize} + \item + Mechanically translate the interpreter to multiple + lower-level targets + \begin{itemize} + \item C-like + \item Java + \item .NET + \end{itemize} + \item + Insert low-level aspects into the code as required by + the target (Object layout, memory management) + \begin{itemize} + \item object layout + \item memory management + \end{itemize} \item + Optionally insert new pervasive features not expressed + in the source + \begin{itemize} + \item continuations + \item dynamic compilation + \end{itemize} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Translation Steps} + \begin{columns}[c] + \begin{column}{5cm} + \includegraphics[width=5cm]{image/arch.pdf} + \end{column} + \begin{column}{7cm} + \begin{itemize} + \item + Generate flow graphs from the RPython program + \item + Peform global type inference on the flow graphs + \item + Transform flow graphs through several steps until they match the level of + the target environment + \item + Weave in translation aspects in the process + \end{itemize} + \end{column} + \end{columns} + +\end{frame} + + +\begin{frame} + \frametitle{Translation Aspects (i)} + Features not present in the source can be + added during translation. + + Example: memory management: + \begin{itemize} + \item + Boehm garbage collector + \item + mark-n-sweep written in RPython, with additional features + \item + reference counting + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Translation Aspects (ii)} + \begin{itemize} + \item + \alert{Stackless transformation}: continuation capture, implemented by + saving the low-level frames' local variables into the heap and back + \begin{itemize} + \item allows arbitrarily deep stack usage + \item uses the C stack as long as possible + \item has the consequence of making RPython do tail call elimination + \end{itemize} + \item + work in progress: turning an interpreter into a just-in-time compiler + is a translation aspect too + \end{itemize} +\end{frame} + +\section{The Prolog interpreter} +\begin{frame} + \frametitle{Prolog Interpreter Implementation} + \begin{itemize} + \item + naive, very simple interpreter + \begin{itemize} + \item uses "structure copying" + \item interprets Prolog terms directly, no bytecode + \end{itemize} + \item + uses continuation passing style similar to BinProlog + \item + Prolog calls mapped to RPython calls + \begin{itemize} + \item possible because stackless allows arbitrary deep recursion + \end{itemize} + \item + implements large parts of the ISO standard (some builtins missing) + \end{itemize} +\end{frame} + + + + +\begin{frame} + \frametitle{Builtins} + \begin{itemize} + \item + builtins implemented in Python + \item + easy to add new ones to interface with libraries + \item + application-specific builtins + \item + examples: + \begin{itemize} + \item functions to download and analyze webpages + \item an imperative hashmap + \end{itemize} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Interpreter Facts} + \begin{itemize} + \item + 2500 lines of Python code in total + \item + 700 of those are for builtins + \item + after translation to C: 14000 line of C code + \item + part of the PyPy distribution at: + http://codespeak.net/pypy + \end{itemize} +\end{frame} + + + +\begin{frame} + \frametitle{Performance (i)} + \includegraphics[scale=0.4]{image/bench.png} +\end{frame} + + +\begin{frame} + \frametitle{Performance (ii)} + \begin{itemize} + \item + performance is quite bad compared to tuned C implementations + \item + performance is pretty good compared to Java and .NET implementations + \item + surprising, since those are often based on the WAM + \item + maybe it's hard to simulate the WAM on such a VM + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Title} + \begin{itemize} + \item + \end{itemize} +\end{frame} + + +\section*{Summary} +\subsection*{Summary} +\begin{frame} + \frametitle{Summary} + + % Keep the summary *very short*. + \begin{itemize} + \item + The construction of virtual machines gets easier when using high-level + languages + \item + XXX + \item + XXX + \end{itemize} +\end{frame} +\subsection*{Outlook} +\begin{frame} + \frametitle{Outlook} + % The following outlook is optional. + \begin{itemize} + \item + \end{itemize} +\end{frame} + + + +\end{document} + + From cfbolz at codespeak.net Tue May 1 23:36:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:36:17 +0200 (CEST) Subject: [pypy-svn] r42554 - pypy/extradoc/talk/badhonnef2007 Message-ID: <20070501213617.48D098061@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:36:16 2007 New Revision: 42554 Modified: pypy/extradoc/talk/badhonnef2007/talk.pdf pypy/extradoc/talk/badhonnef2007/talk.tex Log: reworking the talk Modified: pypy/extradoc/talk/badhonnef2007/talk.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/badhonnef2007/talk.tex ============================================================================== --- pypy/extradoc/talk/badhonnef2007/talk.tex (original) +++ pypy/extradoc/talk/badhonnef2007/talk.tex Tue May 1 23:36:16 2007 @@ -111,9 +111,12 @@ \item Pyrolog is a Prolog interpreter written in RPython \item - RPython is a subset of Python translatable to other languages + RPython (``Restricted Python'') is a subset of Python translatable to other + languages + \item + RPython is designed to be significantly faster than regular Python \item - translation part done with the help of the PyPy project + translation tool-chain part of the PyPy project \end{itemize} \end{frame} @@ -123,14 +126,15 @@ \frametitle{What is PyPy?} \begin{itemize} \item - started as a Python VM implementation - in RPython (a well-chosen subset of Python) + started as a Python VM implementation in Python \item - includes a translation tool-chain + Python itself is too dynamic to be translatable to other languages, need a subset + \item + includes a translation tool-chain for RPython \item is becoming a general environment for writing interpreters (JavaScript, Prolog started) \item - Open source project (MIT license) + Open Source project (MIT license) \item received EU funding for 2.5 years \end{itemize} @@ -140,7 +144,7 @@ \subsection{Motivation} \begin{frame} \frametitle{VMs are still hard} - It is hard to achieve: + Hard to reconcile: \begin{itemize} \item @@ -160,12 +164,13 @@ \begin{itemize} \item - Pervasive decisions: reference counting, single global lock ... + Pervasive decisions: reference counting, global lock \dots \item No dynamic compilation % \pause - \item - Extensions: + \end{itemize} + \begin{block}{ + Extensions:} \begin{itemize} \item \alert{Stackless} (unlimited recursion, coroutines, serializable continuations) @@ -174,23 +179,20 @@ \item \alert{Jython}, \alert{IronPython} \end{itemize} - \end{itemize} + \end{block} \end{frame} \begin{frame} \frametitle{The Python case (ii)} - \begin{itemize} - \item - Extensions have problems + \begin{block}{Problems of Extensions:} \begin{itemize} - \item - need to keep track of CPython - \item - are hard to maintain - \item - Psyco very hard to port to other hardware architectures + \item hard to maintain: need to keep track of CPython + \item Psyco very hard to port to other hardware architectures + \item tedious to write them, Python semantics need to be re-implemented \end{itemize} + \end{block} + \begin{itemize} \item The community wants Python to run everywhere: Jython (Java), IronPython (.NET). @@ -205,26 +207,36 @@ \begin{frame} - \frametitle{The Prolog case} + \frametitle{The Prolog case (i)} \begin{itemize} \item problem mitigated by the fact that Prolog the language does not change \item + the core of Prolog is very simple (at least compared to Python) + \item a lot of implementations out there \item - well-tuned mature C implementations (Sictsus, XSB, SWI, GNU-Prolog) + well-tuned mature C implementations (Sicstus, XSB, SWI, GNU-Prolog) + \item + on CLR (P\#) and JVM (Prolog Caf?, tuProlog) + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{The Prolog case (ii)} + \begin{block}{mature C implementations} \begin{itemize} \item - have sometimes incompatible extensions to core Prolog - \item interfacing with libraries is tedious \item changing the language to experiment is hard \item + often extensions to core Prolog, incompatible between each other + \item fixed implementation decisions (GC, how to generate code, etc.) \end{itemize} - \item - on CLR (P\#) and JVM (Prolog Caf?, tuProlog) + \end{block} + \begin{block}{implementations on CLR and JVM} \begin{itemize} \item interfacing with libraries of the platform mostly easy @@ -233,10 +245,11 @@ \item slow, compared to good C implementations \end{itemize} - \end{itemize} + \end{block} \end{frame} + \subsection{Approach} \begin{frame} \frametitle{PyPy's Approach} @@ -279,72 +292,42 @@ Optionally insert new pervasive features not expressed in the source \begin{itemize} - \item continuations + \item continuations, ``micro-threads'' \item dynamic compilation \end{itemize} \end{itemize} \end{frame} -\begin{frame} - \frametitle{Translation Steps} - \begin{columns}[c] - \begin{column}{5cm} - \includegraphics[width=5cm]{image/arch.pdf} - \end{column} - \begin{column}{7cm} - \begin{itemize} - \item - Generate flow graphs from the RPython program - \item - Peform global type inference on the flow graphs - \item - Transform flow graphs through several steps until they match the level of - the target environment - \item - Weave in translation aspects in the process - \end{itemize} - \end{column} - \end{columns} - -\end{frame} - \begin{frame} \frametitle{Translation Aspects (i)} - Features not present in the source can be - added during translation. - - Example: memory management: + Features not present in the source can be added during translation. \begin{itemize} \item - Boehm garbage collector - \item - mark-n-sweep written in RPython, with additional features + \alert{memory management}: use different GC strategies (Boehm collector, custom mark-n-sweep) \item - reference counting + \alert{Stackless transformation}: allows program to control its stack (continuations, \dots) \end{itemize} \end{frame} \begin{frame} \frametitle{Translation Aspects (ii)} + A \alert{JIT compiler} as a translation aspect \begin{itemize} \item - \alert{Stackless transformation}: continuation capture, implemented by - saving the low-level frames' local variables into the heap and back - \begin{itemize} - \item allows arbitrarily deep stack usage - \item uses the C stack as long as possible - \item has the consequence of making RPython do tail call elimination - \end{itemize} + Transform the interpreter into a JIT compiler, using partial evaluation and + specialization techniques \item - work in progress: turning an interpreter into a just-in-time compiler - is a translation aspect too + Some hints in the interpreter source needed + \item + Current prototype applied to Python interpreter gives impressive speedups \end{itemize} \end{frame} \section{The Prolog interpreter} +\subsection{Interpreter} \begin{frame} \frametitle{Prolog Interpreter Implementation} \begin{itemize} @@ -355,12 +338,9 @@ \item interprets Prolog terms directly, no bytecode \end{itemize} \item - uses continuation passing style similar to BinProlog + uses continuation passing style inspired by BinProlog \item Prolog calls mapped to RPython calls - \begin{itemize} - \item possible because stackless allows arbitrary deep recursion - \end{itemize} \item implements large parts of the ISO standard (some builtins missing) \end{itemize} @@ -399,12 +379,12 @@ after translation to C: 14000 line of C code \item part of the PyPy distribution at: - http://codespeak.net/pypy + \texttt{http://codespeak.net/pypy} \end{itemize} \end{frame} - +\subsection{Performance} \begin{frame} \frametitle{Performance (i)} \includegraphics[scale=0.4]{image/bench.png} @@ -421,14 +401,7 @@ \item surprising, since those are often based on the WAM \item - maybe it's hard to simulate the WAM on such a VM - \end{itemize} -\end{frame} - -\begin{frame} - \frametitle{Title} - \begin{itemize} - \item + maybe the WAM model does not match these VMs very well? \end{itemize} \end{frame} @@ -436,23 +409,61 @@ \section*{Summary} \subsection*{Summary} \begin{frame} - \frametitle{Summary} + \frametitle{Summary} % Keep the summary *very short*. \begin{itemize} \item - The construction of virtual machines gets easier when using high-level - languages + Very simple Prolog interpreter in RPython can compete with interpreters on the JVM, CLR \item - XXX + Interpreter implementation eased by use of a high-level language \item - XXX + Low-level details abstracted away but re-inserted later \end{itemize} \end{frame} + \subsection*{Outlook} \begin{frame} - \frametitle{Outlook} - % The following outlook is optional. + \frametitle{Outlook} + \begin{itemize} + \item + complete the set of builtins + \item + tight language integration between Prolog and Python + \item + apply the dynamic compiler generator to the Prolog interpreter + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Backup slides} + \dots +\end{frame} + +\begin{frame} + \frametitle{Translation Steps} + \begin{columns}[c] + \begin{column}{5cm} + \includegraphics[width=5cm]{image/arch.pdf} + \end{column} + \begin{column}{7cm} + \begin{itemize} + \item + Generate flow graphs from the RPython program + \item + Peform global type inference on the flow graphs + \item + Transform flow graphs through several steps until they match the level of + the target environment + \item + Weave in translation aspects in the process + \end{itemize} + \end{column} + \end{columns} +\end{frame} + +\begin{frame} + \frametitle{Title} \begin{itemize} \item \end{itemize} From cfbolz at codespeak.net Tue May 1 23:41:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:41:30 +0200 (CEST) Subject: [pypy-svn] r42556 - pypy/extradoc/eu-report Message-ID: <20070501214130.86AF3807E@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:41:29 2007 New Revision: 42556 Added: pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-2007-05-01.pdf (contents, props changed) Removed: pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-interim-2007-01-28.pdf Log: final D08.2 report Added: pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-2007-05-01.pdf ============================================================================== Binary file. No diff available. From cfbolz at codespeak.net Tue May 1 23:43:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:43:30 +0200 (CEST) Subject: [pypy-svn] r42557 - pypy/dist/pypy/doc Message-ID: <20070501214330.E3141807C@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:43:30 2007 New Revision: 42557 Modified: pypy/dist/pypy/doc/index-report.txt Log: update link here Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Tue May 1 23:43:30 2007 @@ -12,6 +12,9 @@ Reports of 2007 =============== +`D08.2 JIT Compiler Architecture`_ is a report about the Architecture and +working of our JIT compiler generator. *(2007-05-01)* + `D08.1 JIT Compiler Release`_ reports on our sucessfully including a JIT compiler for Python and the novel framework we used to automatically generate it in PyPy 1.0. *(2007-04-30)* @@ -82,10 +85,6 @@ feedback for it is welcome. *(2007-02-28)* -`Draft D08.2 JIT Compiler Architecture`_ is an interim version of a report -about the Architecture and working of our JIT compiler generator. The report -is still a draft, all feedback for it is welcome. *(2007-01-28)* - .. _`py-lib`: http://codespeak.net/py/current/doc/ .. _`py.test`: http://codespeak.net/py/current/doc/test.html @@ -146,7 +145,7 @@ .. _`D05.4 Encapsulating Low Level Aspects`: http://codespeak.net/svn/pypy/extradoc/eu-report/D05.4_Publish_on_encapsulating_low_level_language_aspects.pdf .. _`D06.1 Core Object Optimization Results`: http://codespeak.net/svn/pypy/extradoc/eu-report/D06.1_Core_Optimizations-2007-04-30.pdf .. _`D07.1 Massive Parallelism and Translation Aspects`: http://codespeak.net/pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-2007-02-28.pdf -.. _`Draft D08.2 JIT Compiler Architecture`: http://codespeak.net/pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-interim-2007-01-28.pdf +.. _`D08.2 JIT Compiler Architecture`: http://codespeak.net/pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-2007-05-01.pdf .. _`D08.1 JIT Compiler Release`: http://codespeak.net/pypy/extradoc/eu-report/D08.1_JIT_Compiler_Release-2007-04-30.pdf .. _`D09.1 Constraint Solving and Semantic Web`: http://codespeak.net/pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-interim-2007-02-28.pdf .. _`D10.1 Aspect-Oriented, Design-by-Contract Programming and RPython static checking`: http://codespeak.net/pypy/extradoc/eu-report/D10.1_Aspect_Oriented_Programming_in_PyPy-2007-03-22.pdf From cfbolz at codespeak.net Tue May 1 23:50:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:50:38 +0200 (CEST) Subject: [pypy-svn] r42559 - pypy/extradoc/eu-report Message-ID: <20070501215038.55FB0807E@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:50:37 2007 New Revision: 42559 Added: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf - copied unchanged from r42558, pypy/eu-tracking/deliverable/D14_4_Report_About_Milestone_Phase_3/D14.4-deliverablereport.pdf Removed: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-interim-2007-2007-04-05.pdf Log: final version of 14.4 From cfbolz at codespeak.net Tue May 1 23:52:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:52:50 +0200 (CEST) Subject: [pypy-svn] r42560 - pypy/dist/pypy/doc Message-ID: <20070501215250.71818807C@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:52:49 2007 New Revision: 42560 Modified: pypy/dist/pypy/doc/index-report.txt Log: update link Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Tue May 1 23:52:49 2007 @@ -12,6 +12,10 @@ Reports of 2007 =============== +`D14.4 PyPy-1.0 Milestone report`_ (for language developers and researchers) +summarizes research & technical results of the PyPy-1.0 release and discusses +related development process and community aspects. *(2007-05-01)* + `D08.2 JIT Compiler Architecture`_ is a report about the Architecture and working of our JIT compiler generator. *(2007-05-01)* @@ -19,11 +23,6 @@ JIT compiler for Python and the novel framework we used to automatically generate it in PyPy 1.0. *(2007-04-30)* -`D14.4 PyPy-1.0 Milestone report`_ (for language developers and researchers) -summarizes research & technical results of the PyPy-1.0 release and discusses -related development process and community aspects. This report is an interim -version, we may still incorporate feedback. *(2007-04-05)* - `D06.1 Core Object Optimization Results`_ documents the optimizations we implemented in the interpreter and object space: dictionary implementations, method call optimizations, etc. The report is still not final @@ -155,7 +154,7 @@ .. _`D14.1 Report about Milestone/Phase 1`: http://codespeak.net/svn/pypy/extradoc/eu-report/D14.1_Report_about_Milestone_Phase_1.pdf .. _`D14.2 Tutorials and Guide Through the PyPy Source Code`: http://codespeak.net/pypy/extradoc/eu-report/D14.2_Tutorials_and_Guide_Through_the_PyPy_Source_Code-2007-03-22.pdf .. _`D14.3 Report about Milestone/Phase 2`: http://codespeak.net/pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf -.. _`D14.4 PyPy-1.0 Milestone report`: http://codespeak.net/pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-interim-2007-2007-04-05.pdf +.. _`D14.4 PyPy-1.0 Milestone report`: http://codespeak.net/pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf .. _`D14.5 Documentation of the development process`: http://codespeak.net/pypy/extradoc/eu-report/D14.5_Documentation_of_the_development_process-2007-03-30.pdf From cfbolz at codespeak.net Tue May 1 23:54:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:54:40 +0200 (CEST) Subject: [pypy-svn] r42561 - pypy/extradoc/eu-report Message-ID: <20070501215440.5EF78807C@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:54:39 2007 New Revision: 42561 Added: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-05-01.pdf - copied unchanged from r42559, pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf Removed: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf Log: remove double 2007 in filename ?! From cfbolz at codespeak.net Tue May 1 23:55:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:55:22 +0200 (CEST) Subject: [pypy-svn] r42562 - pypy/dist/pypy/doc Message-ID: <20070501215522.0EC4C807F@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:55:21 2007 New Revision: 42562 Modified: pypy/dist/pypy/doc/index-report.txt Log: remove it here too Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Tue May 1 23:55:21 2007 @@ -154,7 +154,7 @@ .. _`D14.1 Report about Milestone/Phase 1`: http://codespeak.net/svn/pypy/extradoc/eu-report/D14.1_Report_about_Milestone_Phase_1.pdf .. _`D14.2 Tutorials and Guide Through the PyPy Source Code`: http://codespeak.net/pypy/extradoc/eu-report/D14.2_Tutorials_and_Guide_Through_the_PyPy_Source_Code-2007-03-22.pdf .. _`D14.3 Report about Milestone/Phase 2`: http://codespeak.net/pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf -.. _`D14.4 PyPy-1.0 Milestone report`: http://codespeak.net/pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf +.. _`D14.4 PyPy-1.0 Milestone report`: http://codespeak.net/pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-05-01.pdf .. _`D14.5 Documentation of the development process`: http://codespeak.net/pypy/extradoc/eu-report/D14.5_Documentation_of_the_development_process-2007-03-30.pdf From santagada at codespeak.net Wed May 2 05:57:04 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 2 May 2007 05:57:04 +0200 (CEST) Subject: [pypy-svn] r42570 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070502035704.D126C807C@code0.codespeak.net> Author: santagada Date: Wed May 2 05:57:03 2007 New Revision: 42570 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: started testing of expressions... we are getting closer to the point where this parser is going to take the place of using narcissus + spidermonkey Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Wed May 2 05:57:03 2007 @@ -15,9 +15,9 @@ statement : | [";"] - | [ ";" ] + | | [";"] - | [";"] + | [";"]* | [";"] | [";"] | [";"] @@ -49,6 +49,9 @@ initialiser : ["="] assignmentexpression ; +emptystatement : ";" + ; + expressionstatement : expression ; @@ -88,7 +91,11 @@ ; # XXX this looks wrong to me -caseclauses : caseclause caseclauses +# ans: it was defined in the spec as +# CaseClauses : +# CaseClause +# CaseClauses CaseClause +caseclauses : caseclause+ ; caseclause : ["case"] expression [":"] statementlist? Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Wed May 2 05:57:03 2007 @@ -15,7 +15,7 @@ newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules return newrules -parse = make_parse_function(regexs, setstartrule(rules, start="expression"), eof=True) +parse = make_parse_function(regexs, setstartrule(rules, start="statement"), eof=True) #parse = make_parse_function(regexs, rules) print rules[2].nonterminal Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Wed May 2 05:57:03 2007 @@ -53,7 +53,7 @@ self.counts = {} def general_nonterminal_visit(self, node): - print node + print node.symbol self.counts[node.symbol] = self.counts.get(node.symbol, 0) + 1 for child in node.children: self.dispatch(child) @@ -157,6 +157,8 @@ ]) self.parse_and_compare("++5", 6) self.parse_and_compare("~3", -4) + self.parse("x = 3") + self.parse("x") def test_chained(self): self.parse_and_eval_all(["1 + 2 * 3", @@ -167,3 +169,39 @@ "30 | 3 & 5", ]) +class TestStatements(BaseGrammarTest): + def setup_class(cls): + cls.parse = parse_func('statement') + + def parse_count(self, s): + "parse the expression and return the CountingVisitor" + cv = CountingVisitor() + self.parse(s).visit(cv) + return cv.counts + + def test_block(self): + r = self.parse_count("{x;return;true;if(x);}") + assert r['block'] == 1 + + def test_vardecl(self): + r = self.parse_count("var x;") + assert r['variablestatement'] == 1 + + r = self.parse_count("var x = 2;") + assert r['variablestatement'] == 1 + + def test_empty(self): + for i in range(1,10): + r = self.parse_count('{%s}'%(';'*i)) + assert r['emptystatement'] == i + + def test_if(self): + r = self.parse_count("if(x)return;;") + assert r['ifstatement'] == 1 + assert r.get('emptystatement', 0) == 0 + r = self.parse_count("if(x)if(i)return;") + assert r['ifstatement'] == 2 + r = self.parse_count("if(x)return;else return;") + assert r['ifstatement'] == 1 + + From santagada at codespeak.net Wed May 2 06:03:30 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 2 May 2007 06:03:30 +0200 (CEST) Subject: [pypy-svn] r42571 - pypy/dist/pypy/lang/js Message-ID: <20070502040330.7C5D8807F@code0.codespeak.net> Author: santagada Date: Wed May 2 06:03:30 2007 New Revision: 42571 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt Log: should be a ? not a *... but still I dunno if this is correct or not Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Wed May 2 06:03:30 2007 @@ -17,7 +17,7 @@ | [";"] | | [";"] - | [";"]* + | [";"]? | [";"] | [";"] | [";"] From antocuni at codespeak.net Wed May 2 23:47:08 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 2 May 2007 23:47:08 +0200 (CEST) Subject: [pypy-svn] r42600 - in pypy/dist/pypy/translator: . cli cli/test Message-ID: <20070502214708.7B0B2807F@code0.codespeak.net> Author: antocuni Date: Wed May 2 23:47:07 2007 New Revision: 42600 Added: pypy/dist/pypy/translator/cli/silverpython.py (contents, props changed) pypy/dist/pypy/translator/cli/test/test_silverpython.py (contents, props changed) Modified: pypy/dist/pypy/translator/cli/cts.py pypy/dist/pypy/translator/cli/entrypoint.py pypy/dist/pypy/translator/cli/function.py pypy/dist/pypy/translator/cli/gencli.py pypy/dist/pypy/translator/cli/rte.py pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/driver.py Log: - add support to driver for building libraries instead of standalone executables (i.e., allowing more than one entry-point) - first steps to write a gencli's frontend which allow to compile a rpython module to a .NET dll; the draft name is SilveRPython, suggestions are welcome :-) Modified: pypy/dist/pypy/translator/cli/cts.py ============================================================================== --- pypy/dist/pypy/translator/cli/cts.py (original) +++ pypy/dist/pypy/translator/cli/cts.py Wed May 2 23:47:07 2007 @@ -196,6 +196,9 @@ ret_type, ret_var = self.llvar_to_cts(graph.getreturnvar()) func_name = func_name or graph.name func_name = self.escape_name(func_name) + namespace = getattr(graph.func, '_namespace_', None) + if namespace: + func_name = '%s::%s' % (namespace, func_name) args = [arg for arg in graph.getargs() if arg.concretetype is not ootype.Void] if is_method: Modified: pypy/dist/pypy/translator/cli/entrypoint.py ============================================================================== --- pypy/dist/pypy/translator/cli/entrypoint.py (original) +++ pypy/dist/pypy/translator/cli/entrypoint.py Wed May 2 23:47:07 2007 @@ -19,6 +19,12 @@ self.db = db self.cts = CTS(db) + def ilasm_flags(self): + return [] + + def output_filename(self, il_filename): + return il_filename.replace('.il', '.exe') + class StandaloneEntryPoint(BaseEntryPoint): """ This class produces a 'main' method that converts the argv in a @@ -59,3 +65,21 @@ ilasm.opcode('ret') ilasm.end_function() self.db.pending_function(self.graph) + +class DllEntryPoint(BaseEntryPoint): + def __init__(self, name, graphs): + self.name = name + self.graphs = graphs + + def get_name(self): + return self.name + + def ilasm_flags(self): + return ['/dll'] + + def output_filename(self, il_filename): + return il_filename.replace('.il', '.dll') + + def render(self, ilasm): + for graph in self.graphs: + self.db.pending_function(graph) Modified: pypy/dist/pypy/translator/cli/function.py ============================================================================== --- pypy/dist/pypy/translator/cli/function.py (original) +++ pypy/dist/pypy/translator/cli/function.py Wed May 2 23:47:07 2007 @@ -173,7 +173,18 @@ OOFunction.__init__(self, *args, **kwargs) self._set_args() self._set_locals() - + namespace = getattr(self.graph.func, '_namespace_', None) + str + if namespace: + if '.' in namespace: + self.namespace, self.classname = namespace.rsplit('.', 1) + else: + self.namespace = None + self.classname = namespace + else: + self.namespace = None + self.classname = None + def _create_generator(self, ilasm): return self # Function implements the Generator interface @@ -192,11 +203,20 @@ else: args = self.args meth_type = 'static' - self.ilasm.begin_function(self.name, args, returntype, self.is_entrypoint, meth_type) + + if self.namespace: + self.ilasm.begin_namespace(self.namespace) + if self.classname: + self.ilasm.begin_class(self.classname) + self.ilasm.begin_function(self.name, args, returntype, self.is_entrypoint, meth_type) self.ilasm.locals(self.locals) def end_render(self): self.ilasm.end_function() + if self.classname: + self.ilasm.end_class() + if self.namespace: + self.ilasm.end_namespace() def set_label(self, label): self.ilasm.label(label) Modified: pypy/dist/pypy/translator/cli/gencli.py ============================================================================== --- pypy/dist/pypy/translator/cli/gencli.py (original) +++ pypy/dist/pypy/translator/cli/gencli.py Wed May 2 23:47:07 2007 @@ -87,20 +87,20 @@ ilasm = SDK.ilasm() tmpfile = self.tmpfile.strpath - self._exec_helper(ilasm, tmpfile, + self._exec_helper(ilasm, [tmpfile]+self.entrypoint.ilasm_flags(), 'ilasm failed to assemble (%s):\n%s\n%s', timeout = 900) # Mono's ilasm occasionally deadlocks. We set a timer to avoid # blocking automated test runs forever. - exefile = tmpfile.replace('.il', '.exe') + self.outfile = self.entrypoint.output_filename(tmpfile) if getoption('verify'): peverify = SDK.peverify() - self._exec_helper(peverify, exefile, 'peverify failed to verify (%s):\n%s\n%s') - return exefile + self._exec_helper(peverify, [outfile], 'peverify failed to verify (%s):\n%s\n%s') + return self.outfile - def _exec_helper(self, helper, filename, msg, timeout=None): - args = [helper, filename] + def _exec_helper(self, helper, args, msg, timeout=None): + args = [helper] + args if timeout and not sys.platform.startswith('win'): import os from pypy.tool import autopath @@ -109,5 +109,5 @@ proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() retval = proc.wait() - assert retval == 0, msg % (filename, stdout, stderr) + assert retval == 0, msg % (args[0], stdout, stderr) Modified: pypy/dist/pypy/translator/cli/rte.py ============================================================================== --- pypy/dist/pypy/translator/cli/rte.py (original) +++ pypy/dist/pypy/translator/cli/rte.py Wed May 2 23:47:07 2007 @@ -11,14 +11,9 @@ from py.compat import subprocess from pypy.translator.cli.sdk import SDK from pypy.tool.ansi_print import ansi_log -log = py.log.Producer("cli") -py.log.setconsumer("cli", ansi_log) +log = py.log.Producer("cli") +py.log.setconsumer("cli", ansi_log) -SRC_DIR = os.path.join(os.path.dirname(__file__), 'src/') - -def _filename(name, path=None): - rel_path = os.path.join(SRC_DIR, name) - return os.path.abspath(rel_path) class Target: SOURCES = [] @@ -26,6 +21,12 @@ ALIAS = None FLAGS = [] DEPENDENCIES = [] + SRC_DIR = os.path.join(os.path.dirname(__file__), 'src/') + + def _filename(cls, name, path=None): + rel_path = os.path.join(cls.SRC_DIR, name) + return os.path.abspath(rel_path) + _filename = classmethod(_filename) def get_COMPILER(cls): return SDK.csc() @@ -34,9 +35,9 @@ def get(cls): for dep in cls.DEPENDENCIES: dep.get() - sources = [_filename(src) for src in cls.SOURCES] - out = _filename(cls.OUTPUT) - alias = _filename(cls.ALIAS or cls.OUTPUT) + sources = [cls._filename(src) for src in cls.SOURCES] + out = cls._filename(cls.OUTPUT) + alias = cls._filename(cls.ALIAS or cls.OUTPUT) recompile = True try: src_mtime = max([os.stat(src).st_mtime for src in sources]) @@ -54,14 +55,14 @@ def compile(cls, sources, out): log.red("Compiling %s" % (cls.ALIAS or cls.OUTPUT)) oldcwd = os.getcwd() - os.chdir(SRC_DIR) + os.chdir(cls.SRC_DIR) compiler = subprocess.Popen([cls.get_COMPILER()] + cls.FLAGS + ['/out:%s' % out] + sources, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = compiler.communicate() retval = compiler.wait() assert retval == 0, 'Failed to compile %s: the compiler said:\n %s' % (cls.OUTPUT, stderr) if cls.ALIAS is not None: - alias = _filename(cls.ALIAS) + alias = cls._filename(cls.ALIAS) shutil.copy(out, alias) os.chdir(oldcwd) Added: pypy/dist/pypy/translator/cli/silverpython.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/cli/silverpython.py Wed May 2 23:47:07 2007 @@ -0,0 +1,44 @@ +from pypy.translator.driver import TranslationDriver +from pypy.translator.cli.entrypoint import DllEntryPoint + +class DllDef: + def __init__(self, name, namespace, functions=[], classes=[]): + self.name = name + self.namespace = namespace + self.functions = functions # [(function, annotation), ...] + + def add_function(self, func, inputtypes): + self.functions.append((func, inputtypes)) + + def get_entrypoint(self, bk): + graphs = [bk.getdesc(f).cachedgraph(None) for f, _ in self.functions] + return DllEntryPoint(self.name, graphs) + + def compile(self): + # add all functions to the appropriate namespace + for func, _ in self.functions: + if not hasattr(func, '_namespace_'): + func._namespace_ = self.namespace + driver = TranslationDriver() + driver.setup_library(self) + driver.proceed(['compile_cli']) + return driver + + +class MyClass: + def __init__(self, x): + self.x = x + + def foo(self): + return self.x + +def main(): + dll = DllDef('mylibrary', 'foo', [], [ + (MyClass, [int]), + ]) + driver = dll.compile() + driver.copy_cli_dll() + + +if __name__ == '__main__': + main() Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Wed May 2 23:47:07 2007 @@ -137,15 +137,15 @@ assert False, 'Input type %s not supported' % arg_type -def compile_function(func, annotation=[], graph=None, backend_opt={}): +def compile_function(func, annotation=[], graph=None, backendopt=True): olddefs = patch() - gen = _build_gen(func, annotation, graph, backend_opt) + gen = _build_gen(func, annotation, graph, backendopt) gen.generate_source() exe_name = gen.build_exe() unpatch(*olddefs) # restore original values return CliFunctionWrapper(exe_name) -def _build_gen(func, annotation, graph=None, backend_opt={}): +def _build_gen(func, annotation, graph=None, backendopt=True): try: func = func.im_func except AttributeError: @@ -165,8 +165,9 @@ t.view() t.buildrtyper(type_system="ootype").specialize() - check_virtual_methods(ootype.ROOT) - backend_optimizations(t) + if backendopt: + check_virtual_methods(ootype.ROOT) + backend_optimizations(t) main_graph = t.graphs[0] @@ -242,13 +243,13 @@ self._ann = None self._cli_func = None - def _compile(self, fn, args, ann=None): + def _compile(self, fn, args, ann=None, backendopt=True): if ann is None: ann = [lltype_to_annotation(typeOf(x)) for x in args] if self._func is fn and self._ann == ann: return self._cli_func else: - self._cli_func = compile_function(fn, ann) + self._cli_func = compile_function(fn, ann, backendopt=backendopt) self._func = fn self._ann = ann return self._cli_func @@ -261,8 +262,8 @@ if platform.processor() == 'powerpc': py.test.skip('PowerPC --> %s' % reason) - def interpret(self, fn, args, annotation=None): - f = self._compile(fn, args, annotation) + def interpret(self, fn, args, annotation=None, backendopt=True): + f = self._compile(fn, args, annotation, backendopt) res = f(*args) if isinstance(res, ExceptionWrapper): raise res Added: pypy/dist/pypy/translator/cli/test/test_silverpython.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/cli/test/test_silverpython.py Wed May 2 23:47:07 2007 @@ -0,0 +1,58 @@ +from pypy.tool import udir +from pypy.translator.cli.rte import Target +from pypy.translator.cli.silverpython import DllDef +from pypy.translator.cli.test.runtest import CliFunctionWrapper, CliTest + +TEMPLATE = """ +using System; +class SilveRPytonTest { + public static void Main() { + %s + } +} +""" + +class TestSilveRPython(CliTest): + + def _csharp(self, reference, source): + tmpfile = udir.udir.join('tmp.cs') + tmpfile.write(TEMPLATE % source) + if reference is None: + flags = [] + else: + flags = ['/r:%s' % reference] + + class MyTarget(Target): + SOURCES = [str(tmpfile)] + FLAGS = flags + OUTPUT = 'tmp.exe' + SRC_DIR = str(udir.udir) + + func = CliFunctionWrapper(MyTarget.get()) + return func() + + + def test_compilation(self): + res = self._csharp(None, 'Console.WriteLine(42);') + assert res == 42 + + def test_func_namespace(self): + def foo(x): + return x+1 + def bar(x): + return foo(x) + foo._namespace_ = 'MyNamespace.MyClass' + bar._namespace_ = 'MyClass' + res = self.interpret(bar, [41], backendopt=False) + assert res == 42 + + def test_simple_functions(self): + def foo(x): + return x+1 + def bar(x): + return x*2 + dll = DllDef('test', 'Test', [(foo, [int]), + (bar, [int])]) + dll.compile() + res = self._csharp('test', 'Console.WriteLine("{0}, {1}", Test.foo(42), Test.bar(42));') + assert res == (43, 84) Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Wed May 2 23:47:07 2007 @@ -209,9 +209,14 @@ self.entry_point = entry_point self.translator = translator + self.libdef = None self.translator.driver_instrument_result = self.instrument_result + def setup_library(self, libdef, policy=None, extra={}, empty_translator=None): + self.setup(None, None, policy, extra, empty_translator) + self.libdef = libdef + def instrument_result(self, args): backend, ts = self.get_backend_and_type_system() if backend != 'c' or sys.platform == 'win32': @@ -275,16 +280,22 @@ annmodel.DEBUG = self.config.translation.debug annotator = translator.buildannotator(policy=policy) - - s = annotator.build_types(self.entry_point, self.inputtypes) - - self.sanity_check_annotation() - if self.standalone and s.knowntype != int: - raise Exception("stand-alone program entry point must return an " - "int (and not, e.g., None or always raise an " - "exception).") + + if self.entry_point: + s = annotator.build_types(self.entry_point, self.inputtypes) + + self.sanity_check_annotation() + if self.standalone and s.knowntype != int: + raise Exception("stand-alone program entry point must return an " + "int (and not, e.g., None or always raise an " + "exception).") + return s + else: + assert self.libdef is not None + for func, inputtypes in self.libdef.functions: + annotator.build_types(func, inputtypes) + self.sanity_check_annotation() annotator.simplify() - return s # task_annotate = taskdef(task_annotate, [], "Annotating&simplifying") @@ -619,9 +630,16 @@ from pypy.translator.cli.gencli import GenCli from pypy.translator.cli.entrypoint import get_entrypoint - entry_point_graph = self.translator.graphs[0] - self.gen = GenCli(udir, self.translator, get_entrypoint(entry_point_graph), - config=self.config) + if self.entry_point is not None: # executable mode + entry_point_graph = self.translator.graphs[0] + entry_point = get_entrypoint(entry_point_graph) + else: + # library mode + assert self.libdef is not None + bk = self.translator.annotator.bookkeeper + entry_point = self.libdef.get_entrypoint(bk) + + self.gen = GenCli(udir, self.translator, entry_point, config=self.config) filename = self.gen.generate_source() self.log.info("Wrote %s" % (filename,)) task_source_cli = taskdef(task_source_cli, ["?" + OOBACKENDOPT, OOTYPE], @@ -667,6 +685,15 @@ f.close() os.chmod(newexename, 0755) + def copy_cli_dll(self): + import os.path + import shutil + dllname = self.gen.outfile + usession_path, dll_name = os.path.split(dllname) + pypylib_dll = os.path.join(usession_path, 'pypylib.dll') + shutil.copy(dllname, '.') + shutil.copy(pypylib_dll, '.') + def task_run_cli(self): pass task_run_cli = taskdef(task_run_cli, ['compile_cli'], From antocuni at codespeak.net Wed May 2 23:56:39 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 2 May 2007 23:56:39 +0200 (CEST) Subject: [pypy-svn] r42601 - pypy/dist/pypy/translator Message-ID: <20070502215639.4C25B8080@code0.codespeak.net> Author: antocuni Date: Wed May 2 23:56:38 2007 New Revision: 42601 Modified: pypy/dist/pypy/translator/driver.py Log: oops! Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Wed May 2 23:56:38 2007 @@ -289,13 +289,14 @@ raise Exception("stand-alone program entry point must return an " "int (and not, e.g., None or always raise an " "exception).") + annotator.simplify() return s else: assert self.libdef is not None for func, inputtypes in self.libdef.functions: annotator.build_types(func, inputtypes) self.sanity_check_annotation() - annotator.simplify() + annotator.simplify() # task_annotate = taskdef(task_annotate, [], "Annotating&simplifying") From arigo at codespeak.net Thu May 3 10:47:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 May 2007 10:47:31 +0200 (CEST) Subject: [pypy-svn] r42610 - pypy/extradoc/talk/badhonnef2007 Message-ID: <20070503084731.2BDCD807C@code0.codespeak.net> Author: arigo Date: Thu May 3 10:47:30 2007 New Revision: 42610 Modified: pypy/extradoc/talk/badhonnef2007/talk.tex Log: - repetition - typo Otherwise the talk looks very good to me. Modified: pypy/extradoc/talk/badhonnef2007/talk.tex ============================================================================== --- pypy/extradoc/talk/badhonnef2007/talk.tex (original) +++ pypy/extradoc/talk/badhonnef2007/talk.tex Thu May 3 10:47:30 2007 @@ -284,7 +284,7 @@ \end{itemize} \item Insert low-level aspects into the code as required by - the target (Object layout, memory management) + the target \begin{itemize} \item object layout \item memory management @@ -376,7 +376,7 @@ \item 700 of those are for builtins \item - after translation to C: 14000 line of C code + after translation to C: 14000 lines of C code \item part of the PyPy distribution at: \texttt{http://codespeak.net/pypy} From antocuni at codespeak.net Thu May 3 11:08:11 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 3 May 2007 11:08:11 +0200 (CEST) Subject: [pypy-svn] r42613 - pypy/dist/pypy/translator/cli/test Message-ID: <20070503090811.2A3CC807C@code0.codespeak.net> Author: antocuni Date: Thu May 3 11:08:10 2007 New Revision: 42613 Modified: pypy/dist/pypy/translator/cli/test/test_backendopt.py Log: typo Modified: pypy/dist/pypy/translator/cli/test/test_backendopt.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_backendopt.py (original) +++ pypy/dist/pypy/translator/cli/test/test_backendopt.py Thu May 3 11:08:10 2007 @@ -4,15 +4,11 @@ TestTypedOptimizedSwitchTestCase as c_TestTypedOptimizedSwitchTestCase class CTestCompat: - backend_opt = { - 'merge_if_blocks': True - } - def CodeGenerator(self): return self def getcompiled(self, fn, annotation): - return compile_function(fn, annotation, backend_opt=self.backend_opt) + return compile_function(fn, annotation, backendopt=True) class TestOptimizedSwitchTestCase(CTestCompat, c_TestTypedOptimizedSwitchTestCase): From afa at codespeak.net Thu May 3 22:38:23 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 3 May 2007 22:38:23 +0200 (CEST) Subject: [pypy-svn] r42648 - pypy/dist/pypy/module/select/test Message-ID: <20070503203823.EF5C38075@code0.codespeak.net> Author: afa Date: Thu May 3 22:38:23 2007 New Revision: 42648 Modified: pypy/dist/pypy/module/select/test/test_select.py Log: Refactor test_select.py, to allow multiple kinds of selectable objects: - pipes created with os.pipe() - connected sockets, but without using socketpair(), to have a chance to make it work on win32. Modified: pypy/dist/pypy/module/select/test/test_select.py ============================================================================== --- pypy/dist/pypy/module/select/test/test_select.py (original) +++ pypy/dist/pypy/module/select/test/test_select.py Thu May 3 22:38:23 2007 @@ -1,14 +1,7 @@ import py, sys from pypy.conftest import gettestobjspace -class AppTestSelect: - def setup_class(cls): - if sys.platform == 'win': - py.test.skip("select() doesn't work with pipes, " - "we would need tests using sockets") - space = gettestobjspace(usemodules=('select',)) - cls.space = space - +class _AppTestSelect: def test_sleep(self): import time, select start = time.time() @@ -18,22 +11,22 @@ assert end - start > 0.25 def test_readable(self): - import os, select - readend, writeend = os.pipe() + import select + readend, writeend = getpair() try: iwtd, owtd, ewtd = select.select([readend], [], [], 0) assert iwtd == owtd == ewtd == [] - os.write(writeend, 'X') + writeend.send('X') iwtd, owtd, ewtd = select.select([readend], [], []) assert iwtd == [readend] assert owtd == ewtd == [] finally: - os.close(writeend) - os.close(readend) + writeend.close() + readend.close() def test_write_read(self): - import os, select - readend, writeend = os.pipe() + import select + readend, writeend = getpair() try: total_out = 0 while True: @@ -42,7 +35,7 @@ if owtd == []: break assert owtd == [writeend] - total_out += os.write(writeend, 'x' * 512) + total_out += writeend.send('x' * 512) total_in = 0 while True: iwtd, owtd, ewtd = select.select([readend], [], [], 0) @@ -50,45 +43,45 @@ if iwtd == []: break assert iwtd == [readend] - data = os.read(readend, 4096) + data = readend.recv(4096) assert len(data) > 0 assert data == 'x' * len(data) total_in += len(data) assert total_in == total_out finally: - os.close(writeend) - os.close(readend) + writeend.close() + readend.close() def test_close(self): - import os, select - readend, writeend = os.pipe() + import select + readend, writeend = getpair() try: try: - total_out = os.write(writeend, 'x' * 512) + total_out = writeend.send('x' * 512) finally: - os.close(writeend) + writeend.close() assert 1 <= total_out <= 512 total_in = 0 while True: iwtd, owtd, ewtd = select.select([readend], [], []) assert iwtd == [readend] assert owtd == ewtd == [] - data = os.read(readend, 4096) + data = readend.recv(4096) if len(data) == 0: break assert data == 'x' * len(data) total_in += len(data) assert total_in == total_out finally: - os.close(readend) + readend.close() def test_read_many(self): - import os, select + import select readends = [] writeends = [] try: for i in range(10): - fd1, fd2 = os.pipe() + fd1, fd2 = getpair() readends.append(fd1) writeends.append(fd2) iwtd, owtd, ewtd = select.select(readends, [], [], 0) @@ -96,24 +89,91 @@ for i in range(50): n = (i*3) % 10 - os.write(writeends[n], 'X') + writeends[n].send('X') iwtd, owtd, ewtd = select.select(readends, [], []) assert iwtd == [readends[n]] assert owtd == ewtd == [] - data = os.read(readends[n], 1) + data = readends[n].recv(1) assert data == 'X' finally: for fd in readends + writeends: - os.close(fd) + fd.close() def test_read_end_closed(self): - import os, select - readend, writeend = os.pipe() - os.close(readend) + import select + readend, writeend = getpair() + readend.close() try: iwtd, owtd, ewtd = select.select([], [writeend], []) assert owtd == [writeend] assert iwtd == ewtd == [] finally: - os.close(writeend) + writeend.close() + +class AppTestSelectWithPipes(_AppTestSelect): + "Use a pipe to get pairs of file descriptors" + def setup_class(cls): + if sys.platform == 'win32': + py.test.skip("select() doesn't work with pipes on win32") + space = gettestobjspace(usemodules=('select',)) + cls.space = space + + # Wraps a file descriptor in an socket-like object + space.exec_('''if 1: + import os + class FileAsSocket: + def __init__(self, fd): + self.fd = fd + def fileno(self): + return self.fd + def send(self, data): + return os.write(self.fd, data) + def recv(self, length): + return os.read(self.fd, length) + def close(self): + return os.close(self.fd) + def getpair(): + s1, s2 = os.pipe() + return FileAsSocket(s1), FileAsSocket(s2)''', + space.builtin.w_dict, space.builtin.w_dict) + +class AppTestSelectWithSockets(_AppTestSelect): + """Same tests with connected sockets. + socket.socketpair() does not exists on win32, + so we start our own server.""" + def setup_class(cls): + space = gettestobjspace(usemodules=('select',)) + cls.space = space + + space.setitem(space.builtin.w_dict, space.wrap('getpair'), + space.wrap(cls.getsocketpair)) + + import socket + cls.sock = socket.socket() + + try_ports = [1023] + range(20000, 30000, 437) + for port in try_ports: + print 'binding to port %d:' % (port,), + cls.sockaddress = ('127.0.0.1', port) + try: + cls.sock.bind(cls.sockaddress) + print 'works' + break + except socket.error, e: # should get a "Permission denied" + print e + else: + raise e + + @classmethod + def getsocketpair(cls): + """Helper method which returns a pair of connected sockets. + Note that they become faked objects at AppLevel""" + import thread, socket + + cls.sock.listen(1) + s2 = socket.socket() + thread.start_new_thread(s2.connect, (cls.sockaddress,)) + s1, addr2 = cls.sock.accept() + + return s1, s2 From afa at codespeak.net Thu May 3 23:17:35 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 3 May 2007 23:17:35 +0200 (CEST) Subject: [pypy-svn] r42649 - in pypy/dist/pypy: module/select rlib Message-ID: <20070503211735.2E5128075@code0.codespeak.net> Author: afa Date: Thu May 3 23:17:33 2007 New Revision: 42649 Removed: pypy/dist/pypy/module/select/ctypes_select.py Modified: pypy/dist/pypy/module/select/__init__.py pypy/dist/pypy/module/select/interp_select.py pypy/dist/pypy/rlib/_rsocket_ctypes.py Log: Remove ctypes_select in favor of _rsocket_ctypes. Most of select structs and functions were actually already used by rsocket, and it will be easier to port to win32. Next step: work on Windows to write a poll() replacement. Modified: pypy/dist/pypy/module/select/__init__.py ============================================================================== --- pypy/dist/pypy/module/select/__init__.py (original) +++ pypy/dist/pypy/module/select/__init__.py Thu May 3 23:17:33 2007 @@ -13,9 +13,15 @@ } def buildloaders(cls): - from pypy.module.select import ctypes_select as _c - for constant, value in _c.constants.iteritems(): - Module.interpleveldefs[constant] = "space.wrap(%r)" % value + constantnames = ''' + POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL + POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG'''.split() + + from pypy.rlib._rsocket_ctypes import constants + for name in constantnames: + if name in constants: + value = constants[name] + Module.interpleveldefs[name] = "space.wrap(%r)" % value super(Module, cls).buildloaders() buildloaders = classmethod(buildloaders) Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Thu May 3 23:17:33 2007 @@ -1,7 +1,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import W_Root, ObjSpace, interp2app -from pypy.module.select import ctypes_select as _c +from pypy.rlib import _rsocket_ctypes as _c from pypy.rpython.rctypes.aerrno import geterrno from pypy.interpreter.error import OperationError Modified: pypy/dist/pypy/rlib/_rsocket_ctypes.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_ctypes.py (original) +++ pypy/dist/pypy/rlib/_rsocket_ctypes.py Thu May 3 23:17:33 2007 @@ -22,6 +22,7 @@ 'sys/socket.h', 'sys/un.h', 'sys/poll.h', + 'sys/select.h', 'netinet/in.h', 'netinet/tcp.h', 'unistd.h', @@ -64,8 +65,6 @@ INVALID_SOCKET = ctypes_platform.DefinedConstantInteger('INVALID_SOCKET') INET_ADDRSTRLEN = ctypes_platform.DefinedConstantInteger('INET_ADDRSTRLEN') INET6_ADDRSTRLEN= ctypes_platform.DefinedConstantInteger('INET6_ADDRSTRLEN') - POLLIN = ctypes_platform.DefinedConstantInteger('POLLIN') - POLLOUT = ctypes_platform.DefinedConstantInteger('POLLOUT') EINPROGRESS = ctypes_platform.DefinedConstantInteger('EINPROGRESS') WSAEINPROGRESS = ctypes_platform.DefinedConstantInteger('WSAEINPROGRESS') EWOULDBLOCK = ctypes_platform.DefinedConstantInteger('EWOULDBLOCK') @@ -134,6 +133,9 @@ TCP_LINGER2 TCP_MAXSEG TCP_NODELAY TCP_QUICKACK TCP_SYNCNT TCP_WINDOW_CLAMP IPX_TYPE + +POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL +POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG '''.split() for name in constant_names: @@ -313,8 +315,6 @@ FIONBIO = cConfig.FIONBIO INET_ADDRSTRLEN = cConfig.INET_ADDRSTRLEN INET6_ADDRSTRLEN = cConfig.INET6_ADDRSTRLEN -POLLIN = cConfig.POLLIN -POLLOUT = cConfig.POLLOUT EINPROGRESS = cConfig.EINPROGRESS or cConfig.WSAEINPROGRESS EWOULDBLOCK = cConfig.EWOULDBLOCK or cConfig.WSAEWOULDBLOCK EAFNOSUPPORT = cConfig.EAFNOSUPPORT or cConfig.WSAEAFNOSUPPORT From afa at codespeak.net Thu May 3 23:56:27 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 3 May 2007 23:56:27 +0200 (CEST) Subject: [pypy-svn] r42651 - in pypy/dist/pypy: module/select module/select/test rlib Message-ID: <20070503215627.467FF8075@code0.codespeak.net> Author: afa Date: Thu May 3 23:56:26 2007 New Revision: 42651 Modified: pypy/dist/pypy/module/select/interp_select.py pypy/dist/pypy/module/select/test/test_select.py pypy/dist/pypy/rlib/_rsocket_ctypes.py Log: win32 implementation of poll(). Had to change some tests: - select.select() requires at least one socket, even on CPython - different behaviour when closing a socket with unread data. Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Thu May 3 23:56:26 2007 @@ -2,6 +2,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import W_Root, ObjSpace, interp2app from pypy.rlib import _rsocket_ctypes as _c +from ctypes import POINTER, byref from pypy.rpython.rctypes.aerrno import geterrno from pypy.interpreter.error import OperationError @@ -50,43 +51,136 @@ space.wrap(fd)) unregister.unwrap_spec = ['self', ObjSpace, W_Root] - def poll(self, space, w_timeout=None): - if space.is_w(w_timeout, space.w_None): - timeout = -1 - else: - timeout = space.int_w(w_timeout) - - numfd = len(self.fddict) - buf = _c.create_string_buffer(_c.sizeof(_c.pollfd) * numfd) - pollfds = _c.cast(buf, _c.POINTER(_c.pollfd)) - i = 0 - for fd, events in self.fddict.iteritems(): - pollfds[i].fd = fd - pollfds[i].events = events - i += 1 - - # XXX Temporary hack for releasing the GIL - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - ret = _c.poll(pollfds, numfd, timeout) - if GIL is not None: GIL.acquire(True) - - if ret < 0: - errno = geterrno() - w_module = space.getbuiltinmodule('select') - w_errortype = space.getattr(w_module, space.wrap('error')) - message = _c.strerror(errno) - raise OperationError(w_errortype, - space.newtuple([space.wrap(errno), - space.wrap(message)])) - - retval_w = [] - for i in range(numfd): - pollfd = pollfds[i] - if pollfd.revents: - retval_w.append(space.newtuple([space.wrap(pollfd.fd), - space.wrap(pollfd.revents)])) - return space.newlist(retval_w) + if hasattr(_c, 'poll'): + def poll(self, space, w_timeout=None): + if space.is_w(w_timeout, space.w_None): + timeout = -1 + else: + timeout = space.int_w(w_timeout) + + numfd = len(self.fddict) + buf = _c.create_string_buffer(_c.sizeof(_c.pollfd) * numfd) + pollfds = _c.cast(buf, POINTER(_c.pollfd)) + i = 0 + for fd, events in self.fddict.iteritems(): + pollfds[i].fd = fd + pollfds[i].events = events + i += 1 + + # XXX Temporary hack for releasing the GIL + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + ret = _c.poll(pollfds, numfd, timeout) + if GIL is not None: GIL.acquire(True) + + if ret < 0: + errno = geterrno() + w_module = space.getbuiltinmodule('select') + w_errortype = space.getattr(w_module, space.wrap('error')) + message = _c.strerror(errno) + raise OperationError(w_errortype, + space.newtuple([space.wrap(errno), + space.wrap(message)])) + + retval_w = [] + for i in range(numfd): + pollfd = pollfds[i] + if pollfd.revents: + retval_w.append(space.newtuple([space.wrap(pollfd.fd), + space.wrap(pollfd.revents)])) + return space.newlist(retval_w) + + elif hasattr(_c, 'WSAEventSelect'): + # win32 implementation + def poll(self, space, w_timeout=None): + numfd = len(self.fddict) + + socketevents = _c.ARRAY(_c.WSAEVENT, numfd)() + + numevents = 0 + eventdict = {} + + for fd, events in self.fddict.iteritems(): + # select desired events + wsaEvents = 0 + if events & _c.POLLIN: + wsaEvents |= _c.FD_READ | _c.FD_ACCEPT | _c.FD_CLOSE + if events & _c.POLLOUT: + wsaEvents |= _c.FD_WRITE | _c.FD_CONNECT | _c.FD_CLOSE + + # if no events then ignore socket + if wsaEvents == 0: + continue + + # select socket for desired events + event = _c.WSACreateEvent() + _c.WSAEventSelect(fd, event, wsaEvents) + + eventdict[fd] = event + socketevents[numevents] = event + numevents += 1 + + # if no sockets then return immediately + if numevents == 0: + return space.newlist([]) + + # prepare timeout + if space.is_w(w_timeout, space.w_None): + timeout = -1 + else: + timeout = space.int_w(w_timeout) + if timeout < 0: + timeout = _c.INFINITE + + # XXX Temporary hack for releasing the GIL + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + ret = _c.WSAWaitForMultipleEvents(numevents, socketevents, + False, timeout, False) + if GIL is not None: GIL.acquire(True) + + if ret == _c.WSA_WAIT_TIMEOUT: + return space.newlist([]) + + if ret < 0: # WSA_WAIT_FAILED is unsigned... + from pypy.rlib._rsocket_ctypes import socket_strerror, geterrno + errno = geterrno() + w_module = space.getbuiltinmodule('select') + w_errortype = space.getattr(w_module, space.wrap('error')) + message = socket_strerror(errno) + raise OperationError(w_errortype, + space.newtuple([space.wrap(errno), + space.wrap(message)])) + + retval_w = [] + info = _c.WSANETWORKEVENTS() + for fd, event in eventdict.iteritems(): + if _c.WSAEnumNetworkEvents(fd, event, byref(info)) < 0: + continue + revents = 0 + if info.lNetworkEvents & _c.FD_READ: + revents |= _c.POLLIN + if info.lNetworkEvents & _c.FD_ACCEPT: + revents |= _c.POLLIN + if info.lNetworkEvents & _c.FD_WRITE: + revents |= _c.POLLOUT + if info.lNetworkEvents & _c.FD_CONNECT: + if info.iErrorCode[_c.FD_CONNECT_BIT]: + revents |= _c.POLLERR + else: + revents |= _c.POLLOUT + if info.lNetworkEvents & _c.FD_CLOSE: + if info.iErrorCode[_c.FD_CLOSE_BIT]: + revents |= _c.POLLERR + else: + if self.fddict[fd] & _c.POLLIN: + revents |= _c.POLLIN + if self.fddict[fd] & _c.POLLOUT: + revents |= _c.POLLOUT + if revents: + retval_w.append(space.newtuple([space.wrap(fd), + space.wrap(revents)])) + return space.newlist(retval_w) poll.unwrap_spec = ['self', ObjSpace, W_Root] pollmethods = {} Modified: pypy/dist/pypy/module/select/test/test_select.py ============================================================================== --- pypy/dist/pypy/module/select/test/test_select.py (original) +++ pypy/dist/pypy/module/select/test/test_select.py Thu May 3 23:56:26 2007 @@ -4,8 +4,9 @@ class _AppTestSelect: def test_sleep(self): import time, select + readend, writeend = getpair() start = time.time() - iwtd, owtd, ewtd = select.select([], [], [], 0.3) + iwtd, owtd, ewtd = select.select([readend], [], [], 0.3) end = time.time() assert iwtd == owtd == ewtd == [] assert end - start > 0.25 @@ -53,13 +54,16 @@ readend.close() def test_close(self): - import select + import select, sys readend, writeend = getpair() try: try: total_out = writeend.send('x' * 512) finally: - writeend.close() + # win32 sends the 'closed' event immediately, even when + # more data is available + if sys.platform != 'win32': + writeend.close() assert 1 <= total_out <= 512 total_in = 0 while True: @@ -71,6 +75,9 @@ break assert data == 'x' * len(data) total_in += len(data) + # win32: check that closing the socket exits the loop + if sys.platform == 'win32' and total_in == total_out: + writeend.close() assert total_in == total_out finally: readend.close() Modified: pypy/dist/pypy/rlib/_rsocket_ctypes.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_ctypes.py (original) +++ pypy/dist/pypy/rlib/_rsocket_ctypes.py Thu May 3 23:56:26 2007 @@ -136,6 +136,10 @@ POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG + +FD_READ FD_WRITE FD_ACCEPT FD_CONNECT FD_CLOSE +WSA_WAIT_TIMEOUT WSA_WAIT_FAILED INFINITE +FD_CONNECT_BIT FD_CLOSE_BIT '''.split() for name in constant_names: @@ -164,7 +168,13 @@ ('INADDR_NONE', 0xffffffff), ('SHUT_RD', 0), ('SHUT_WR', 1), - ('SHUT_RDWR', 2)] + ('SHUT_RDWR', 2), + ('POLLIN', 1), + ('POLLPRI', 2), + ('POLLOUT', 4), + ('POLLERR', 8), + ('POLLHUP', 16), + ] for name, default in constants_w_defaults: setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) @@ -249,6 +259,13 @@ [('fd', socketfd_type), ('events', c_short), ('revents', c_short)]) +if _MS_WINDOWS: + CConfig.WSAEVENT = ctypes_platform.SimpleType('WSAEVENT', c_void_p) + CConfig.WSANETWORKEVENTS = ctypes_platform.Struct('WSANETWORKEVENTS', + [('lNetworkEvents', c_long), + ('iErrorCode', c_int * 10), #FD_MAX_EVENTS + ]) + CConfig.timeval = ctypes_platform.Struct('struct timeval', [('tv_sec', c_long), @@ -351,6 +368,9 @@ if _POSIX: nfds_t = cConfig.nfds_t pollfd = cConfig.pollfd +if MS_WINDOWS: + WSAEVENT = cConfig.WSAEVENT + WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS timeval = cConfig.timeval if MS_WINDOWS: fd_set = cConfig.fd_set @@ -363,7 +383,7 @@ # functions if MS_WINDOWS: from ctypes import windll - dllname = util.find_library('wsock32') + dllname = util.find_library('ws2_32') assert dllname is not None socketdll = windll.LoadLibrary(dllname) else: @@ -573,6 +593,24 @@ POINTER(timeval)] select.restype = c_int + WSACreateEvent = socketdll.WSACreateEvent + WSACreateEvent.argtypes = [] + WSACreateEvent.restype = WSAEVENT + + WSAEventSelect = socketdll.WSAEventSelect + WSAEventSelect.argtypes = [socketfd_type, WSAEVENT, c_long] + WSAEventSelect.restype = c_int + + WSAWaitForMultipleEvents = socketdll.WSAWaitForMultipleEvents + WSAWaitForMultipleEvents.argtypes = [c_long, POINTER(WSAEVENT), + c_int, c_long, c_int] + WSAWaitForMultipleEvents.restype = c_long + + WSAEnumNetworkEvents = socketdll.WSAEnumNetworkEvents + WSAEnumNetworkEvents.argtypes = [socketfd_type, WSAEVENT, + POINTER(WSANETWORKEVENTS)] + WSAEnumNetworkEvents.restype = c_int + if MS_WINDOWS: WSAData = cConfig.WSAData WSAStartup = socketdll.WSAStartup From afa at codespeak.net Fri May 4 00:01:22 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 May 2007 00:01:22 +0200 (CEST) Subject: [pypy-svn] r42652 - in pypy/dist/pypy: module/select rlib Message-ID: <20070503220122.ED2CB8075@code0.codespeak.net> Author: afa Date: Fri May 4 00:01:21 2007 New Revision: 42652 Modified: pypy/dist/pypy/module/select/interp_select.py pypy/dist/pypy/rlib/_rsocket_ctypes.py Log: Close Events handle after use Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Fri May 4 00:01:21 2007 @@ -180,6 +180,9 @@ if revents: retval_w.append(space.newtuple([space.wrap(fd), space.wrap(revents)])) + + _c.WSACloseEvent(event) + return space.newlist(retval_w) poll.unwrap_spec = ['self', ObjSpace, W_Root] Modified: pypy/dist/pypy/rlib/_rsocket_ctypes.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_ctypes.py (original) +++ pypy/dist/pypy/rlib/_rsocket_ctypes.py Fri May 4 00:01:21 2007 @@ -597,6 +597,10 @@ WSACreateEvent.argtypes = [] WSACreateEvent.restype = WSAEVENT + WSACloseEvent = socketdll.WSACloseEvent + WSACloseEvent.argtypes = [WSAEVENT] + WSACloseEvent.restype = c_int + WSAEventSelect = socketdll.WSAEventSelect WSAEventSelect.argtypes = [socketfd_type, WSAEVENT, c_long] WSAEventSelect.restype = c_int From santagada at codespeak.net Fri May 4 00:37:11 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Fri, 4 May 2007 00:37:11 +0200 (CEST) Subject: [pypy-svn] r42653 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070503223711.76D8A8075@code0.codespeak.net> Author: santagada Date: Fri May 4 00:37:10 2007 New Revision: 42653 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: lots of new statement tests and a function declaration tests Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Fri May 4 00:37:10 2007 @@ -5,31 +5,31 @@ program : sourceelements EOF ; -sourceelements : sourceelement sourceelements +sourceelements : sourceelement >sourceelements< | sourceelement ; -sourceelement : functiondeclaration - | statement +sourceelement : + | ; -statement : +statement : [";"]? | [";"] | | [";"] - | [";"]? - | [";"] + | + | | [";"] | [";"] | [";"] | [";"] - | [";"] + | | | [";"] | [";"] ; -block : "{" statementlist? "}" +block : ["{"] statementlist? ["}"] ; statementlist : statement >statementlist< @@ -49,7 +49,7 @@ initialiser : ["="] assignmentexpression ; -emptystatement : ";" +emptystatement : [";"] ; expressionstatement : expression @@ -59,7 +59,8 @@ | ["if"] ["("] expression [")"] statement ; -iterationstatement : "do" statement ["while"] ["("] expression [")"] +iterationstatement + : "do" statement ["while"] ["("] expression [")"] | "while" ["("] expression [")"] statement | "for" ["("]expressionnoin? [";"] expression? [";"] expression? [")"] statement | "for" ["("] ["var"] variabledeclarationlistnoin [";"] expression? [";"] expression? [")"] statement @@ -172,8 +173,8 @@ elision : ","+ ; -objectliteral : "{" "}" - | "{" propertynameandvaluelist "}" +objectliteral : ["{"] ["}"] + | ["{"] propertynameandvaluelist ["}"] ; propertynameandvaluelist : hpropertynameandvaluelist* propertyname ":" assignmentexpression @@ -187,17 +188,17 @@ | ; -functiondeclaration : "function" identifier "(" formalparameterlist? ")" "{" functionbody "}" +functiondeclaration : ["function"] identifier ["("] formalparameterlist? [")"] ["{"] functionbody ["}"] ; -functionexpression : "function" identifier? "(" formalparameterlist? ")" "{" functionbody "}" +functionexpression : ["function"] identifier? ["("] formalparameterlist? [")"] ["{"] functionbody ["}"] ; formalparameterlist : identifier [","] >formalparameterlist< | identifier ; -functionbody : sourceelements +functionbody : ; begmemberexpression : @@ -220,11 +221,11 @@ | memberexpression arguments ; -arguments : "(" ")" - | "(" argumentlist ")" +arguments : ["("] [")"] + | "(" ")" ; -argumentlist : assignmentexpression "," >argumentlist< +argumentlist : assignmentexpression [","] >argumentlist< | assignmentexpression; lefthandsideexpression : Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Fri May 4 00:37:10 2007 @@ -15,7 +15,7 @@ newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules return newrules -parse = make_parse_function(regexs, setstartrule(rules, start="statement"), eof=True) +parse = make_parse_function(regexs, rules, eof=True) #parse = make_parse_function(regexs, rules) print rules[2].nonterminal Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Fri May 4 00:37:10 2007 @@ -191,17 +191,48 @@ assert r['variablestatement'] == 1 def test_empty(self): + self.parse(";") for i in range(1,10): r = self.parse_count('{%s}'%(';'*i)) assert r['emptystatement'] == i def test_if(self): - r = self.parse_count("if(x)return;;") + r = self.parse_count("if(x)return;") assert r['ifstatement'] == 1 assert r.get('emptystatement', 0) == 0 r = self.parse_count("if(x)if(i)return;") assert r['ifstatement'] == 2 r = self.parse_count("if(x)return;else return;") assert r['ifstatement'] == 1 + + def test_iteration(self): + self.parse('for(;;);') + self.parse('for(x;;);') + self.parse('for(;x>0;);') + self.parse('for(;;x++);') + self.parse('for(var x = 1;;);') + self.parse('for(x in z);') + self.parse('for(var x in z);') + self.parse('while(1);') + self.parse('do ; while(1)') + + def test_continue_return_break(self): + self.parse('return;') + self.parse('return x+y;') + self.parse('break;') + self.parse('continue;') + self.parse('break label;') + self.parse('continue label;') + + def test_labeled(self): + self.parse('label: x+y;') + +class TestFunctionDeclaration(BaseGrammarTest): + def setup_class(cls): + cls.parse = parse_func('functiondeclaration') + + def test_simpledecl(self): + self.parse('function x () {;}') + self.parse('function z (a,b,c,d,e) {;}') From santagada at codespeak.net Fri May 4 01:31:25 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Fri, 4 May 2007 01:31:25 +0200 (CEST) Subject: [pypy-svn] r42654 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070503233125.991D88076@code0.codespeak.net> Author: santagada Date: Fri May 4 01:31:24 2007 New Revision: 42654 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: testing almost everything of the parser.... should start the trasition to this parser tomorrow Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Fri May 4 01:31:24 2007 @@ -22,11 +22,11 @@ | [";"] | [";"] | [";"] - | [";"] + | | - | + | [";"]? | [";"] - | [";"] + | ; block : ["{"] statementlist? ["}"] @@ -96,7 +96,7 @@ # CaseClauses : # CaseClause # CaseClauses CaseClause -caseclauses : caseclause+ +caseclauses : >caseclause<+ ; caseclause : ["case"] expression [":"] statementlist? @@ -174,14 +174,14 @@ ; objectliteral : ["{"] ["}"] - | ["{"] propertynameandvaluelist ["}"] + | ["{"] >propertynameandvaluelist< ["}"] ; -propertynameandvaluelist : hpropertynameandvaluelist* propertyname ":" assignmentexpression +propertynameandvaluelist : propertynameandvalue ([","] propertynameandvalue)* ; -hpropertynameandvaluelist : propertyname ":" assignmentexpression "," - ; +propertynameandvalue : propertyname [":"] assignmentexpression + ; propertyname : | Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Fri May 4 01:31:24 2007 @@ -15,8 +15,10 @@ newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules return newrules -parse = make_parse_function(regexs, rules, eof=True) -#parse = make_parse_function(regexs, rules) +if len(sys.argv) == 1: + parse = make_parse_function(regexs, rules, eof=True) +else: + parse = make_parse_function(regexs, setstartrule(rules,sys.argv[1]), eof=True) print rules[2].nonterminal source = raw_input() Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Fri May 4 01:31:24 2007 @@ -168,7 +168,23 @@ "2 << 4 << 4", "30 | 3 & 5", ]) - + + def test_primary(self): + self.parse('this') + self.parse('(x)') + self.parse('((((x))))') + self.parse('(x * (x * x)) + x - x') + + def test_array_literal(self): + self.parse('[1,2,3,4]') + self.parse('[1,2,]') + self.parse('[1]') + + def test_object_literal(self): + self.parse('{}') + self.parse('{x:1}') #per spec {x:1,} should not be supported + self.parse('{x:1,y:2}') + class TestStatements(BaseGrammarTest): def setup_class(cls): cls.parse = parse_func('statement') @@ -216,16 +232,41 @@ self.parse('while(1);') self.parse('do ; while(1)') - def test_continue_return_break(self): + def test_continue_return_break_throw(self): self.parse('return;') self.parse('return x+y;') self.parse('break;') self.parse('continue;') self.parse('break label;') self.parse('continue label;') + self.parse('throw (5+5);') + def test_with(self): + self.parse('with(x);') + def test_labeled(self): self.parse('label: x+y;') + + def test_switch(self): + self.parse("""switch(x) { + case 1: break; + case 2: break; + default: ; + } + + """) + self.parse("""switch(x) { + case 1: break; + case 2: break; + default: ; + case 3: break; + } + + """) + def test_try(self): + self.parse('try {x;} catch (e) {x;}') + self.parse('try {x;} catch (e) {x;} finally {x;}') + self.parse('try {x;} finally {x;}') class TestFunctionDeclaration(BaseGrammarTest): def setup_class(cls): From cfbolz at codespeak.net Fri May 4 08:52:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 4 May 2007 08:52:13 +0200 (CEST) Subject: [pypy-svn] r42657 - pypy/extradoc/talk/badhonnef2007 Message-ID: <20070504065213.3A5828071@code0.codespeak.net> Author: cfbolz Date: Fri May 4 08:52:10 2007 New Revision: 42657 Modified: pypy/extradoc/talk/badhonnef2007/talk.pdf pypy/extradoc/talk/badhonnef2007/talk.tex Log: some changes Modified: pypy/extradoc/talk/badhonnef2007/talk.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/badhonnef2007/talk.tex ============================================================================== --- pypy/extradoc/talk/badhonnef2007/talk.tex (original) +++ pypy/extradoc/talk/badhonnef2007/talk.tex Fri May 4 08:52:10 2007 @@ -11,7 +11,7 @@ \usetheme{Warsaw} % or ... - \setbeamercovered{transparent} + %\setbeamercovered{transparent} % or whatever (possibly just delete it) } @@ -78,11 +78,11 @@ \titlepage \end{frame} -\begin{frame} - \frametitle{Outline} - \tableofcontents +%\begin{frame} +% \frametitle{Outline} +% \tableofcontents % You might wish to add the option [pausesections] -\end{frame} +%\end{frame} % Structuring a talk is a difficult task and the following structure @@ -109,14 +109,18 @@ \begin{itemize} \item - Pyrolog is a Prolog interpreter written in RPython + Pyrolog is a Prolog interpreter written in Python + \item + want to compile it to get interesting performance + \item + translation tool-chain part of the PyPy project + \item + Python itself is too dynamic to be translatable to other languages, need a subset \item RPython (``Restricted Python'') is a subset of Python translatable to other languages \item RPython is designed to be significantly faster than regular Python - \item - translation tool-chain part of the PyPy project \end{itemize} \end{frame} @@ -128,9 +132,7 @@ \item started as a Python VM implementation in Python \item - Python itself is too dynamic to be translatable to other languages, need a subset - \item - includes a translation tool-chain for RPython + most important part: translation tool-chain for RPython \item is becoming a general environment for writing interpreters (JavaScript, Prolog started) \item @@ -159,7 +161,7 @@ \begin{frame} - \frametitle{The Python case (i)} + \frametitle{The Python case} CPython (the reference implementation) is a straightforward, portable VM. \begin{itemize} @@ -167,15 +169,15 @@ Pervasive decisions: reference counting, global lock \dots \item No dynamic compilation -% \pause \end{itemize} + \pause \begin{block}{ Extensions:} \begin{itemize} \item - \alert{Stackless} (unlimited recursion, coroutines, serializable continuations) + \alert{Stackless} (unlimited recursion, coroutines, green threads) \item - \alert{Psyco} (run-time specializer) + \alert{Psyco} (run-time specializing compiler) \item \alert{Jython}, \alert{IronPython} \end{itemize} @@ -183,28 +185,6 @@ \end{frame} -\begin{frame} - \frametitle{The Python case (ii)} - \begin{block}{Problems of Extensions:} - \begin{itemize} - \item hard to maintain: need to keep track of CPython - \item Psyco very hard to port to other hardware architectures - \item tedious to write them, Python semantics need to be re-implemented - \end{itemize} - \end{block} - \begin{itemize} - \item - The community wants Python to run everywhere: - Jython (Java), IronPython (.NET). - Lots of effort and duplication. - - \item - At various points various incompatibilities between - the implementations - - \end{itemize} -\end{frame} - \begin{frame} \frametitle{The Prolog case (i)} @@ -232,16 +212,15 @@ changing the language to experiment is hard \item often extensions to core Prolog, incompatible between each other - \item - fixed implementation decisions (GC, how to generate code, etc.) \end{itemize} \end{block} + \pause \begin{block}{implementations on CLR and JVM} \begin{itemize} \item interfacing with libraries of the platform mostly easy \item - no extensions to core Prolog (like tabling, coroutines) + no extensions to core Prolog (like tabling, coroutines, constraints) \item slow, compared to good C implementations \end{itemize} @@ -279,16 +258,19 @@ lower-level targets \begin{itemize} \item C-like - \item Java \item .NET + \item Java \end{itemize} + \pause \item Insert low-level aspects into the code as required by the target \begin{itemize} \item object layout \item memory management - \end