[pypy-svn] r39609 - in pypy/dist/pypy: config doc/config interpreter interpreter/astcompiler interpreter/pyparser interpreter/pyparser/test interpreter/stablecompiler module/dyngram module/recparser module/recparser/hooksamples module/recparser/test
afayolle at codespeak.net
afayolle at codespeak.net
Wed Feb 28 18:30:52 CET 2007
Author: afayolle
Date: Wed Feb 28 18:30:48 2007
New Revision: 39609
Added:
pypy/dist/pypy/doc/config/objspace.usemodules.dyngram.txt (contents, props changed)
pypy/dist/pypy/interpreter/pyparser/asthelper.py (contents, props changed)
pypy/dist/pypy/interpreter/pyparser/test/expressions.py (contents, props changed)
pypy/dist/pypy/interpreter/pyparser/test/test_parser.py (contents, props changed)
pypy/dist/pypy/module/dyngram/ (props changed)
pypy/dist/pypy/module/dyngram/__init__.py (contents, props changed)
pypy/dist/pypy/module/recparser/hooksamples/ (props changed)
pypy/dist/pypy/module/recparser/hooksamples/constchanger.py (contents, props changed)
pypy/dist/pypy/module/recparser/hooksamples/tracer.py (contents, props changed)
pypy/dist/pypy/module/recparser/test/test_dyn_grammarrules.py (contents, props changed)
Modified:
pypy/dist/pypy/config/pypyoption.py
pypy/dist/pypy/interpreter/astcompiler/ast.py
pypy/dist/pypy/interpreter/astcompiler/ast.txt
pypy/dist/pypy/interpreter/astcompiler/astgen.py
pypy/dist/pypy/interpreter/pycompiler.py
pypy/dist/pypy/interpreter/pyparser/astbuilder.py
pypy/dist/pypy/interpreter/pyparser/ebnfgrammar.py
pypy/dist/pypy/interpreter/pyparser/ebnflexer.py
pypy/dist/pypy/interpreter/pyparser/ebnfparse.py
pypy/dist/pypy/interpreter/pyparser/grammar.py
pypy/dist/pypy/interpreter/pyparser/pysymbol.py
pypy/dist/pypy/interpreter/pyparser/pythonlexer.py
pypy/dist/pypy/interpreter/pyparser/pythonparse.py
pypy/dist/pypy/interpreter/pyparser/pythonutil.py
pypy/dist/pypy/interpreter/pyparser/pytoken.py
pypy/dist/pypy/interpreter/pyparser/syntaxtree.py
pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py
pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py
pypy/dist/pypy/interpreter/pyparser/test/test_lookahead.py
pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py
pypy/dist/pypy/interpreter/pyparser/test/test_samples.py
pypy/dist/pypy/interpreter/pyparser/tuplebuilder.py
pypy/dist/pypy/interpreter/stablecompiler/transformer.py
pypy/dist/pypy/module/recparser/__init__.py
pypy/dist/pypy/module/recparser/codegen.py
pypy/dist/pypy/module/recparser/compat.py
pypy/dist/pypy/module/recparser/pyparser.py
pypy/dist/pypy/module/recparser/test/test_parser.py
Log:
merge ast-experiments branch in the trunk.
The ast branch being in a weird state, I'm merging this by hand (as suggested
on IRC by mwh and cfbolz). Here follows the commit messages for the
ast-experiments branch.
------------------------------------------------------------------------
r39399 | afayolle | 2007-02-26 08:45:45 +0100 (lun, 26 fév 2007) | 5 lines
merge trunk with branch
svn merge -r 38798:39398 svn+ssh://codespeak.net/svn/pypy/dist
------------------------------------------------------------------------
r39210 | mwh | 2007-02-19 15:49:22 +0100 (lun, 19 fév 2007) | 2 lines
fix shallow failures in test_dyn_grammarrules.py
------------------------------------------------------------------------
r38879 | adim | 2007-02-15 14:26:12 +0100 (jeu, 15 fév 2007) | 1 line
small doc for the --withmod-dyngram option
------------------------------------------------------------------------
r38877 | adim | 2007-02-15 14:13:49 +0100 (jeu, 15 fév 2007) | 4 lines
- made a special "dyngram" module that exports the runtime grammar
modifications ability
------------------------------------------------------------------------
r38854 | adim | 2007-02-14 17:33:38 +0100 (mer, 14 fév 2007) | 7 lines
merged revision 38798 into the branch
fixed pyparser's unit tests
(svn merge -r 35032:38798 http://codespeak.net/svn/pypy/dist)
------------------------------------------------------------------------
r38796 | adim | 2007-02-14 11:53:28 +0100 (mer, 14 fév 2007) | 1 line
small rpython fixes
------------------------------------------------------------------------
r36913 | syt | 2007-01-18 10:07:03 +0100 (jeu, 18 jan 2007) | 1 line
make abstract builder wrappable
------------------------------------------------------------------------
r36912 | syt | 2007-01-18 10:06:29 +0100 (jeu, 18 jan 2007) | 1 line
fix target
------------------------------------------------------------------------
r36911 | syt | 2007-01-18 10:05:45 +0100 (jeu, 18 jan 2007) | 5 lines
Leysin sprint work:
more tests and bug fixes on the parser package
------------------------------------------------------------------------
r36910 | syt | 2007-01-18 10:01:45 +0100 (jeu, 18 jan 2007) | 5 lines
Leysin sprint work (mwh around):
refactoring and fixes to make it tranlatable (not yet there though)
------------------------------------------------------------------------
r36451 | adim | 2007-01-11 10:36:52 +0100 (jeu, 11 jan 2007) | 1 line
try to avoid using global PYTHON_PARSER
------------------------------------------------------------------------
r36450 | adim | 2007-01-11 10:34:19 +0100 (jeu, 11 jan 2007) | 2 lines
use parser's tokens dict and get rid of setattr() usage in pytoken
------------------------------------------------------------------------
r35939 | adim | 2006-12-21 17:40:55 +0100 (jeu, 21 déc 2006) | 1 line
renamed get_pyparser_for_version to make_pyparser
------------------------------------------------------------------------
r35934 | adim | 2006-12-21 16:19:02 +0100 (jeu, 21 déc 2006) | 2 lines
added a hook to modify the grammar rules at applevel
------------------------------------------------------------------------
r35933 | adim | 2006-12-21 16:13:54 +0100 (jeu, 21 déc 2006) | 5 lines
- added a reference to the parser in AstBuilder, and tried to avoid using
PYTHON_PARSER as much as possible.
- split astbuilder.py in 2 modules
------------------------------------------------------------------------
r35754 | adim | 2006-12-14 17:28:05 +0100 (jeu, 14 déc 2006) | 6 lines
small pyparser refactorings
- removed some unused code
- eased a bit python parser creation (build_parser / get_pyparser)
------------------------------------------------------------------------
r35729 | adim | 2006-12-14 11:17:18 +0100 (jeu, 14 déc 2006) | 1 line
removing old / unused code
------------------------------------------------------------------------
r35066 | adim | 2006-11-28 10:56:19 +0100 (mar, 28 nov 2006) | 4 lines
for some reason, svn failed to completly commit the merge yesterday :-(
This is the second part of the merge
------------------------------------------------------------------------
r35047 | adim | 2006-11-27 18:15:58 +0100 (lun, 27 nov 2006) | 6 lines
merging revision 35032 into the branch
Note: still need to backport keyword management from trunk
(svn merge -r 22393:35032 http://codespeak.net/svn/pypy/dist)
------------------------------------------------------------------------
r22813 | ludal | 2006-01-29 02:21:18 +0100 (dim, 29 jan 2006) | 4 lines
refactoring III
most tests pass. a problem remains with TupleBuilder messing on import xxx
------------------------------------------------------------------------
r22790 | ludal | 2006-01-28 15:13:53 +0100 (sam, 28 jan 2006) | 4 lines
refactor, part II
forgot the parser class
make pythonlexer.py use it
------------------------------------------------------------------------
r22761 | ludal | 2006-01-28 02:10:44 +0100 (sam, 28 jan 2006) | 4 lines
big refactoring, of the parser -- part I
isolates management of symbols and grammar rules into a Parser class
------------------------------------------------------------------------
r22604 | adim | 2006-01-24 17:28:50 +0100 (mar, 24 jan 2006) | 1 line
oops, forgot to checkin tracer update
------------------------------------------------------------------------
r22583 | ludal | 2006-01-24 13:18:01 +0100 (mar, 24 jan 2006) | 2 lines
bugfixes
------------------------------------------------------------------------
r22570 | adim | 2006-01-24 10:55:56 +0100 (mar, 24 jan 2006) | 1 line
updated assign tracer example with insert_before / insert_after
------------------------------------------------------------------------
r22556 | adim | 2006-01-24 09:52:24 +0100 (mar, 24 jan 2006) | 3 lines
added insert_after and insert_before methods on appropriate Node classes
(exposed those methods at applevel)
------------------------------------------------------------------------
r22540 | ludal | 2006-01-23 18:14:22 +0100 (lun, 23 jan 2006) | 4 lines
(ludal,adim)
debugging and first attempt to replace old with new grammar parser
------------------------------------------------------------------------
r22533 | ludal | 2006-01-23 16:42:50 +0100 (lun, 23 jan 2006) | 5 lines
(adim,ludal)
a new (not yet annotatable) EBNFParser that will build grammar parsers at runtime
------------------------------------------------------------------------
r22399 | adim | 2006-01-18 17:27:36 +0100 (mer, 18 jan 2006) | 4 lines
- added a parent attribute to AST nodes
- added a few hook examples (they will probably be used as a basis for improvements)
------------------------------------------------------------------------
r22393 | adim | 2006-01-18 16:36:14 +0100 (mer, 18 jan 2006) | 3 lines
create a branch to explore ast manipulation without having to make sure PyPy compiles
before each checkin
------------------------------------------------------------------------
Modified: pypy/dist/pypy/config/pypyoption.py
==============================================================================
--- pypy/dist/pypy/config/pypyoption.py (original)
+++ pypy/dist/pypy/config/pypyoption.py Wed Feb 28 18:30:48 2007
@@ -24,7 +24,7 @@
working_modules = default_modules.copy()
working_modules.update(dict.fromkeys(
["rsocket", "unicodedata", "mmap", "fcntl", "rctime", "select",
- "crypt", "signal",
+ "crypt", "signal", "dyngram",
]
))
Added: pypy/dist/pypy/doc/config/objspace.usemodules.dyngram.txt
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/doc/config/objspace.usemodules.dyngram.txt Wed Feb 28 18:30:48 2007
@@ -0,0 +1,7 @@
+Use the 'dyngram' module.
+
+The 'dyngram' module exports the 'insert_grammar_rule' function to
+application-level code. This function allows to modify dynamically
+the python the grammar.
+
+
Modified: pypy/dist/pypy/interpreter/astcompiler/ast.py
==============================================================================
--- pypy/dist/pypy/interpreter/astcompiler/ast.py (original)
+++ pypy/dist/pypy/interpreter/astcompiler/ast.py Wed Feb 28 18:30:48 2007
@@ -30,6 +30,7 @@
def __init__(self, lineno = -1):
self.lineno = lineno
self.filename = ""
+ self.parent = None
#self.scope = None
def getChildren(self):
@@ -61,6 +62,12 @@
def descr_repr( self, space ):
return space.wrap( self.__repr__() )
+ def fget_parent(space, self):
+ return space.wrap(self.parent)
+
+ def fset_parent(space, self, w_parent):
+ self.parent = space.interp_w(Node, w_parent, can_be_None=False)
+
def descr_getChildNodes( self, space ):
lst = self.getChildNodes()
return space.newlist( [ space.wrap( it ) for it in lst ] )
@@ -84,6 +91,7 @@
mutate = interp2app(descr_node_mutate, unwrap_spec=[ ObjSpace, W_Root, W_Root ] ),
lineno = interp_attrproperty('lineno', cls=Node),
filename = interp_attrproperty('filename', cls=Node),
+ parent=GetSetProperty(Node.fget_parent, Node.fset_parent),
)
Node.typedef.acceptable_as_base_class = False
@@ -363,6 +371,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_And_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(And, w_subtype)
@@ -391,6 +408,8 @@
accept=interp2app(descr_And_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_And_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(And.fget_nodes, And.fset_nodes ),
+ insert_after=interp2app(And.descr_insert_after.im_func, unwrap_spec=[ObjSpace, And, Node, W_Root]),
+ insert_before=interp2app(And.descr_insert_before.im_func, unwrap_spec=[ObjSpace, And, Node, W_Root]),
)
And.typedef.acceptable_as_base_class = False
@@ -536,6 +555,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_AssList_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(AssList, w_subtype)
@@ -564,6 +592,8 @@
accept=interp2app(descr_AssList_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_AssList_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(AssList.fget_nodes, AssList.fset_nodes ),
+ insert_after=interp2app(AssList.descr_insert_after.im_func, unwrap_spec=[ObjSpace, AssList, Node, W_Root]),
+ insert_before=interp2app(AssList.descr_insert_before.im_func, unwrap_spec=[ObjSpace, AssList, Node, W_Root]),
)
AssList.typedef.acceptable_as_base_class = False
@@ -671,6 +701,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_AssTuple_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(AssTuple, w_subtype)
@@ -699,6 +738,8 @@
accept=interp2app(descr_AssTuple_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_AssTuple_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(AssTuple.fget_nodes, AssTuple.fset_nodes ),
+ insert_after=interp2app(AssTuple.descr_insert_after.im_func, unwrap_spec=[ObjSpace, AssTuple, Node, W_Root]),
+ insert_before=interp2app(AssTuple.descr_insert_before.im_func, unwrap_spec=[ObjSpace, AssTuple, Node, W_Root]),
)
AssTuple.typedef.acceptable_as_base_class = False
@@ -820,6 +861,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def fget_expr( space, self):
return space.wrap(self.expr)
def fset_expr( space, self, w_arg):
@@ -858,6 +908,8 @@
accept=interp2app(descr_Assign_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Assign_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Assign.fget_nodes, Assign.fset_nodes ),
+ insert_after=interp2app(Assign.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Assign, Node, W_Root]),
+ insert_before=interp2app(Assign.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Assign, Node, W_Root]),
expr=GetSetProperty(Assign.fget_expr, Assign.fset_expr ),
)
Assign.typedef.acceptable_as_base_class = False
@@ -1100,6 +1152,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_Bitand_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(Bitand, w_subtype)
@@ -1128,6 +1189,8 @@
accept=interp2app(descr_Bitand_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Bitand_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Bitand.fget_nodes, Bitand.fset_nodes ),
+ insert_after=interp2app(Bitand.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Bitand, Node, W_Root]),
+ insert_before=interp2app(Bitand.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Bitand, Node, W_Root]),
)
Bitand.typedef.acceptable_as_base_class = False
@@ -1166,6 +1229,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_Bitor_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(Bitor, w_subtype)
@@ -1194,6 +1266,8 @@
accept=interp2app(descr_Bitor_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Bitor_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Bitor.fget_nodes, Bitor.fset_nodes ),
+ insert_after=interp2app(Bitor.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Bitor, Node, W_Root]),
+ insert_before=interp2app(Bitor.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Bitor, Node, W_Root]),
)
Bitor.typedef.acceptable_as_base_class = False
@@ -1232,6 +1306,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_Bitxor_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(Bitxor, w_subtype)
@@ -1260,6 +1343,8 @@
accept=interp2app(descr_Bitxor_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Bitxor_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Bitxor.fget_nodes, Bitxor.fset_nodes ),
+ insert_after=interp2app(Bitxor.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Bitxor, Node, W_Root]),
+ insert_before=interp2app(Bitxor.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Bitxor, Node, W_Root]),
)
Bitxor.typedef.acceptable_as_base_class = False
@@ -1834,6 +1919,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_Decorators_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(Decorators, w_subtype)
@@ -1862,6 +1956,8 @@
accept=interp2app(descr_Decorators_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Decorators_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Decorators.fget_nodes, Decorators.fset_nodes ),
+ insert_after=interp2app(Decorators.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Decorators, Node, W_Root]),
+ insert_before=interp2app(Decorators.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Decorators, Node, W_Root]),
)
Decorators.typedef.acceptable_as_base_class = False
@@ -3544,6 +3640,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_List_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(List, w_subtype)
@@ -3572,6 +3677,8 @@
accept=interp2app(descr_List_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_List_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(List.fget_nodes, List.fset_nodes ),
+ insert_after=interp2app(List.descr_insert_after.im_func, unwrap_spec=[ObjSpace, List, Node, W_Root]),
+ insert_before=interp2app(List.descr_insert_before.im_func, unwrap_spec=[ObjSpace, List, Node, W_Root]),
)
List.typedef.acceptable_as_base_class = False
@@ -4172,6 +4279,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_Or_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(Or, w_subtype)
@@ -4200,6 +4316,8 @@
accept=interp2app(descr_Or_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Or_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Or.fget_nodes, Or.fset_nodes ),
+ insert_after=interp2app(Or.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Or, Node, W_Root]),
+ insert_before=interp2app(Or.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Or, Node, W_Root]),
)
Or.typedef.acceptable_as_base_class = False
@@ -4350,6 +4468,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def fget_dest( space, self):
if self.dest is None:
return space.w_None
@@ -4392,6 +4519,8 @@
accept=interp2app(descr_Print_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Print_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Print.fget_nodes, Print.fset_nodes ),
+ insert_after=interp2app(Print.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Print, Node, W_Root]),
+ insert_before=interp2app(Print.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Print, Node, W_Root]),
dest=GetSetProperty(Print.fget_dest, Print.fset_dest ),
)
Print.typedef.acceptable_as_base_class = False
@@ -4439,6 +4568,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def fget_dest( space, self):
if self.dest is None:
return space.w_None
@@ -4481,6 +4619,8 @@
accept=interp2app(descr_Printnl_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Printnl_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Printnl.fget_nodes, Printnl.fset_nodes ),
+ insert_after=interp2app(Printnl.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Printnl, Node, W_Root]),
+ insert_before=interp2app(Printnl.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Printnl, Node, W_Root]),
dest=GetSetProperty(Printnl.fget_dest, Printnl.fset_dest ),
)
Printnl.typedef.acceptable_as_base_class = False
@@ -4856,6 +4996,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_Sliceobj_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(Sliceobj, w_subtype)
@@ -4884,6 +5033,8 @@
accept=interp2app(descr_Sliceobj_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Sliceobj_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Sliceobj.fget_nodes, Sliceobj.fset_nodes ),
+ insert_after=interp2app(Sliceobj.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Sliceobj, Node, W_Root]),
+ insert_before=interp2app(Sliceobj.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Sliceobj, Node, W_Root]),
)
Sliceobj.typedef.acceptable_as_base_class = False
@@ -4922,6 +5073,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_Stmt_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(Stmt, w_subtype)
@@ -4950,6 +5110,8 @@
accept=interp2app(descr_Stmt_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Stmt_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Stmt.fget_nodes, Stmt.fset_nodes ),
+ insert_after=interp2app(Stmt.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Stmt, Node, W_Root]),
+ insert_before=interp2app(Stmt.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Stmt, Node, W_Root]),
)
Stmt.typedef.acceptable_as_base_class = False
@@ -5352,6 +5514,15 @@
del self.nodes[:]
for w_itm in space.unpackiterable(w_arg):
self.nodes.append( space.interp_w(Node, w_itm))
+ def descr_insert_after(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node) + 1
+ self.nodes[index:index] = added_nodes
+
+ def descr_insert_before(space, self, node, w_added_nodes):
+ added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]
+ index = self.nodes.index(node)
+ self.nodes[index:index] = added_nodes
def descr_Tuple_new(space, w_subtype, w_nodes, lineno=-1):
self = space.allocate_instance(Tuple, w_subtype)
@@ -5380,6 +5551,8 @@
accept=interp2app(descr_Tuple_accept, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
mutate=interp2app(descr_Tuple_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] ),
nodes=GetSetProperty(Tuple.fget_nodes, Tuple.fset_nodes ),
+ insert_after=interp2app(Tuple.descr_insert_after.im_func, unwrap_spec=[ObjSpace, Tuple, Node, W_Root]),
+ insert_before=interp2app(Tuple.descr_insert_before.im_func, unwrap_spec=[ObjSpace, Tuple, Node, W_Root]),
)
Tuple.typedef.acceptable_as_base_class = False
Modified: pypy/dist/pypy/interpreter/astcompiler/ast.txt
==============================================================================
--- pypy/dist/pypy/interpreter/astcompiler/ast.txt (original)
+++ pypy/dist/pypy/interpreter/astcompiler/ast.txt Wed Feb 28 18:30:48 2007
@@ -71,7 +71,7 @@
CallFunc: node, args!, star_args& = None, dstar_args& = None
Keyword: name*str, expr
Subscript: expr, flags*int, sub
-Ellipsis:
+Ellipsis:
Sliceobj: nodes!
Slice: expr, flags*int, lower&, upper&
Assert: test, fail&
@@ -337,6 +337,31 @@
return space.call_method(w_visitor, "visitCompare", w_self)
+def descr_Compare_mutate(space, w_self, w_visitor):
+ w_expr = space.getattr(w_self, space.wrap("expr"))
+ w_mutate_expr = space.getattr(w_expr, space.wrap("mutate"))
+ w_mutate_expr_args = Arguments(space, [ w_visitor ])
+ w_new_expr = space.call_args(w_mutate_expr, w_mutate_expr_args)
+ space.setattr(w_self, space.wrap("expr"), w_new_expr)
+
+ w_list = space.getattr(w_self, space.wrap("ops"))
+ list_w = space.unpackiterable(w_list)
+ newlist_w = []
+ for w_item in list_w:
+ w_opname, w_node = space.unpackiterable(w_item, 2)
+
+ w_node_mutate = space.getattr(w_node, space.wrap("mutate"))
+ w_node_mutate_args = Arguments(space, [ w_visitor ])
+ w_newnode = space.call_args(w_node_mutate, w_node_mutate_args)
+
+ newlist_w.append(space.newtuple([w_opname, w_newnode]))
+ w_newlist = space.newlist(newlist_w)
+ space.setattr(w_self, space.wrap("ops"), w_newlist)
+ w_visitCompare = space.getattr(w_visitor, space.wrap("visitCompare"))
+ w_visitCompare_args = Arguments(space, [ w_self ])
+ return space.call_args(w_visitCompare, w_visitCompare_args)
+
+
def descr_Dict_new(space, w_subtype, w_items, lineno=-1):
self = space.allocate_instance(Dict, w_subtype)
items = []
Modified: pypy/dist/pypy/interpreter/astcompiler/astgen.py
==============================================================================
--- pypy/dist/pypy/interpreter/astcompiler/astgen.py (original)
+++ pypy/dist/pypy/interpreter/astcompiler/astgen.py Wed Feb 28 18:30:48 2007
@@ -319,6 +319,40 @@
print >> buf, " self.%s[:] = newlist"%(argname)
print >> buf, " return visitor.visit%s(self)" % self.name
+ def _gen_insertnodes_func(self, buf):
+ print >> buf, " def descr_insert_after(space, self, node, w_added_nodes):"
+ print >> buf, " added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]"
+ print >> buf, " index = self.nodes.index(node) + 1"
+ print >> buf, " self.nodes[index:index] = added_nodes"
+ print >> buf
+ print >> buf, " def descr_insert_before(space, self, node, w_added_nodes):"
+ print >> buf, " added_nodes = [space.interp_w(Node, w_node) for w_node in space.unpackiterable(w_added_nodes)]"
+ print >> buf, " index = self.nodes.index(node)"
+ print >> buf, " self.nodes[index:index] = added_nodes"
+
+
+ def _gen_mutate(self, buf):
+ print >> buf, " def mutate(self, visitor):"
+ if len(self.argnames) != 0:
+ for argname in self.argnames:
+ if argname in self.mutate_nodes:
+ for line in self.mutate_nodes[argname]:
+ if line.strip():
+ print >> buf, ' ' + line
+ elif self.argprops[argname] == P_NODE:
+ print >> buf, " self.%s = self.%s.mutate(visitor)" % (argname,argname)
+ elif self.argprops[argname] == P_NONE:
+ print >> buf, " if self.%s is not None:" % (argname,)
+ print >> buf, " self.%s = self.%s.mutate(visitor)" % (argname,argname)
+ elif self.argprops[argname] == P_NESTED:
+ print >> buf, " newlist = []"
+ print >> buf, " for n in self.%s:"%(argname)
+ print >> buf, " item = n.mutate(visitor)"
+ print >> buf, " if item is not None:"
+ print >> buf, " newlist.append(item)"
+ print >> buf, " self.%s[:] = newlist"%(argname)
+ print >> buf, " return visitor.visit%s(self)" % self.name
+
def _gen_fget_func(self, buf, attr, prop ):
# FGET
print >> buf, " def fget_%s( space, self):" % attr
@@ -370,6 +404,8 @@
if "fset_%s" % attr not in self.additional_methods:
self._gen_fset_func( buf, attr, prop )
+ if prop[attr] == P_NESTED and attr == 'nodes':
+ self._gen_insertnodes_func(buf)
def _gen_descr_mutate(self, buf):
if self.applevel_mutate:
@@ -426,6 +462,9 @@
print >> buf, " mutate=interp2app(descr_%s_mutate, unwrap_spec=[ObjSpace, W_Root, W_Root] )," % self.name
for attr in self.argnames:
print >> buf, " %s=GetSetProperty(%s.fget_%s, %s.fset_%s )," % (attr,self.name,attr,self.name,attr)
+ if self.argprops[attr] == P_NESTED and attr == "nodes":
+ print >> buf, " insert_after=interp2app(%s.descr_insert_after.im_func, unwrap_spec=[ObjSpace, %s, Node, W_Root])," % (self.name, self.name)
+ print >> buf, " insert_before=interp2app(%s.descr_insert_before.im_func, unwrap_spec=[ObjSpace, %s, Node, W_Root])," % (self.name, self.name)
print >> buf, " )"
print >> buf, "%s.typedef.acceptable_as_base_class = False" % self.name
@@ -660,6 +699,7 @@
def __init__(self, lineno = -1):
self.lineno = lineno
self.filename = ""
+ self.parent = None
#self.scope = None
def getChildren(self):
@@ -691,6 +731,12 @@
def descr_repr( self, space ):
return space.wrap( self.__repr__() )
+ def fget_parent(space, self):
+ return space.wrap(self.parent)
+
+ def fset_parent(space, self, w_parent):
+ self.parent = space.interp_w(Node, w_parent, can_be_None=False)
+
def descr_getChildNodes( self, space ):
lst = self.getChildNodes()
return space.newlist( [ space.wrap( it ) for it in lst ] )
@@ -714,6 +760,7 @@
mutate = interp2app(descr_node_mutate, unwrap_spec=[ ObjSpace, W_Root, W_Root ] ),
lineno = interp_attrproperty('lineno', cls=Node),
filename = interp_attrproperty('filename', cls=Node),
+ parent=GetSetProperty(Node.fget_parent, Node.fset_parent),
)
Node.typedef.acceptable_as_base_class = False
Modified: pypy/dist/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/dist/pypy/interpreter/pycompiler.py (original)
+++ pypy/dist/pypy/interpreter/pycompiler.py Wed Feb 28 18:30:48 2007
@@ -189,6 +189,7 @@
warnings.warn_explicit = old_warn_explicit
+
########
class PythonAstCompiler(PyCodeCompiler):
"""Uses the stdlib's python implementation of compiler
@@ -198,6 +199,13 @@
of incomplete inputs (e.g. we shouldn't re-compile from sracth
the whole source after having only added a new '\n')
"""
+ def __init__(self, space):
+ from pyparser.pythonparse import PYTHON_PARSER
+ PyCodeCompiler.__init__(self, space)
+ self.parser = PYTHON_PARSER
+ self.additional_rules = {}
+
+
def compile(self, source, filename, mode, flags):
from pyparser.error import SyntaxError
from pypy.interpreter import astcompiler
@@ -205,15 +213,18 @@
from pypy.interpreter.astcompiler.pycodegen import InteractiveCodeGenerator
from pypy.interpreter.astcompiler.pycodegen import ExpressionCodeGenerator
from pypy.interpreter.astcompiler.ast import Node
- from pyparser.pythonutil import AstBuilder, PYTHON_PARSER, TARGET_DICT
+ from pyparser.astbuilder import AstBuilder
from pypy.interpreter.pycode import PyCode
+ from pypy.interpreter.function import Function
flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat)
space = self.space
try:
- builder = AstBuilder(space=space)
- target_rule = TARGET_DICT[mode]
- PYTHON_PARSER.parse_source(source, target_rule, builder, flags)
+ builder = AstBuilder(self.parser, space=space)
+ for rulename, buildfunc in self.additional_rules.iteritems():
+ assert isinstance(buildfunc, Function)
+ builder.user_build_rules[rulename] = buildfunc
+ self.parser.parse_source(source, mode, builder, flags)
ast_tree = builder.rule_stack[-1]
encoding = builder.source_encoding
except SyntaxError, e:
@@ -251,4 +262,29 @@
def install_compiler_hook(space, w_callable):
# if not space.get( w_callable ):
# raise OperationError( space.w_TypeError( space.wrap( "must have a callable" ) )
- space.default_compiler.w_compile_hook = w_callable
+ space.default_compiler.w_compile_hook = w_callable
+
+def insert_grammar_rule(space, w_rule, w_buildfuncs):
+ """inserts new grammar rules to the default compiler"""
+ from pypy.interpreter import function
+ rule = space.str_w(w_rule)
+ #buildfuncs_w = w_buildfuncs.content
+ buildfuncs = {}
+ #for w_name, w_func in buildfuncs_w.iteritems():
+ # buildfuncs[space.str_w(w_name)] = space.unwrap(w_func)
+ w_iter = space.iter(w_buildfuncs)
+ while 1:
+ try:
+ w_key = space.next(w_iter)
+ w_func = space.getitem(w_buildfuncs, w_key)
+ buildfuncs[space.str_w(w_key)] = space.interp_w(function.Function, w_func)
+ except OperationError, e:
+ if not e.match(space, space.w_StopIteration):
+ raise
+ break
+ space.default_compiler.additional_rules = buildfuncs
+ space.default_compiler.parser.insert_rule(rule)
+
+# XXX cyclic import
+#from pypy.interpreter.baseobjspace import ObjSpace
+#insert_grammar_rule.unwrap_spec = [ObjSpace, str, dict]
Modified: pypy/dist/pypy/interpreter/pyparser/astbuilder.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/astbuilder.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/astbuilder.py Wed Feb 28 18:30:48 2007
@@ -1,518 +1,18 @@
"""This module provides the astbuilder class which is to be used
-by GrammarElements to directly build the AST during parsing
+by GrammarElements to directly build the AS during parsing
without going through the nested tuples step
"""
from grammar import BaseGrammarBuilder, AbstractContext
+
+from pypy.interpreter.function import Function
from pypy.interpreter.astcompiler import ast, consts
-from pypy.interpreter.pyparser import pythonparse
-import pypy.interpreter.pyparser.pytoken as tok
+# from pypy.interpreter.pyparser import pythonparse
+#import pypy.interpreter.pyparser.pytoken as tok
from pypy.interpreter.pyparser.error import SyntaxError
from pypy.interpreter.pyparser.parsestring import parsestr
-
-sym = pythonparse.PYTHON_PARSER.symbols
-
-DEBUG_MODE = 0
-
-### Parsing utilites #################################################
-def parse_except_clause(tokens):
- """parses 'except' [test [',' test]] ':' suite
- and returns a 4-tuple : (tokens_read, expr1, expr2, except_body)
- """
- lineno = tokens[0].lineno
- clause_length = 1
- # Read until end of except clause (bound by following 'else',
- # or 'except' or end of tokens)
- while clause_length < len(tokens):
- token = tokens[clause_length]
- if isinstance(token, TokenObject) and \
- (token.get_value() == 'except' or token.get_value() == 'else'):
- break
- clause_length += 1
- if clause_length == 3:
- # case 'except: body'
- return (3, None, None, tokens[2])
- elif clause_length == 4:
- # case 'except Exception: body':
- return (4, tokens[1], None, tokens[3])
- else:
- # case 'except Exception, exc: body'
- return (6, tokens[1], to_lvalue(tokens[3], consts.OP_ASSIGN), tokens[5])
-
-
-def parse_dotted_names(tokens):
- """parses NAME('.' NAME)* and returns full dotted name
-
- this function doesn't assume that the <tokens> list ends after the
- last 'NAME' element
- """
- first = tokens[0]
- assert isinstance(first, TokenObject)
- name = first.get_value()
- l = len(tokens)
- index = 1
- for index in range(1, l, 2):
- token = tokens[index]
- assert isinstance(token, TokenObject)
- if token.name != tok.DOT:
- break
- token = tokens[index+1]
- assert isinstance(token, TokenObject)
- name += '.'
- value = token.get_value()
- name += value
- return (index, name)
-
-def parse_argument(tokens):
- """parses function call arguments"""
- l = len(tokens)
- index = 0
- arguments = []
- last_token = None
- building_kw = False
- kw_built = False
- stararg_token = None
- dstararg_token = None
- while index < l:
- cur_token = tokens[index]
- if not isinstance(cur_token, TokenObject):
- index += 1
- if not building_kw:
- arguments.append(cur_token)
- else:
- last_token = arguments.pop()
- assert isinstance(last_token, ast.Name) # used by rtyper
- arguments.append(ast.Keyword(last_token.varname, cur_token, last_token.lineno))
- building_kw = False
- kw_built = True
- continue
- elif cur_token.name == tok.COMMA:
- index += 1
- continue
- elif cur_token.name == tok.EQUAL:
- index += 1
- building_kw = True
- continue
- elif cur_token.name == tok.STAR or cur_token.name == tok.DOUBLESTAR:
- index += 1
- if cur_token.name == tok.STAR:
- stararg_token = tokens[index]
- index += 1
- if index >= l:
- break
- index += 2 # Skip COMMA and DOUBLESTAR
- dstararg_token = tokens[index]
- break
- elif cur_token.get_value() == 'for':
- if len(arguments) != 1:
- raise SyntaxError("invalid syntax", cur_token.lineno,
- cur_token.col)
- expr = arguments[0]
- genexpr_for = parse_genexpr_for(tokens[index:])
- genexpr_for[0].is_outmost = True
- gexp = ast.GenExpr(ast.GenExprInner(expr, genexpr_for, expr.lineno), expr.lineno)
- arguments[0] = gexp
- break
- return arguments, stararg_token, dstararg_token
-
-
-def parse_fpdef(tokens, index):
- """fpdef: fpdef: NAME | '(' fplist ')'
- fplist: fpdef (',' fpdef)* [',']
-
- This intend to be a RPYTHON compliant implementation of _parse_fpdef,
- but it can't work with the default compiler.
- We switched to use astcompiler module now
- """
- nodes = []
- comma = False
- while True:
- token = tokens[index]
- index += 1
- assert isinstance(token, TokenObject)
- if token.name == tok.LPAR: # nested item
- index, node = parse_fpdef(tokens, index)
- elif token.name == tok.RPAR: # end of current nesting
- break
- else: # name
- val = token.get_value()
- node = ast.AssName(val, consts.OP_ASSIGN, token.lineno)
- nodes.append(node)
-
- token = tokens[index]
- index += 1
- assert isinstance(token, TokenObject)
- if token.name == tok.COMMA:
- comma = True
- else:
- assert token.name == tok.RPAR
- break
- if len(nodes) == 1 and not comma:
- node = nodes[0]
- else:
- node = ast.AssTuple(nodes, token.lineno)
- return index, node
-
-def parse_arglist(tokens):
- """returns names, defaults, flags"""
- l = len(tokens)
- index = 0
- defaults = []
- names = []
- flags = 0
- first_with_default = -1
- while index < l:
- cur_token = tokens[index]
- index += 1
- if not isinstance(cur_token, TokenObject):
- # XXX: think of another way to write this test
- defaults.append(cur_token)
- if first_with_default == -1:
- first_with_default = len(names) - 1
- elif cur_token.name == tok.COMMA:
- # We could skip test COMMA by incrementing index cleverly
- # but we might do some experiment on the grammar at some point
- continue
- elif cur_token.name == tok.LPAR:
- index, node = parse_fpdef(tokens, index)
- names.append(node)
- elif cur_token.name == tok.STAR or cur_token.name == tok.DOUBLESTAR:
- if cur_token.name == tok.STAR:
- cur_token = tokens[index]
- assert isinstance(cur_token, TokenObject)
- index += 1
- if cur_token.name == tok.NAME:
- val = cur_token.get_value()
- names.append( ast.AssName( val, consts.OP_ASSIGN ) )
- flags |= consts.CO_VARARGS
- index += 1
- if index >= l:
- break
- else:
- # still more tokens to read
- cur_token = tokens[index]
- index += 1
- else:
- raise SyntaxError("incomplete varags", cur_token.lineno,
- cur_token.col)
- assert isinstance(cur_token, TokenObject)
- if cur_token.name != tok.DOUBLESTAR:
- raise SyntaxError("Unexpected token", cur_token.lineno,
- cur_token.col)
- cur_token = tokens[index]
- index += 1
- assert isinstance(cur_token, TokenObject)
- if cur_token.name == tok.NAME:
- val = cur_token.get_value()
- names.append( ast.AssName( val, consts.OP_ASSIGN ) )
- flags |= consts.CO_VARKEYWORDS
- index += 1
- else:
- raise SyntaxError("incomplete varags", cur_token.lineno,
- cur_token.col)
- if index < l:
- token = tokens[index]
- raise SyntaxError("unexpected token" , token.lineno,
- token.col)
- elif cur_token.name == tok.NAME:
- val = cur_token.get_value()
- names.append( ast.AssName( val, consts.OP_ASSIGN ) )
-
- if first_with_default != -1:
- num_expected_with_default = len(names) - first_with_default
- if flags & consts.CO_VARKEYWORDS:
- num_expected_with_default -= 1
- if flags & consts.CO_VARARGS:
- num_expected_with_default -= 1
- if len(defaults) != num_expected_with_default:
- raise SyntaxError('non-default argument follows default argument',
- tokens[0].lineno, tokens[0].col)
- return names, defaults, flags
-
-
-def parse_listcomp(tokens):
- """parses 'for j in k for i in j if i %2 == 0' and returns
- a GenExprFor instance
- XXX: refactor with listmaker ?
- """
- list_fors = []
- ifs = []
- index = 0
- if tokens:
- lineno = tokens[0].lineno
- else:
- lineno = -1
- while index < len(tokens):
- token = tokens[index]
- assert isinstance(token, TokenObject) # rtyper info + check
- if token.get_value() == 'for':
- index += 1 # skip 'for'
- ass_node = to_lvalue(tokens[index], consts.OP_ASSIGN)
- index += 2 # skip 'in'
- iterables = [tokens[index]]
- index += 1
- while index < len(tokens):
- tok2 = tokens[index]
- if not isinstance(tok2, TokenObject):
- break
- if tok2.name != tok.COMMA:
- break
- iterables.append(tokens[index+1])
- index += 2
- if len(iterables) == 1:
- iterable = iterables[0]
- else:
- iterable = ast.Tuple(iterables, token.lineno)
- while index < len(tokens):
- token = tokens[index]
- assert isinstance(token, TokenObject) # rtyper info
- if token.get_value() == 'if':
- ifs.append(ast.ListCompIf(tokens[index+1], token.lineno))
- index += 2
- else:
- break
- list_fors.append(ast.ListCompFor(ass_node, iterable, ifs, lineno))
- ifs = []
- else:
- assert False, 'Unexpected token: expecting for in listcomp'
- #
- # Original implementation:
- #
- # if tokens[index].get_value() == 'for':
- # index += 1 # skip 'for'
- # ass_node = to_lvalue(tokens[index], consts.OP_ASSIGN)
- # index += 2 # skip 'in'
- # iterable = tokens[index]
- # index += 1
- # while index < len(tokens) and tokens[index].get_value() == 'if':
- # ifs.append(ast.ListCompIf(tokens[index+1]))
- # index += 2
- # list_fors.append(ast.ListCompFor(ass_node, iterable, ifs))
- # ifs = []
- # else:
- # raise ValueError('Unexpected token: %s' % tokens[index])
- return list_fors
-
-
-def parse_genexpr_for(tokens):
- """parses 'for j in k for i in j if i %2 == 0' and returns
- a GenExprFor instance
- XXX: if RPYTHON supports to pass a class object to a function,
- we could refactor parse_listcomp and parse_genexpr_for,
- and call :
- - parse_listcomp(tokens, forclass=ast.GenExprFor, ifclass=...)
- or:
- - parse_listcomp(tokens, forclass=ast.ListCompFor, ifclass=...)
- """
- genexpr_fors = []
- ifs = []
- index = 0
- if tokens:
- lineno = tokens[0].lineno
- else:
- lineno = -1
- while index < len(tokens):
- token = tokens[index]
- assert isinstance(token, TokenObject) # rtyper info + check
- if token.get_value() == 'for':
- index += 1 # skip 'for'
- ass_node = to_lvalue(tokens[index], consts.OP_ASSIGN)
- index += 2 # skip 'in'
- iterable = tokens[index]
- index += 1
- while index < len(tokens):
- token = tokens[index]
- assert isinstance(token, TokenObject) # rtyper info
- if token.get_value() == 'if':
- ifs.append(ast.GenExprIf(tokens[index+1], token.lineno))
- index += 2
- else:
- break
- genexpr_fors.append(ast.GenExprFor(ass_node, iterable, ifs, lineno))
- ifs = []
- else:
- raise SyntaxError('invalid syntax',
- token.lineno, token.col)
- return genexpr_fors
-
-
-def get_docstring(builder,stmt):
- """parses a Stmt node.
-
- If a docstring if found, the Discard node is **removed**
- from <stmt> and the docstring is returned.
-
- If no docstring is found, <stmt> is left unchanged
- and None is returned
- """
- if not isinstance(stmt, ast.Stmt):
- return None
- doc = builder.wrap_none()
- if len(stmt.nodes):
- first_child = stmt.nodes[0]
- if isinstance(first_child, ast.Discard):
- expr = first_child.expr
- if builder.is_basestring_const(expr):
- # This *is* a docstring, remove it from stmt list
- assert isinstance(expr, ast.Const)
- del stmt.nodes[0]
- doc = expr.value
- return doc
-
-
-def to_lvalue(ast_node, flags):
- lineno = ast_node.lineno
- if isinstance( ast_node, ast.Name ):
- return ast.AssName(ast_node.varname, flags, lineno)
- # return ast.AssName(ast_node.name, flags)
- elif isinstance(ast_node, ast.Tuple):
- nodes = []
- # FIXME: should ast_node.getChildren() but it's not annotable
- # because of flatten()
- for node in ast_node.nodes:
- nodes.append(to_lvalue(node, flags))
- return ast.AssTuple(nodes, lineno)
- elif isinstance(ast_node, ast.List):
- nodes = []
- # FIXME: should ast_node.getChildren() but it's not annotable
- # because of flatten()
- for node in ast_node.nodes:
- nodes.append(to_lvalue(node, flags))
- return ast.AssList(nodes, lineno)
- elif isinstance(ast_node, ast.Getattr):
- expr = ast_node.expr
- assert isinstance(ast_node, ast.Getattr)
- attrname = ast_node.attrname
- return ast.AssAttr(expr, attrname, flags, lineno)
- elif isinstance(ast_node, ast.Subscript):
- ast_node.flags = flags
- return ast_node
- elif isinstance(ast_node, ast.Slice):
- ast_node.flags = flags
- return ast_node
- else:
- if isinstance(ast_node, ast.GenExpr):
- raise SyntaxError("assign to generator expression not possible",
- lineno, 0, '')
- elif isinstance(ast_node, ast.ListComp):
- raise SyntaxError("can't assign to list comprehension",
- lineno, 0, '')
- elif isinstance(ast_node, ast.CallFunc):
- if flags == consts.OP_DELETE:
- raise SyntaxError("can't delete function call",
- lineno, 0, '')
- else:
- raise SyntaxError("can't assign to function call",
- lineno, 0, '')
- else:
- raise SyntaxError("can't assign to non-lvalue",
- lineno, 0, '')
-
-def is_augassign( ast_node ):
- if ( isinstance( ast_node, ast.Name ) or
- isinstance( ast_node, ast.Slice ) or
- isinstance( ast_node, ast.Subscript ) or
- isinstance( ast_node, ast.Getattr ) ):
- return True
- return False
-
-def get_atoms(builder, nb):
- atoms = []
- i = nb
- while i>0:
- obj = builder.pop()
- if isinstance(obj, BaseRuleObject):
- i += obj.count
- else:
- atoms.append( obj )
- i -= 1
- atoms.reverse()
- return atoms
-
-#def eval_string(value):
-# """temporary implementation
-#
-# FIXME: need to be finished (check compile.c (parsestr) and
-# stringobject.c (PyString_DecodeEscape()) for complete implementation)
-# """
-# # return eval(value)
-# if len(value) == 2:
-# return ''
-# result = ''
-# length = len(value)
-# quotetype = value[0]
-# index = 1
-# while index < length and value[index] == quotetype:
-# index += 1
-# if index == 6:
-# # empty strings like """""" or ''''''
-# return ''
-# # XXX: is it RPYTHON to do this value[index:-index]
-# chars = [char for char in value[index:len(value)-index]]
-# result = ''.join(chars)
-# result = result.replace('\\\\', '\\')
-# d = {'\\b' : '\b', '\\f' : '\f', '\\t' : '\t', '\\n' : '\n',
-# '\\r' : '\r', '\\v' : '\v', '\\a' : '\a',
-# }
-# for escaped, value in d.items():
-# result = result.replace(escaped, value)
-# return result
-
-
-## misc utilities, especially for power: rule
-def reduce_callfunc(obj, arglist):
- """generic factory for CallFunc nodes"""
- assert isinstance(arglist, ArglistObject)
- return ast.CallFunc(obj, arglist.arguments,
- arglist.stararg, arglist.dstararg, arglist.lineno)
-
-def reduce_subscript(obj, subscript):
- """generic factory for Subscript nodes"""
- assert isinstance(subscript, SubscriptObject)
- return ast.Subscript(obj, consts.OP_APPLY, subscript.value, subscript.lineno)
-
-def reduce_slice(obj, sliceobj):
- """generic factory for Slice nodes"""
- assert isinstance(sliceobj, SlicelistObject)
- if sliceobj.fake_rulename == 'slice':
- start = sliceobj.value[0]
- end = sliceobj.value[1]
- return ast.Slice(obj, consts.OP_APPLY, start, end, sliceobj.lineno)
- else:
- return ast.Subscript(obj, consts.OP_APPLY, ast.Sliceobj(sliceobj.value,
- sliceobj.lineno), sliceobj.lineno)
-
-def parse_attraccess(tokens):
- """parses token list like ['a', '.', 'b', '.', 'c', ...]
-
- and returns an ast node : ast.Getattr(Getattr(Name('a'), 'b'), 'c' ...)
- """
- token = tokens[0]
- # XXX HACK for when parse_attraccess is called from build_decorator
- if isinstance(token, TokenObject):
- val = token.get_value()
- result = ast.Name(val, token.lineno)
- else:
- result = token
- index = 1
- while index < len(tokens):
- token = tokens[index]
- if isinstance(token, TokenObject) and token.name == tok.DOT:
- index += 1
- token = tokens[index]
- assert isinstance(token, TokenObject)
- result = ast.Getattr(result, token.get_value(), token.lineno)
- elif isinstance(token, ArglistObject):
- result = reduce_callfunc(result, token)
- elif isinstance(token, SubscriptObject):
- result = reduce_subscript(result, token)
- elif isinstance(token, SlicelistObject):
- result = reduce_slice(result, token)
- else:
- assert False, "Don't know how to handle index %s of %s" % (index, len(tokens))
- index += 1
- return result
-
+from pypy.interpreter.gateway import interp2app
+from asthelper import *
## building functions helpers
## --------------------------
@@ -546,31 +46,31 @@
top = atoms[0]
if isinstance(top, TokenObject):
# assert isinstance(top, TokenObject) # rtyper
- if top.name == tok.LPAR:
+ if top.name == builder.parser.tokens['LPAR']:
if len(atoms) == 2:
builder.push(ast.Tuple([], top.lineno))
else:
builder.push( atoms[1] )
- elif top.name == tok.LSQB:
+ elif top.name == builder.parser.tokens['LSQB']:
if len(atoms) == 2:
builder.push(ast.List([], top.lineno))
else:
list_node = atoms[1]
list_node.lineno = top.lineno
builder.push(list_node)
- elif top.name == tok.LBRACE:
+ elif top.name == builder.parser.tokens['LBRACE']:
items = []
for index in range(1, len(atoms)-1, 4):
# a : b , c : d
# ^ +1 +2 +3 +4
items.append((atoms[index], atoms[index+2]))
builder.push(ast.Dict(items, top.lineno))
- elif top.name == tok.NAME:
+ elif top.name == builder.parser.tokens['NAME']:
val = top.get_value()
builder.push( ast.Name(val, top.lineno) )
- elif top.name == tok.NUMBER:
+ elif top.name == builder.parser.tokens['NUMBER']:
builder.push(ast.Const(builder.eval_number(top.get_value()), top.lineno))
- elif top.name == tok.STRING:
+ elif top.name == builder.parser.tokens['STRING']:
# need to concatenate strings in atoms
s = ''
if len(atoms) == 1:
@@ -586,7 +86,7 @@
accum.append(parsestr(builder.space, builder.source_encoding, token.get_value()))
w_s = space.call_method(empty, 'join', space.newlist(accum))
builder.push(ast.Const(w_s, top.lineno))
- elif top.name == tok.BACKQUOTE:
+ elif top.name == builder.parser.tokens['BACKQUOTE']:
builder.push(ast.Backquote(atoms[1], atoms[1].lineno))
else:
raise SyntaxError("unexpected tokens", top.lineno, top.col)
@@ -607,11 +107,11 @@
else:
lineno = atoms[0].lineno
token = atoms[-2]
- if isinstance(token, TokenObject) and token.name == tok.DOUBLESTAR:
- obj = parse_attraccess(slicecut(atoms, 0, -2))
+ if isinstance(token, TokenObject) and token.name == builder.parser.tokens['DOUBLESTAR']:
+ obj = parse_attraccess(slicecut(atoms, 0, -2), builder)
builder.push(ast.Power( obj, atoms[-1], lineno))
else:
- obj = parse_attraccess(atoms)
+ obj = parse_attraccess(atoms, builder)
builder.push(obj)
def build_factor(builder, nb):
@@ -622,11 +122,11 @@
token = atoms[0]
lineno = token.lineno
if isinstance(token, TokenObject):
- if token.name == tok.PLUS:
+ if token.name == builder.parser.tokens['PLUS']:
builder.push( ast.UnaryAdd( atoms[1], lineno) )
- if token.name == tok.MINUS:
+ if token.name == builder.parser.tokens['MINUS']:
builder.push( ast.UnarySub( atoms[1], lineno) )
- if token.name == tok.TILDE:
+ if token.name == builder.parser.tokens['TILDE']:
builder.push( ast.Invert( atoms[1], lineno) )
def build_term(builder, nb):
@@ -637,13 +137,13 @@
right = atoms[i]
op_node = atoms[i-1]
assert isinstance(op_node, TokenObject)
- if op_node.name == tok.STAR:
+ if op_node.name == builder.parser.tokens['STAR']:
left = ast.Mul( left, right, left.lineno )
- elif op_node.name == tok.SLASH:
+ elif op_node.name == builder.parser.tokens['SLASH']:
left = ast.Div( left, right, left.lineno )
- elif op_node.name == tok.PERCENT:
+ elif op_node.name == builder.parser.tokens['PERCENT']:
left = ast.Mod( left, right, left.lineno )
- elif op_node.name == tok.DOUBLESLASH:
+ elif op_node.name == builder.parser.tokens['DOUBLESLASH']:
left = ast.FloorDiv( left, right, left.lineno )
else:
token = atoms[i-1]
@@ -658,9 +158,9 @@
right = atoms[i]
op_node = atoms[i-1]
assert isinstance(op_node, TokenObject)
- if op_node.name == tok.PLUS:
+ if op_node.name == builder.parser.tokens['PLUS']:
left = ast.Add( left, right, left.lineno)
- elif op_node.name == tok.MINUS:
+ elif op_node.name == builder.parser.tokens['MINUS']:
left = ast.Sub( left, right, left.lineno)
else:
token = atoms[i-1]
@@ -676,9 +176,9 @@
right = atoms[i]
op_node = atoms[i-1]
assert isinstance(op_node, TokenObject)
- if op_node.name == tok.LEFTSHIFT:
+ if op_node.name == builder.parser.tokens['LEFTSHIFT']:
left = ast.LeftShift( left, right, lineno )
- elif op_node.name == tok.RIGHTSHIFT:
+ elif op_node.name == builder.parser.tokens['RIGHTSHIFT']:
left = ast.RightShift( left, right, lineno )
else:
token = atoms[i-1]
@@ -727,7 +227,7 @@
# 'is', 'is not', 'not' or 'not in' => tok.get_value()
token = atoms[i]
assert isinstance(token, TokenObject)
- op_name = tok.tok_rpunct.get(token.name, token.get_value())
+ op_name = builder.parser.tok_rvalues.get(token.name, token.get_value())
ops.append((op_name, atoms[i+1]))
builder.push(ast.Compare(atoms[0], ops, atoms[0].lineno))
@@ -755,15 +255,18 @@
lineno = token.lineno
assert isinstance(token, TokenObject)
if token.get_value() == 'not':
- builder.push(TokenObject(tok.NAME, 'not in', lineno))
+ builder.push(TokenObject(builder.parser.tokens['NAME'], 'not in', lineno, builder.parser))
else:
- builder.push(TokenObject(tok.NAME, 'is not', lineno))
+ builder.push(TokenObject(builder.parser.tokens['NAME'], 'is not', lineno, builder.parser))
else:
assert False, "TODO" # uh ?
def build_or_test(builder, nb):
return build_binary_expr(builder, nb, ast.Or)
+def build_or_test(builder, nb):
+ return build_binary_expr(builder, nb, ast.Or)
+
def build_and_test(builder, nb):
return build_binary_expr(builder, nb, ast.And)
@@ -811,7 +314,7 @@
return
op = atoms[1]
assert isinstance(op, TokenObject)
- if op.name == tok.EQUAL:
+ if op.name == builder.parser.tokens['EQUAL']:
nodes = []
for i in range(0,l-2,2):
lvalue = to_lvalue(atoms[i], consts.OP_ASSIGN)
@@ -845,7 +348,7 @@
lineno = -1
for n in range(0,l,2):
node = atoms[n]
- if isinstance(node, TokenObject) and node.name == tok.NEWLINE:
+ if isinstance(node, TokenObject) and node.name == builder.parser.tokens['NEWLINE']:
nodes.append(ast.Discard(ast.Const(builder.wrap_none()), node.lineno))
else:
nodes.append(node)
@@ -871,10 +374,10 @@
for node in atoms:
if isinstance(node, ast.Stmt):
stmts.extend(node.nodes)
- elif isinstance(node, TokenObject) and node.name == tok.ENDMARKER:
+ elif isinstance(node, TokenObject) and node.name == builder.parser.tokens['ENDMARKER']:
# XXX Can't we just remove the last element of the list ?
break
- elif isinstance(node, TokenObject) and node.name == tok.NEWLINE:
+ elif isinstance(node, TokenObject) and node.name == builder.parser.tokens['NEWLINE']:
continue
else:
stmts.append(node)
@@ -894,11 +397,12 @@
l = len(atoms)
if l == 1 or l==2:
atom0 = atoms[0]
- if isinstance(atom0, TokenObject) and atom0.name == tok.NEWLINE:
- atom0 = ast.Pass(atom0.lineno)
+ if isinstance(atom0, TokenObject) and atom0.name == builder.parser.tokens['NEWLINE']:
+ # atom0 = ast.Pass(atom0.lineno) # break test_astcompiler
+ atom0 = ast.Stmt([], atom0.lineno) # break test_astbuilder
elif not isinstance(atom0, ast.Stmt):
atom0 = ast.Stmt([atom0], atom0.lineno)
- builder.push(ast.Module(builder.wrap_none(), atom0, atom0.lineno))
+ builder.push(ast.Module(builder.space.w_None, atom0, atom0.lineno))
else:
assert False, "Forbidden path"
@@ -914,7 +418,7 @@
return
items = []
token = atoms[1]
- if isinstance(token, TokenObject) and token.name == tok.COMMA:
+ if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COMMA']:
for i in range(0, l, 2): # this is atoms not 1
items.append(atoms[i])
else:
@@ -944,22 +448,23 @@
atoms = get_atoms(builder, nb)
lineno = atoms[0].lineno
code = atoms[-1]
- names, defaults, flags = parse_arglist(slicecut(atoms, 1, -2))
+ names, defaults, flags = parse_arglist(slicecut(atoms, 1, -2), builder)
builder.push(ast.Lambda(names, defaults, flags, code, lineno))
+
def build_trailer(builder, nb):
"""trailer: '(' ')' | '(' arglist ')' | '[' subscriptlist ']' | '.' NAME
"""
atoms = get_atoms(builder, nb)
first_token = atoms[0]
# Case 1 : '(' ...
- if isinstance(first_token, TokenObject) and first_token.name == tok.LPAR:
- if len(atoms) == 2: # and atoms[1].token == tok.RPAR:
+ if isinstance(first_token, TokenObject) and first_token.name == builder.parser.tokens['LPAR']:
+ if len(atoms) == 2: # and atoms[1].token == builder.parser.tokens['RPAR']:
builder.push(ArglistObject([], None, None, first_token.lineno))
elif len(atoms) == 3: # '(' Arglist ')'
# push arglist on the stack
builder.push(atoms[1])
- elif isinstance(first_token, TokenObject) and first_token.name == tok.LSQB:
+ elif isinstance(first_token, TokenObject) and first_token.name == builder.parser.tokens['LSQB']:
if len(atoms) == 3 and isinstance(atoms[1], SlicelistObject):
builder.push(atoms[1])
else:
@@ -994,6 +499,7 @@
else:
assert False, "Trailer reducing implementation incomplete !"
+
def build_arglist(builder, nb):
"""
arglist: (argument ',')* ( '*' test [',' '**' test] |
@@ -1002,7 +508,7 @@
[argument ','] )
"""
atoms = get_atoms(builder, nb)
- arguments, stararg, dstararg = parse_argument(atoms)
+ arguments, stararg, dstararg = parse_argument(atoms, builder)
if atoms:
lineno = atoms[0].lineno
else:
@@ -1010,16 +516,17 @@
builder.push(ArglistObject(arguments, stararg, dstararg, lineno))
+
def build_subscript(builder, nb):
"""'.' '.' '.' | [test] ':' [test] [':' [test]] | test"""
atoms = get_atoms(builder, nb)
token = atoms[0]
lineno = token.lineno
- if isinstance(token, TokenObject) and token.name == tok.DOT:
+ if isinstance(token, TokenObject) and token.name == builder.parser.tokens['DOT']:
# Ellipsis:
builder.push(ast.Ellipsis(lineno))
elif len(atoms) == 1:
- if isinstance(token, TokenObject) and token.name == tok.COLON:
+ if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COLON']:
sliceinfos = [None, None, None]
builder.push(SlicelistObject('slice', sliceinfos, lineno))
else:
@@ -1029,7 +536,7 @@
sliceinfos = [None, None, None]
infosindex = 0
for token in atoms:
- if isinstance(token, TokenObject) and token.name == tok.COLON:
+ if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COLON']:
infosindex += 1
else:
sliceinfos[infosindex] = token
@@ -1044,7 +551,6 @@
else:
builder.push(SlicelistObject('slice', sliceinfos, lineno))
-
def build_listmaker(builder, nb):
"""listmaker: test ( list_for | (',' test)* [','] )"""
atoms = get_atoms(builder, nb)
@@ -1055,7 +561,7 @@
if token.get_value() == 'for':
# list comp
expr = atoms[0]
- list_for = parse_listcomp(atoms[1:])
+ list_for = parse_listcomp(atoms[1:], builder)
builder.push(ast.ListComp(expr, list_for, lineno))
return
# regular list building (like in [1, 2, 3,])
@@ -1077,13 +583,15 @@
nodes = []
# remove '@', '(' and ')' from atoms and use parse_attraccess
for token in atoms[1:]:
- if isinstance(token, TokenObject) and \
- token.name in (tok.LPAR, tok.RPAR, tok.NEWLINE):
+ if isinstance(token, TokenObject) and (
+ token.name == builder.parser.tokens['LPAR']
+ or token.name == builder.parser.tokens['RPAR']
+ or token.name == builder.parser.tokens['NEWLINE']):
# skip those ones
continue
else:
nodes.append(token)
- obj = parse_attraccess(nodes)
+ obj = parse_attraccess(nodes, builder)
builder.push(obj)
def build_funcdef(builder, nb):
@@ -1112,7 +620,7 @@
arglist = []
index = 3
arglist = slicecut(atoms, 3, -3)
- names, default, flags = parse_arglist(arglist)
+ names, default, flags = parse_arglist(arglist, builder)
funcname_token = atoms[1]
assert isinstance(funcname_token, TokenObject)
funcname = funcname_token.get_value()
@@ -1293,7 +801,7 @@
while index < l:
as_name = None
# dotted name (a.b.c)
- incr, name = parse_dotted_names(atoms[index:])
+ incr, name = parse_dotted_names(atoms[index:], builder)
index += incr
# 'as' value
if index < l:
@@ -1310,11 +818,11 @@
while index<l:
atom = atoms[index]
# for atom in atoms[index:]:
- if isinstance(atom, TokenObject) and atom.name == tok.COMMA:
+ if isinstance(atom, TokenObject) and atom.name == builder.parser.tokens['COMMA']:
break
index += 1
## while index < l and isinstance(atoms[index], TokenObject) and \
-## atoms[index].name != tok.COMMA:
+## atoms[index].name != builder.parser.tokens['COMMA']:
## index += 1
index += 1
builder.push(ast.Import(names, atoms[0].lineno))
@@ -1329,14 +837,14 @@
"""
atoms = get_atoms(builder, nb)
index = 1
- incr, from_name = parse_dotted_names(atoms[index:])
+ incr, from_name = parse_dotted_names(atoms[index:], builder)
index += (incr + 1) # skip 'import'
token = atoms[index]
assert isinstance(token, TokenObject) # XXX
- if token.name == tok.STAR:
+ if token.name == builder.parser.tokens['STAR']:
names = [('*', None)]
else:
- if token.name == tok.LPAR:
+ if token.name == builder.parser.tokens['LPAR']:
# mutli-line imports
tokens = slicecut( atoms, index+1, -1 )
else:
@@ -1417,14 +925,14 @@
start = 1
if l > 1:
token = atoms[1]
- if isinstance(token, TokenObject) and token.name == tok.RIGHTSHIFT:
+ if isinstance(token, TokenObject) and token.name == builder.parser.tokens['RIGHTSHIFT']:
dest = atoms[2]
# skip following comma
start = 4
for index in range(start, l, 2):
items.append(atoms[index])
last_token = atoms[-1]
- if isinstance(last_token, TokenObject) and last_token.name == tok.COMMA:
+ if isinstance(last_token, TokenObject) and last_token.name == builder.parser.tokens['COMMA']:
builder.push(ast.Print(items, dest, atoms[0].lineno))
else:
builder.push(ast.Printnl(items, dest, atoms[0].lineno))
@@ -1464,8 +972,8 @@
"""
atoms = get_atoms(builder, nb)
- l = len(atoms)
handlers = []
+ l = len(atoms)
else_ = None
body = atoms[2]
token = atoms[3]
@@ -1544,133 +1052,7 @@
'eval_input' : build_eval_input,
'with_stmt' : build_with_stmt,
}
-
-# Build two almost identical ASTRULES dictionaries
-ASTRULES = dict([(sym[key], value) for (key, value) in
- ASTRULES_Template.iteritems() if key in sym])
-del ASTRULES_Template
-
-## Stack elements definitions ###################################
-
-class BaseRuleObject(ast.Node):
- """Base class for unnamed rules"""
- def __init__(self, count, lineno):
- self.count = count
- self.lineno = lineno # src.getline()
- self.col = 0 # src.getcol()
-
-
-class RuleObject(BaseRuleObject):
- """A simple object used to wrap a rule or token"""
- def __init__(self, name, count, lineno):
- BaseRuleObject.__init__(self, count, lineno)
- self.rulename = name
-
- def __str__(self):
- return "<Rule: %s/%d>" % (sym.sym_name[self.rulename], self.count)
-
- def __repr__(self):
- return "<Rule: %s/%d>" % (sym.sym_name[self.rulename], self.count)
-
-
-class TempRuleObject(BaseRuleObject):
- """used to keep track of how many items get_atom() should pop"""
-
- def __init__(self, name, count, lineno):
- BaseRuleObject.__init__(self, count, lineno)
- self.temp_rulename = name
-
- def __str__(self):
- return "<Rule: %s/%d>" % (self.temp_rulename, self.count)
-
- def __repr__(self):
- return "<Rule: %s/%d>" % (self.temp_rulename, self.count)
-
-
-class TokenObject(ast.Node):
- """A simple object used to wrap a rule or token"""
- def __init__(self, name, value, lineno):
- self.name = name
- self.value = value
- self.count = 0
- # self.line = 0 # src.getline()
- self.col = 0 # src.getcol()
- self.lineno = lineno
-
- def get_name(self):
- return tok.tok_rpunct.get(self.name,
- tok.tok_name.get(self.name, str(self.name)))
-
- def get_value(self):
- value = self.value
- if value is None:
- value = ''
- return value
-
- def __str__(self):
- return "<Token: (%s,%s)>" % (self.get_name(), self.value)
-
- def __repr__(self):
- return "<Token: (%r,%s)>" % (self.get_name(), self.value)
-
-
-class ObjectAccessor(ast.Node):
- """base class for ArglistObject, SubscriptObject and SlicelistObject
-
- FIXME: think about a more appropriate name
- """
-
-class ArglistObject(ObjectAccessor):
- """helper class to build function's arg list
- """
- def __init__(self, arguments, stararg, dstararg, lineno):
- self.fake_rulename = 'arglist'
- self.arguments = arguments
- self.stararg = stararg
- self.dstararg = dstararg
- self.lineno = lineno
-
- def __str__(self):
- return "<ArgList: (%s, %s, %s)>" % self.value
-
- def __repr__(self):
- return "<ArgList: (%s, %s, %s)>" % self.value
-
-class SubscriptObject(ObjectAccessor):
- """helper class to build subscript list
-
- self.value represents the __getitem__ argument
- """
- def __init__(self, name, value, lineno):
- self.fake_rulename = name
- self.value = value
- self.lineno = lineno
-
- def __str__(self):
- return "<SubscriptList: (%s)>" % self.value
-
- def __repr__(self):
- return "<SubscriptList: (%s)>" % self.value
-
-class SlicelistObject(ObjectAccessor):
- """helper class to build slice objects
-
- self.value is a list [start, end, step]
- self.fake_rulename can either be 'slice' or 'sliceobj' depending
- on if a step is specfied or not (see Python's AST
- for more information on that)
- """
- def __init__(self, name, value, lineno):
- self.fake_rulename = name
- self.value = value
- self.lineno = lineno
-
- def __str__(self):
- return "<SliceList: (%s)>" % self.value
-
- def __repr__(self):
- return "<SliceList: (%s)>" % self.value
-
+
class AstBuilderContext(AbstractContext):
"""specific context management for AstBuidler"""
@@ -1681,97 +1063,87 @@
class AstBuilder(BaseGrammarBuilder):
"""A builder that directly produce the AST"""
- def __init__(self, rules=None, debug=0, space=None):
- BaseGrammarBuilder.__init__(self, rules, debug)
+ def __init__(self, parser, debug=0, space=None):
+ BaseGrammarBuilder.__init__(self, parser, debug)
self.rule_stack = []
self.space = space
self.source_encoding = None
self.with_enabled = False
+ self.build_rules = ASTRULES_Template
+ self.user_build_rules = {}
def enable_with(self):
if self.with_enabled:
return
self.with_enabled = True
- self.keywords.update({'with':None, 'as': None})
-
+ # XXX
+ # self.keywords.update({'with':None, 'as': None})
+
def context(self):
return AstBuilderContext(self.rule_stack)
def restore(self, ctx):
-## if DEBUG_MODE:
-## print "Restoring context (%s)" % (len(ctx.rule_stack))
assert isinstance(ctx, AstBuilderContext)
assert len(self.rule_stack) >= ctx.d
del self.rule_stack[ctx.d:]
- #self.rule_stack = ctx.rule_stack
def pop(self):
return self.rule_stack.pop(-1)
def push(self, obj):
self.rule_stack.append(obj)
- if not isinstance(obj, RuleObject) and not isinstance(obj, TokenObject):
-## if DEBUG_MODE:
-## print "Pushed:", str(obj), len(self.rule_stack)
- pass
- elif isinstance(obj, TempRuleObject):
-## if DEBUG_MODE:
-## print "Pushed:", str(obj), len(self.rule_stack)
- pass
- # print "\t", self.rule_stack
def push_tok(self, name, value, src ):
- self.push( TokenObject( name, value, src._token_lnum ) )
+ self.push( TokenObject( name, value, src._token_lnum, self.parser ) )
def push_rule(self, name, count, src ):
- self.push( RuleObject( name, count, src._token_lnum ) )
+ self.push( RuleObject( name, count, src._token_lnum, self.parser ) )
def alternative( self, rule, source ):
# Do nothing, keep rule on top of the stack
-## rule_stack = self.rule_stack[:]
if rule.is_root():
-## if DEBUG_MODE:
-## print "ALT:", sym.sym_name[rule.codename], self.rule_stack
- builder_func = ASTRULES.get(rule.codename, None)
- if builder_func:
- builder_func(self, 1)
+ rulename = self.parser.sym_name[rule.codename]
+ # builder_func = ASTRULES.get(rule.codename, None)
+ w_func = self.user_build_rules.get(rulename, None)
+ # user defined (applevel) function
+ if w_func:
+ w_items = self.space.newlist( [self.space.wrap( it ) for it in get_atoms(self, 1)] )
+ w_astnode = self.space.call_function(w_func, w_items)
+ astnode = self.space.interp_w(ast.Node, w_astnode, can_be_None=False)
+ self.push(astnode)
else:
-## if DEBUG_MODE:
-## print "No reducing implementation for %s, just push it on stack" % (
-## sym.sym_name[rule.codename])
- self.push_rule(rule.codename, 1, source)
+ builder_func = self.build_rules.get(rulename, None)
+ if builder_func:
+ builder_func(self, 1)
+ else:
+ self.push_rule(rule.codename, 1, source)
else:
self.push_rule(rule.codename, 1, source)
-## if DEBUG_MODE > 1:
-## show_stack(rule_stack, self.rule_stack)
-## x = raw_input("Continue ?")
return True
def sequence(self, rule, source, elts_number):
""" """
-## rule_stack = self.rule_stack[:]
if rule.is_root():
-## if DEBUG_MODE:
-## print "SEQ:", sym.sym_name[rule.codename]
- builder_func = ASTRULES.get(rule.codename, None)
- if builder_func:
- # print "REDUCING SEQUENCE %s" % sym.sym_name[rule.codename]
- builder_func(self, elts_number)
+ rulename = self.parser.sym_name[rule.codename]
+ # builder_func = ASTRULES.get(rule.codename, None)
+ w_func = self.user_build_rules.get(rulename, None)
+ # user defined (applevel) function
+ if w_func:
+ w_items = self.space.newlist( [self.space.wrap( it ) for it in get_atoms(self, elts_number)] )
+ w_astnode = self.space.call_function(w_func, w_items)
+ astnode = self.space.interp_w(ast.Node, w_astnode, can_be_None=False)
+ self.push(astnode)
else:
-## if DEBUG_MODE:
-## print "No reducing implementation for %s, just push it on stack" % (
-## sym.sym_name[rule.codename])
- self.push_rule(rule.codename, elts_number, source)
+ builder_func = self.build_rules.get(rulename, None)
+ if builder_func:
+ builder_func(self, elts_number)
+ else:
+ self.push_rule(rule.codename, elts_number, source)
else:
self.push_rule(rule.codename, elts_number, source)
-## if DEBUG_MODE > 1:
-## show_stack(rule_stack, self.rule_stack)
-## raw_input("Continue ?")
return True
def token(self, name, value, source):
-## if DEBUG_MODE:
-## print "TOK:", tok.tok_name[name], name, value
self.push_tok(name, value, source)
return True
@@ -1799,7 +1171,7 @@
return space.call_function(f, space.wrap(value))
def is_basestring_const(self, expr):
- if not isinstance(expr,ast.Const):
+ if not isinstance(expr, ast.Const):
return False
space = self.space
return space.is_true(space.isinstance(expr.value,space.w_basestring))
Added: pypy/dist/pypy/interpreter/pyparser/asthelper.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/interpreter/pyparser/asthelper.py Wed Feb 28 18:30:48 2007
@@ -0,0 +1,635 @@
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
+from pypy.interpreter.astcompiler import ast, consts
+from pypy.interpreter.pyparser.error import SyntaxError
+
+
+### Parsing utilites #################################################
+def parse_except_clause(tokens):
+ """parses 'except' [test [',' test]] ':' suite
+ and returns a 4-tuple : (tokens_read, expr1, expr2, except_body)
+ """
+ lineno = tokens[0].lineno
+ clause_length = 1
+ # Read until end of except clause (bound by following 'else',
+ # or 'except' or end of tokens)
+ while clause_length < len(tokens):
+ token = tokens[clause_length]
+ if isinstance(token, TokenObject) and \
+ (token.get_value() == 'except' or token.get_value() == 'else'):
+ break
+ clause_length += 1
+ if clause_length == 3:
+ # case 'except: body'
+ return (3, None, None, tokens[2])
+ elif clause_length == 4:
+ # case 'except Exception: body':
+ return (4, tokens[1], None, tokens[3])
+ else:
+ # case 'except Exception, exc: body'
+ return (6, tokens[1], to_lvalue(tokens[3], consts.OP_ASSIGN), tokens[5])
+
+
+def parse_dotted_names(tokens, builder):
+ """parses NAME('.' NAME)* and returns full dotted name
+
+ this function doesn't assume that the <tokens> list ends after the
+ last 'NAME' element
+ """
+ first = tokens[0]
+ assert isinstance(first, TokenObject)
+ name = first.get_value()
+ l = len(tokens)
+ index = 1
+ for index in range(1, l, 2):
+ token = tokens[index]
+ assert isinstance(token, TokenObject)
+ if token.name != builder.parser.tokens['DOT']:
+ break
+ token = tokens[index+1]
+ assert isinstance(token, TokenObject)
+ name += '.'
+ value = token.get_value()
+ name += value
+ return (index, name)
+
+def parse_argument(tokens, builder):
+ """parses function call arguments"""
+ l = len(tokens)
+ index = 0
+ arguments = []
+ last_token = None
+ building_kw = False
+ kw_built = False
+ stararg_token = None
+ dstararg_token = None
+ while index < l:
+ cur_token = tokens[index]
+ if not isinstance(cur_token, TokenObject):
+ index += 1
+ if not building_kw:
+ arguments.append(cur_token)
+ else:
+ last_token = arguments.pop()
+ assert isinstance(last_token, ast.Name) # used by rtyper
+ arguments.append(ast.Keyword(last_token.varname, cur_token, last_token.lineno))
+ building_kw = False
+ kw_built = True
+ continue
+ elif cur_token.name == builder.parser.tokens['COMMA']:
+ index += 1
+ continue
+ elif cur_token.name == builder.parser.tokens['EQUAL']:
+ index += 1
+ building_kw = True
+ continue
+ elif cur_token.name == builder.parser.tokens['STAR'] or cur_token.name == builder.parser.tokens['DOUBLESTAR']:
+ index += 1
+ if cur_token.name == builder.parser.tokens['STAR']:
+ stararg_token = tokens[index]
+ index += 1
+ if index >= l:
+ break
+ index += 2 # Skip COMMA and DOUBLESTAR
+ dstararg_token = tokens[index]
+ break
+ elif cur_token.get_value() == 'for':
+ if len(arguments) != 1:
+ raise SyntaxError("invalid syntax", cur_token.lineno,
+ cur_token.col)
+ expr = arguments[0]
+ genexpr_for = parse_genexpr_for(tokens[index:])
+ genexpr_for[0].is_outmost = True
+ gexp = ast.GenExpr(ast.GenExprInner(expr, genexpr_for, expr.lineno), expr.lineno)
+ arguments[0] = gexp
+ break
+ return arguments, stararg_token, dstararg_token
+
+
+def parse_fpdef(tokens, index, builder):
+ """fpdef: fpdef: NAME | '(' fplist ')'
+ fplist: fpdef (',' fpdef)* [',']
+
+ This intend to be a RPYTHON compliant implementation of _parse_fpdef,
+ but it can't work with the default compiler.
+ We switched to use astcompiler module now
+ """
+ nodes = []
+ comma = False
+ while True:
+ token = tokens[index]
+ index += 1
+ assert isinstance(token, TokenObject)
+ if token.name == builder.parser.tokens['LPAR']: # nested item
+ index, node = parse_fpdef(tokens, index, builder)
+ elif token.name == builder.parser.tokens['RPAR']: # end of current nesting
+ break
+ else: # name
+ val = token.get_value()
+ node = ast.AssName(val, consts.OP_ASSIGN, token.lineno)
+ nodes.append(node)
+
+ token = tokens[index]
+ index += 1
+ assert isinstance(token, TokenObject)
+ if token.name == builder.parser.tokens['COMMA']:
+ comma = True
+ else:
+ assert token.name == builder.parser.tokens['RPAR']
+ break
+ if len(nodes) == 1 and not comma:
+ node = nodes[0]
+ else:
+ node = ast.AssTuple(nodes, token.lineno)
+ return index, node
+
+def parse_arglist(tokens, builder):
+ """returns names, defaults, flags"""
+ l = len(tokens)
+ index = 0
+ defaults = []
+ names = []
+ flags = 0
+ first_with_default = -1
+ while index < l:
+ cur_token = tokens[index]
+ index += 1
+ if not isinstance(cur_token, TokenObject):
+ # XXX: think of another way to write this test
+ defaults.append(cur_token)
+ if first_with_default == -1:
+ first_with_default = len(names) - 1
+ elif cur_token.name == builder.parser.tokens['COMMA']:
+ # We could skip test COMMA by incrementing index cleverly
+ # but we might do some experiment on the grammar at some point
+ continue
+ elif cur_token.name == builder.parser.tokens['LPAR']:
+ index, node = parse_fpdef(tokens, index, builder)
+ names.append(node)
+ elif cur_token.name == builder.parser.tokens['STAR'] or cur_token.name == builder.parser.tokens['DOUBLESTAR']:
+ if cur_token.name == builder.parser.tokens['STAR']:
+ cur_token = tokens[index]
+ assert isinstance(cur_token, TokenObject)
+ index += 1
+ if cur_token.name == builder.parser.tokens['NAME']:
+ val = cur_token.get_value()
+ names.append( ast.AssName( val, consts.OP_ASSIGN ) )
+ flags |= consts.CO_VARARGS
+ index += 1
+ if index >= l:
+ break
+ else:
+ # still more tokens to read
+ cur_token = tokens[index]
+ index += 1
+ else:
+ raise SyntaxError("incomplete varags", cur_token.lineno,
+ cur_token.col)
+ assert isinstance(cur_token, TokenObject)
+ if cur_token.name != builder.parser.tokens['DOUBLESTAR']:
+ raise SyntaxError("Unexpected token", cur_token.lineno,
+ cur_token.col)
+ cur_token = tokens[index]
+ index += 1
+ assert isinstance(cur_token, TokenObject)
+ if cur_token.name == builder.parser.tokens['NAME']:
+ val = cur_token.get_value()
+ names.append( ast.AssName( val, consts.OP_ASSIGN ) )
+ flags |= consts.CO_VARKEYWORDS
+ index += 1
+ else:
+ raise SyntaxError("incomplete varags", cur_token.lineno,
+ cur_token.col)
+ if index < l:
+ token = tokens[index]
+ raise SyntaxError("unexpected token" , token.lineno,
+ token.col)
+ elif cur_token.name == builder.parser.tokens['NAME']:
+ val = cur_token.get_value()
+ names.append( ast.AssName( val, consts.OP_ASSIGN ) )
+
+ if first_with_default != -1:
+ num_expected_with_default = len(names) - first_with_default
+ if flags & consts.CO_VARKEYWORDS:
+ num_expected_with_default -= 1
+ if flags & consts.CO_VARARGS:
+ num_expected_with_default -= 1
+ if len(defaults) != num_expected_with_default:
+ raise SyntaxError('non-default argument follows default argument',
+ tokens[0].lineno, tokens[0].col)
+ return names, defaults, flags
+
+
+def parse_listcomp(tokens, builder):
+ """parses 'for j in k for i in j if i %2 == 0' and returns
+ a GenExprFor instance
+ XXX: refactor with listmaker ?
+ """
+ list_fors = []
+ ifs = []
+ index = 0
+ if tokens:
+ lineno = tokens[0].lineno
+ else:
+ lineno = -1
+ while index < len(tokens):
+ token = tokens[index]
+ assert isinstance(token, TokenObject) # rtyper info + check
+ if token.get_value() == 'for':
+ index += 1 # skip 'for'
+ ass_node = to_lvalue(tokens[index], consts.OP_ASSIGN)
+ index += 2 # skip 'in'
+ iterables = [tokens[index]]
+ index += 1
+ while index < len(tokens):
+ tok2 = tokens[index]
+ if not isinstance(tok2, TokenObject):
+ break
+ if tok2.name != builder.parser.tokens['COMMA']:
+ break
+ iterables.append(tokens[index+1])
+ index += 2
+ if len(iterables) == 1:
+ iterable = iterables[0]
+ else:
+ iterable = ast.Tuple(iterables, token.lineno)
+ while index < len(tokens):
+ token = tokens[index]
+ assert isinstance(token, TokenObject) # rtyper info
+ if token.get_value() == 'if':
+ ifs.append(ast.ListCompIf(tokens[index+1], token.lineno))
+ index += 2
+ else:
+ break
+ list_fors.append(ast.ListCompFor(ass_node, iterable, ifs, lineno))
+ ifs = []
+ else:
+ assert False, 'Unexpected token: expecting for in listcomp'
+ #
+ # Original implementation:
+ #
+ # if tokens[index].get_value() == 'for':
+ # index += 1 # skip 'for'
+ # ass_node = to_lvalue(tokens[index], consts.OP_ASSIGN)
+ # index += 2 # skip 'in'
+ # iterable = tokens[index]
+ # index += 1
+ # while index < len(tokens) and tokens[index].get_value() == 'if':
+ # ifs.append(ast.ListCompIf(tokens[index+1]))
+ # index += 2
+ # list_fors.append(ast.ListCompFor(ass_node, iterable, ifs))
+ # ifs = []
+ # else:
+ # raise ValueError('Unexpected token: %s' % tokens[index])
+ return list_fors
+
+
+def parse_genexpr_for(tokens):
+ """parses 'for j in k for i in j if i %2 == 0' and returns
+ a GenExprFor instance
+ XXX: if RPYTHON supports to pass a class object to a function,
+ we could refactor parse_listcomp and parse_genexpr_for,
+ and call :
+ - parse_listcomp(tokens, forclass=ast.GenExprFor, ifclass=...)
+ or:
+ - parse_listcomp(tokens, forclass=ast.ListCompFor, ifclass=...)
+ """
+ genexpr_fors = []
+ ifs = []
+ index = 0
+ if tokens:
+ lineno = tokens[0].lineno
+ else:
+ lineno = -1
+ while index < len(tokens):
+ token = tokens[index]
+ assert isinstance(token, TokenObject) # rtyper info + check
+ if token.get_value() == 'for':
+ index += 1 # skip 'for'
+ ass_node = to_lvalue(tokens[index], consts.OP_ASSIGN)
+ index += 2 # skip 'in'
+ iterable = tokens[index]
+ index += 1
+ while index < len(tokens):
+ token = tokens[index]
+ assert isinstance(token, TokenObject) # rtyper info
+ if token.get_value() == 'if':
+ ifs.append(ast.GenExprIf(tokens[index+1], token.lineno))
+ index += 2
+ else:
+ break
+ genexpr_fors.append(ast.GenExprFor(ass_node, iterable, ifs, lineno))
+ ifs = []
+ else:
+ raise SyntaxError('invalid syntax',
+ token.lineno, token.col)
+ return genexpr_fors
+
+def get_docstring(builder,stmt):
+ """parses a Stmt node.
+
+ If a docstring if found, the Discard node is **removed**
+ from <stmt> and the docstring is returned.
+
+ If no docstring is found, <stmt> is left unchanged
+ and None is returned
+ """
+ if not isinstance(stmt, ast.Stmt):
+ return None
+ doc = builder.wrap_none()
+ if len(stmt.nodes):
+ first_child = stmt.nodes[0]
+ if isinstance(first_child, ast.Discard):
+ expr = first_child.expr
+ if builder.is_basestring_const(expr):
+ # This *is* a docstring, remove it from stmt list
+ assert isinstance(expr, ast.Const)
+ del stmt.nodes[0]
+ doc = expr.value
+ return doc
+
+
+def to_lvalue(ast_node, flags):
+ lineno = ast_node.lineno
+ if isinstance( ast_node, ast.Name ):
+ return ast.AssName(ast_node.varname, flags, lineno)
+ # return ast.AssName(ast_node.name, flags)
+ elif isinstance(ast_node, ast.Tuple):
+ nodes = []
+ # FIXME: should ast_node.getChildren() but it's not annotable
+ # because of flatten()
+ for node in ast_node.nodes:
+ nodes.append(to_lvalue(node, flags))
+ return ast.AssTuple(nodes, lineno)
+ elif isinstance(ast_node, ast.List):
+ nodes = []
+ # FIXME: should ast_node.getChildren() but it's not annotable
+ # because of flatten()
+ for node in ast_node.nodes:
+ nodes.append(to_lvalue(node, flags))
+ return ast.AssList(nodes, lineno)
+ elif isinstance(ast_node, ast.Getattr):
+ expr = ast_node.expr
+ assert isinstance(ast_node, ast.Getattr)
+ attrname = ast_node.attrname
+ return ast.AssAttr(expr, attrname, flags, lineno)
+ elif isinstance(ast_node, ast.Subscript):
+ ast_node.flags = flags
+ return ast_node
+ elif isinstance(ast_node, ast.Slice):
+ ast_node.flags = flags
+ return ast_node
+ else:
+ if isinstance(ast_node, ast.GenExpr):
+ raise SyntaxError("assign to generator expression not possible",
+ lineno, 0, '')
+ elif isinstance(ast_node, ast.ListComp):
+ raise SyntaxError("can't assign to list comprehension",
+ lineno, 0, '')
+ elif isinstance(ast_node, ast.CallFunc):
+ if flags == consts.OP_DELETE:
+ raise SyntaxError("can't delete function call",
+ lineno, 0, '')
+ else:
+ raise SyntaxError("can't assign to function call",
+ lineno, 0, '')
+ else:
+ raise SyntaxError("can't assign to non-lvalue",
+ lineno, 0, '')
+
+def is_augassign( ast_node ):
+ if ( isinstance( ast_node, ast.Name ) or
+ isinstance( ast_node, ast.Slice ) or
+ isinstance( ast_node, ast.Subscript ) or
+ isinstance( ast_node, ast.Getattr ) ):
+ return True
+ return False
+
+def get_atoms(builder, nb):
+ atoms = []
+ i = nb
+ while i>0:
+ obj = builder.pop()
+ if isinstance(obj, BaseRuleObject):
+ i += obj.count
+ else:
+ atoms.append( obj )
+ i -= 1
+ atoms.reverse()
+ return atoms
+
+#def eval_string(value):
+# """temporary implementation
+#
+# FIXME: need to be finished (check compile.c (parsestr) and
+# stringobject.c (PyString_DecodeEscape()) for complete implementation)
+# """
+# # return eval(value)
+# if len(value) == 2:
+# return ''
+# result = ''
+# length = len(value)
+# quotetype = value[0]
+# index = 1
+# while index < length and value[index] == quotetype:
+# index += 1
+# if index == 6:
+# # empty strings like """""" or ''''''
+# return ''
+# # XXX: is it RPYTHON to do this value[index:-index]
+# chars = [char for char in value[index:len(value)-index]]
+# result = ''.join(chars)
+# result = result.replace('\\\\', '\\')
+# d = {'\\b' : '\b', '\\f' : '\f', '\\t' : '\t', '\\n' : '\n',
+# '\\r' : '\r', '\\v' : '\v', '\\a' : '\a',
+# }
+# for escaped, value in d.items():
+# result = result.replace(escaped, value)
+# return result
+
+
+## misc utilities, especially for power: rule
+def reduce_callfunc(obj, arglist):
+ """generic factory for CallFunc nodes"""
+ assert isinstance(arglist, ArglistObject)
+ return ast.CallFunc(obj, arglist.arguments,
+ arglist.stararg, arglist.dstararg, arglist.lineno)
+
+def reduce_subscript(obj, subscript):
+ """generic factory for Subscript nodes"""
+ assert isinstance(subscript, SubscriptObject)
+ return ast.Subscript(obj, consts.OP_APPLY, subscript.value, subscript.lineno)
+
+def reduce_slice(obj, sliceobj):
+ """generic factory for Slice nodes"""
+ assert isinstance(sliceobj, SlicelistObject)
+ if sliceobj.fake_rulename == 'slice':
+ start = sliceobj.value[0]
+ end = sliceobj.value[1]
+ return ast.Slice(obj, consts.OP_APPLY, start, end, sliceobj.lineno)
+ else:
+ return ast.Subscript(obj, consts.OP_APPLY, ast.Sliceobj(sliceobj.value,
+ sliceobj.lineno), sliceobj.lineno)
+
+def parse_attraccess(tokens, builder):
+ """parses token list like ['a', '.', 'b', '.', 'c', ...]
+
+ and returns an ast node : ast.Getattr(Getattr(Name('a'), 'b'), 'c' ...)
+ """
+ token = tokens[0]
+ # XXX HACK for when parse_attraccess is called from build_decorator
+ if isinstance(token, TokenObject):
+ val = token.get_value()
+ result = ast.Name(val, token.lineno)
+ else:
+ result = token
+ index = 1
+ while index < len(tokens):
+ token = tokens[index]
+ if isinstance(token, TokenObject) and token.name == builder.parser.tokens['DOT']:
+ index += 1
+ token = tokens[index]
+ assert isinstance(token, TokenObject)
+ result = ast.Getattr(result, token.get_value(), token.lineno)
+ elif isinstance(token, ArglistObject):
+ result = reduce_callfunc(result, token)
+ elif isinstance(token, SubscriptObject):
+ result = reduce_subscript(result, token)
+ elif isinstance(token, SlicelistObject):
+ result = reduce_slice(result, token)
+ else:
+ assert False, "Don't know how to handle index %s of %s" % (index, len(tokens))
+ index += 1
+ return result
+
+
+
+## Stack elements definitions ###################################
+
+class BaseRuleObject(ast.Node):
+ """Base class for unnamed rules"""
+ def __init__(self, count, lineno):
+ self.count = count
+ self.lineno = lineno # src.getline()
+ self.col = 0 # src.getcol()
+
+
+class RuleObject(BaseRuleObject):
+ """A simple object used to wrap a rule or token"""
+ def __init__(self, name, count, lineno, parser):
+ BaseRuleObject.__init__(self, count, lineno)
+ self.rulename = name
+ self.parser = parser
+
+ def __str__(self):
+ return "<Rule: %s/%d>" % ( self.parser.symbol_repr(self.rulename), self.count)
+
+ def __repr__(self):
+ return "<Rule: %s/%d>" % ( self.parser.symbol_repr(self.rulename), self.count)
+
+
+class TempRuleObject(BaseRuleObject):
+ """used to keep track of how many items get_atom() should pop"""
+ def __init__(self, name, count, lineno):
+ BaseRuleObject.__init__(self, count, lineno)
+ self.temp_rulename = name
+
+ def __str__(self):
+ return "<Rule: %s/%d>" % (self.temp_rulename, self.count)
+
+ def __repr__(self):
+ return "<Rule: %s/%d>" % (self.temp_rulename, self.count)
+
+
+class TokenObject(ast.Node):
+ """A simple object used to wrap a rule or token"""
+ def __init__(self, name, value, lineno, parser):
+ self.name = name
+ self.value = value
+ self.count = 0
+ # self.line = 0 # src.getline()
+ self.col = 0 # src.getcol()
+ self.lineno = lineno
+ self.parser = parser
+
+ def get_name(self):
+ tokname = self.parser.tok_name.get(self.name, str(self.name))
+ return self.parser.tok_rvalues.get(self.name, tokname)
+
+ def get_value(self):
+ value = self.value
+ if value is None:
+ value = ''
+ return value
+
+ def descr_fget_value(space, self):
+ value = self.get_value()
+ return space.wrap(value)
+
+ def __str__(self):
+ return "<Token: (%s,%s)>" % (self.get_name(), self.value)
+
+ def __repr__(self):
+ return "<Token: (%r,%s)>" % (self.get_name(), self.value)
+
+TokenObject.typedef = TypeDef('BuildToken',
+ name=interp_attrproperty('name', cls=TokenObject),
+ lineno=interp_attrproperty('lineno', cls=TokenObject),
+ value=GetSetProperty(TokenObject.descr_fget_value))
+
+class ObjectAccessor(ast.Node):
+ """base class for ArglistObject, SubscriptObject and SlicelistObject
+
+ FIXME: think about a more appropriate name
+ """
+
+class ArglistObject(ObjectAccessor):
+ """helper class to build function's arg list
+ """
+ def __init__(self, arguments, stararg, dstararg, lineno):
+ self.fake_rulename = 'arglist'
+ self.arguments = arguments
+ self.stararg = stararg
+ self.dstararg = dstararg
+ self.lineno = lineno
+
+ def __str__(self):
+ return "<ArgList: (%s, %s, %s)>" % self.value
+
+ def __repr__(self):
+ return "<ArgList: (%s, %s, %s)>" % self.value
+
+class SubscriptObject(ObjectAccessor):
+ """helper class to build subscript list
+
+ self.value represents the __getitem__ argument
+ """
+ def __init__(self, name, value, lineno):
+ self.fake_rulename = name
+ self.value = value
+ self.lineno = lineno
+
+ def __str__(self):
+ return "<SubscriptList: (%s)>" % self.value
+
+ def __repr__(self):
+ return "<SubscriptList: (%s)>" % self.value
+
+class SlicelistObject(ObjectAccessor):
+ """helper class to build slice objects
+
+ self.value is a list [start, end, step]
+ self.fake_rulename can either be 'slice' or 'sliceobj' depending
+ on if a step is specfied or not (see Python's AST
+ for more information on that)
+ """
+ def __init__(self, name, value, lineno):
+ self.fake_rulename = name
+ self.value = value
+ self.lineno = lineno
+
+ def __str__(self):
+ return "<SliceList: (%s)>" % self.value
+
+ def __repr__(self):
+ return "<SliceList: (%s)>" % self.value
+
Modified: pypy/dist/pypy/interpreter/pyparser/ebnfgrammar.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/ebnfgrammar.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/ebnfgrammar.py Wed Feb 28 18:30:48 2007
@@ -1,47 +1,18 @@
# This module contains the grammar parser
# and the symbol mappings
-from grammar import BaseGrammarBuilder, Alternative, Sequence, Token, \
- KleeneStar, GrammarElement, build_first_sets, EmptyToken
+from grammar import Alternative, Sequence, Token, KleeneStar, \
+ GrammarElement, Parser
+class GrammarParser(Parser):
+ pass
-sym_map = {}
-sym_rmap = {}
-_count = 0
-
-def g_add_symbol( name ):
- global _count
- if name in sym_rmap:
- return sym_rmap[name]
- val = _count
- _count += 1
- sym_map[val] = name
- sym_rmap[name] = val
- return val
-
-
-tok_map = {}
-tok_rmap = {}
-
-def g_add_token( **kwargs ):
- global _count
- assert len(kwargs) == 1
- sym, name = kwargs.popitem()
- if name in tok_rmap:
- return tok_rmap[name]
- val = _count
- _count += 1
- tok_map[val] = name
- tok_rmap[name] = val
- sym_map[val] = sym
- sym_rmap[sym] = val
- return val
-
-g_add_token( EOF='EOF' )
+GRAMMAR_GRAMMAR = GrammarParser()
def grammar_grammar():
- """NOT RPYTHON (mostly because of g_add_token I suppose)
+ """
+ (mostly because of g_add_token I suppose)
Builds the grammar for the grammar file
Here's the description of the grammar's grammar ::
@@ -51,59 +22,56 @@
alternative: sequence ( '|' sequence )+
star: '*' | '+'
- sequence: (SYMBOL star? | STRING | option | group star? )+
+ sequence: (SYMBOL star? | STRING | option | group )+
option: '[' alternative ']'
group: '(' alternative ')' star?
"""
- global sym_map
- S = g_add_symbol
- T = g_add_token
+ p = GRAMMAR_GRAMMAR
+ p.add_token('EOF','EOF')
+
# star: '*' | '+'
- star = Alternative( S("star"), [Token(T(TOK_STAR='*')), Token(T(TOK_ADD='+'))] )
- star_opt = KleeneStar ( S("star_opt"), 0, 1, rule=star )
+ star = p.Alternative_n( "star", [p.Token_n('TOK_STAR', '*'), p.Token_n('TOK_ADD', '+')] )
+ star_opt = p.KleeneStar_n( "star_opt", 0, 1, rule=star )
# rule: SYMBOL ':' alternative
- symbol = Sequence( S("symbol"), [Token(T(TOK_SYMBOL='SYMBOL')), star_opt] )
- symboldef = Token( T(TOK_SYMDEF="SYMDEF") )
- alternative = Sequence( S("alternative"), [])
- rule = Sequence( S("rule"), [symboldef, alternative] )
+ symbol = p.Sequence_n( "symbol", [p.Token_n('TOK_SYMBOL'), star_opt] )
+ symboldef = p.Token_n( 'TOK_SYMDEF' )
+ alternative = p.Sequence_n( "alternative", [])
+ rule = p.Sequence_n( "rule", [symboldef, alternative] )
# grammar: rule+
- grammar = KleeneStar( S("grammar"), _min=1, rule=rule )
+ grammar = p.KleeneStar_n( "grammar", _min=1, rule=rule )
# alternative: sequence ( '|' sequence )*
- sequence = KleeneStar( S("sequence"), 1 )
- seq_cont_list = Sequence( S("seq_cont_list"), [Token(T(TOK_BAR='|')), sequence] )
- sequence_cont = KleeneStar( S("sequence_cont"),0, rule=seq_cont_list )
-
+ sequence = p.KleeneStar_n( "sequence", 1 )
+ seq_cont_list = p.Sequence_n( "seq_cont_list", [p.Token_n('TOK_BAR', '|'), sequence] )
+ sequence_cont = p.KleeneStar_n( "sequence_cont",0, rule=seq_cont_list )
+
alternative.args = [ sequence, sequence_cont ]
# option: '[' alternative ']'
- option = Sequence( S("option"), [Token(T(TOK_LBRACKET='[')), alternative, Token(T(TOK_RBRACKET=']'))] )
+ option = p.Sequence_n( "option", [p.Token_n('TOK_LBRACKET', '['), alternative, p.Token_n('TOK_RBRACKET', ']')] )
# group: '(' alternative ')'
- group = Sequence( S("group"), [Token(T(TOK_LPAR='(')), alternative, Token(T(TOK_RPAR=')')), star_opt] )
+ group = p.Sequence_n( "group", [p.Token_n('TOK_LPAR', '('), alternative, p.Token_n('TOK_RPAR', ')'), star_opt] )
# sequence: (SYMBOL | STRING | option | group )+
- string = Token(T(TOK_STRING='STRING'))
- alt = Alternative( S("sequence_alt"), [symbol, string, option, group] )
+ string = p.Token_n('TOK_STRING')
+ alt = p.Alternative_n( "sequence_alt", [symbol, string, option, group] )
sequence.args = [ alt ]
+ p.root_rules['grammar'] = grammar
+ p.build_first_sets()
+ return p
- rules = [ star, star_opt, symbol, alternative, rule, grammar, sequence,
- seq_cont_list, sequence_cont, option, group, alt ]
- build_first_sets( rules )
- return grammar
-
-
-GRAMMAR_GRAMMAR = grammar_grammar()
-for _sym, _value in sym_rmap.items():
- globals()[_sym] = _value
+grammar_grammar()
+for _sym, _value in GRAMMAR_GRAMMAR.symbols.items():
+ assert not hasattr( GRAMMAR_GRAMMAR, _sym ), _sym
+ setattr(GRAMMAR_GRAMMAR, _sym, _value )
+
+for _sym, _value in GRAMMAR_GRAMMAR.tokens.items():
+ assert not hasattr( GRAMMAR_GRAMMAR, _sym )
+ setattr(GRAMMAR_GRAMMAR, _sym, _value )
-# cleanup
-del _sym
-del _value
del grammar_grammar
-del g_add_symbol
-del g_add_token
Modified: pypy/dist/pypy/interpreter/pyparser/ebnflexer.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/ebnflexer.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/ebnflexer.py Wed Feb 28 18:30:48 2007
@@ -3,8 +3,8 @@
analyser in grammar.py
"""
-from grammar import TokenSource, Token
-from ebnfgrammar import *
+from grammar import TokenSource, Token, AbstractContext
+from ebnfgrammar import GRAMMAR_GRAMMAR as G
def match_symbol( input, start, stop ):
@@ -15,6 +15,12 @@
idx+=1
return idx
+
+class GrammarSourceContext(AbstractContext):
+ def __init__(self, pos, peek):
+ self.pos = pos
+ self.peek = peek
+
class GrammarSource(TokenSource):
"""Fully RPython - see targetebnflexer.py
The grammar tokenizer
@@ -25,8 +31,9 @@
SYMBOL: a rule symbol usually appearing right of a SYMDEF
tokens: '[', ']', '(' ,')', '*', '+', '|'
"""
- def __init__(self, inpstring ):
- TokenSource.__init__(self)
+ def __init__(self, parser, inpstring):
+ # TokenSource.__init__(self)
+ self.parser = parser
self.input = inpstring
self.pos = 0
self.begin = 0
@@ -36,7 +43,7 @@
def context(self):
"""returns an opaque context object, used to backtrack
to a well known position in the parser"""
- return self.pos, self._peeked
+ return GrammarSourceContext( self.pos, self._peeked )
def offset(self, ctx=None):
"""Returns the current parsing position from the start
@@ -44,14 +51,16 @@
if ctx is None:
return self.pos
else:
- assert type(ctx)==int
- return ctx
+ assert isinstance(ctx, GrammarSourceContext)
+ return ctx.pos
def restore(self, ctx):
"""restore the context provided by context()"""
- self.pos, self._peeked = ctx
+ assert isinstance( ctx, GrammarSourceContext )
+ self.pos = ctx.pos
+ self._peeked = ctx.peek
- def current_line(self):
+ def current_linesource(self):
pos = idx = self.begin
inp = self.input
end = len(inp)
@@ -65,7 +74,6 @@
def current_lineno(self):
return self.current_line
-
def skip_empty_lines(self, input, start, end ):
idx = start
# assume beginning of a line
@@ -117,17 +125,18 @@
# means backtracking more than one token
# will re-tokenize the stream (but this is the
# grammar lexer so we don't care really!)
+ _p = self.parser
if self._peeked is not None:
peeked = self._peeked
self._peeked = None
return peeked
-
+
pos = self.pos
inp = self.input
end = len(self.input)
pos = self.skip_empty_lines(inp,pos,end)
if pos==end:
- return Token(EOF, None)
+ return _p.build_token( _p.EOF, None)
# at this point nextchar is not a white space nor \n
nextchr = inp[pos]
@@ -139,22 +148,22 @@
self.pos = npos
_endpos = npos - 1
assert _endpos>=0
- return Token(TOK_STRING,inp[pos+1:_endpos])
+ return _p.build_token( _p.TOK_STRING, inp[pos+1:_endpos])
else:
npos = match_symbol( inp, pos, end)
if npos!=pos:
self.pos = npos
if npos!=end and inp[npos]==":":
self.pos += 1
- return Token(TOK_SYMDEF,inp[pos:npos])
+ return _p.build_token( _p.TOK_SYMDEF, inp[pos:npos])
else:
- return Token(TOK_SYMBOL,inp[pos:npos])
-
+ return _p.build_token( _p.TOK_SYMBOL, inp[pos:npos])
+
# we still have pos!=end here
chr = inp[pos]
if chr in "[]()*+|":
self.pos = pos+1
- return Token(tok_rmap[chr], chr)
+ return _p.build_token( _p.tok_values[chr], chr)
self.RaiseError( "Unknown token" )
def peek(self):
Modified: pypy/dist/pypy/interpreter/pyparser/ebnfparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/ebnfparse.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/ebnfparse.py Wed Feb 28 18:30:48 2007
@@ -1,15 +1,31 @@
-#!/usr/bin/env python
-from grammar import BaseGrammarBuilder, Alternative, Sequence, Token, \
- KleeneStar, GrammarElement, build_first_sets, EmptyToken
-from ebnflexer import GrammarSource
-import ebnfgrammar
-from ebnfgrammar import GRAMMAR_GRAMMAR, sym_map
-from syntaxtree import AbstractSyntaxVisitor
-import pytoken
-import pysymbol
+from grammar import Token, GrammarProxy
+from grammar import AbstractBuilder, AbstractContext
+
+
+ORDA = ord("A")
+ORDZ = ord(&quo