[pypy-svn] r40771 - in pypy/dist/pypy: interpreter/pyparser module/recparser

[adim at codespeak.net]

Author: adim
Date: Mon Mar 19 14:47:40 2007
New Revision: 40771

Modified:
   pypy/dist/pypy/interpreter/pyparser/pythonparse.py
   pypy/dist/pypy/module/recparser/pyparser.py
Log:
modified recparser's unwrap_syntax_tree in order to make modified-2.4.1/test/test_parser.py
pass again



Modified: pypy/dist/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonparse.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py	Mon Mar 19 14:47:40 2007
@@ -94,17 +94,15 @@
     def __init__(self): # , predefined_symbols=None):
         grammar.Parser.__init__(self)
         pytoken.setup_tokens(self)
+        # remember how many tokens were loaded
+        self._basetokens_count = self._sym_count
         # if predefined_symbols:
         #     self.load_symbols(predefined_symbols)
         self.keywords = []
-        
-        # XXX (adim): this is trunk's keyword management
-        # self.with_grammar = None
-        # self.keywords = dict.fromkeys(grammar_builder.keywords)
-        # # Only when with_statement is enabled
-        # self.keywords.pop('with', None)
-        # self.keywords.pop('as', None)
-        
+
+    def is_base_token(self, tokvalue):
+        return tokvalue < 0 or tokvalue >= self._basetokens_count
+            
     def parse_source(self, textsrc, mode, builder, flags=0):
         """Parse a python source according to goal"""
         goal = self.targets[mode]
@@ -126,7 +124,6 @@
 
 
     def parse_lines(self, lines, goal, builder, flags=0):
-        # XXX (adim): this is trunk's keyword management
         # builder.keywords = self.keywords.copy()
         # if flags & CO_FUTURE_WITH_STATEMENT:
         #     builder.enable_with()

Modified: pypy/dist/pypy/module/recparser/pyparser.py
==============================================================================
--- pypy/dist/pypy/module/recparser/pyparser.py	(original)
+++ pypy/dist/pypy/module/recparser/pyparser.py	Mon Mar 19 14:47:40 2007
@@ -183,11 +183,9 @@
 
 def unwrap_syntax_tree( space, w_sequence ):
     items = space.unpackiterable( w_sequence )
+    parser = space.default_compiler.parser
     nodetype = space.int_w( items[0] )
-    is_syntax = True
-    if nodetype>=0 and nodetype < pytoken.N_TOKENS:
-        is_syntax = False
-    if is_syntax:
+    if parser.is_base_token(nodetype):
         nodes = []
         for w_node in items[1:]:
             node = unwrap_syntax_tree( space, w_node )