[pypy-svn] r48908 - in pypy/branch/dist-future-fixing/pypy/interpreter: . pyparser pyparser/test
ac at codespeak.net
ac at codespeak.net
Wed Nov 21 17:05:03 CET 2007
Author: ac
Date: Wed Nov 21 17:05:03 2007
New Revision: 48908
Modified:
pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py
pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py
pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py
pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py
pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py
pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py
pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py
pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py
pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py
Log:
(jacob, arre)
__future__ imports and 'with'-statements work again. Yay!
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py Wed Nov 21 17:05:03 2007
@@ -2,6 +2,8 @@
General classes for bytecode compilers.
Compiler instances are stored into 'space.getexecutioncontext().compiler'.
"""
+
+import sys
from codeop import PyCF_DONT_IMPLY_DEDENT
from pypy.interpreter.error import OperationError
@@ -111,7 +113,7 @@
else:
return 0
-from pypy.interpreter.pyparser.future import futureFlags
+from pypy.interpreter.pyparser import future
class CPythonCompiler(PyCodeCompiler):
"""Faked implementation of a compiler, using the underlying compile()."""
@@ -119,8 +121,10 @@
def __init__(self, space):
self.space = space
self.w_compile_hook = space.w_None
-
- self.compiler_flags = futureFlags.allowed_flags
+ if sys.version_info >= (2.5):
+ self.compiler_flags = future.futureFlags_2_5.allowed_flags
+ else:
+ self.compiler_flags = future.futureFlags_2_4.allowed_flags
def compile(self, source, filename, mode, flags):
from pypy.tool import stdlib___future__
@@ -218,8 +222,11 @@
self.grammar_version = override_version or space.config.objspace.pyversion
self.parser = make_pyparser(self.grammar_version)
self.additional_rules = {}
-
- self.compiler_flags = futureFlags.allowed_flags
+ if self.grammar_version >= '2.5':
+ self.futureFlags = future.futureFlags_2_5
+ else:
+ self.futureFlags = future.futureFlags_2_4
+ self.compiler_flags = self.futureFlags.allowed_flags
def compile(self, source, filename, mode, flags):
from pyparser.error import SyntaxError
@@ -241,7 +248,7 @@
for rulename, buildfunc in self.additional_rules.iteritems():
assert isinstance(buildfunc, Function)
builder.user_build_rules[rulename] = buildfunc
- flags |= getFutures(source)
+ flags |= getFutures(self.futureFlags, source)
self.parser.parse_source(source, mode, builder, flags)
ast_tree = builder.rule_stack[-1]
encoding = builder.source_encoding
@@ -261,7 +268,7 @@
raise
try:
astcompiler.misc.set_filename(filename, ast_tree)
- flag_names = futureFlags.get_flag_names(space, flags)
+ flag_names = self.futureFlags.get_flag_names(space, flags)
if mode == 'exec':
codegenerator = ModuleCodeGenerator(space, ast_tree, flag_names)
elif mode == 'single':
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py Wed Nov 21 17:05:03 2007
@@ -869,13 +869,6 @@
names.append((name, as_name))
if index < l: # case ','
index += 1
-## if from_name == '__future__':
-## for name, asname in names:
-## if name == 'with_statement':
-## # found from __future__ import with_statement
-## if not builder.with_enabled:
-## builder.enable_with()
-## #raise pythonparse.AlternateGrammarException()
builder.push(ast.From(from_name, names, atoms[0].lineno))
@@ -1069,17 +1062,9 @@
self.rule_stack = []
self.space = space
self.source_encoding = None
-## self.with_enabled = False
self.build_rules = ASTRULES_Template
self.user_build_rules = {}
-## def enable_with(self):
-## if self.with_enabled:
-## return
-## self.with_enabled = True
-## # XXX
-## # self.keywords.update({'with':None, 'as': None})
-
def context(self):
return AstBuilderContext(self.rule_stack)
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py Wed Nov 21 17:05:03 2007
@@ -40,9 +40,6 @@
class NameToken(Token):
"""A token that is not a keyword"""
- def __init__(self, parser, keywords=None):
- Token.__init__(self, parser.tokens['NAME'])
- self.keywords = keywords
def match(self, source, builder, level=0):
"""Matches a token.
@@ -60,7 +57,7 @@
if tk.codename == self.codename:
# XXX (adim): this is trunk's keyword management
# if tk.value not in builder.keywords:
- if tk.value not in self.keywords:
+ if not tk.isKeyword:
ret = builder.token( tk.codename, tk.value, source )
return ret
source.restore( ctx )
@@ -78,7 +75,7 @@
return False
# XXX (adim): this is trunk's keyword management
# if other.value in builder.keywords:
- if other.value in self.keywords:
+ if other.isKeyword:
return False
return True
@@ -107,7 +104,7 @@
self.keywords = []
NAME = dest_parser.add_token(Token('NAME'))
# NAME = dest_parser.tokens['NAME']
- self.tokens[NAME] = NameToken(dest_parser, keywords=self.keywords)
+ self.tokens[NAME] = NameToken(NAME)
# XXX Temporary. We should be able to get rid of it later
self.parser = dest_parser
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py Wed Nov 21 17:05:03 2007
@@ -27,8 +27,8 @@
from pypy.interpreter.astcompiler.consts import CO_GENERATOR_ALLOWED, \
CO_FUTURE_DIVISION, CO_FUTURE_WITH_STATEMENT
-def getFutures(source):
- futures = FutureAutomaton(source)
+def getFutures(futureFlags, source):
+ futures = FutureAutomaton(futureFlags, source)
try:
futures.start()
except (IndexError, DoneException), e:
@@ -62,7 +62,8 @@
precede a future statement.
"""
- def __init__(self, string):
+ def __init__(self, futureFlags, string):
+ self.futureFlags = futureFlags
self.s = string
self.pos = 0
self.docstringConsumed = False
@@ -237,7 +238,7 @@
def setFlag(self, feature):
try:
- self.flags |= futureFlags.compiler_features[feature]
+ self.flags |= self.futureFlags.compiler_features[feature]
except IndexError:
pass
@@ -271,12 +272,5 @@
flag_names.append(name)
return flag_names
-# XXX This is a hack to deal with the fact that we currently are
-# using the Python 2.4.1 libraries even when running Python 2.5
-# and that we have a hacked __future__ module.
-from pypy.config.pypyoption import get_pypy_config
-config = get_pypy_config(translating=False)
-if config.objspace.pyversion == '2.4' and False:
- futureFlags = FutureFlags((2, 4, 4, 'final', 0))
-else:
- futureFlags = FutureFlags((2, 5, 0, 'final', 0))
+futureFlags_2_4 = FutureFlags((2, 4, 4, 'final', 0))
+futureFlags_2_5 = FutureFlags((2, 5, 0, 'final', 0))
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py Wed Nov 21 17:05:03 2007
@@ -655,6 +655,7 @@
class Token(GrammarElement):
"""Represents a Token in a grammar rule (a lexer token)"""
+ isKeyword = True
def __init__(self, codename, value=None):
GrammarElement.__init__(self, codename)
self.value = value
@@ -679,7 +680,7 @@
ctx = source.context()
tk = source.next()
- if tk.codename == self.codename:
+ if tk.codename == self.codename and tk.isKeyword:
if self.value is None:
ret = builder.token( tk.codename, tk.value, source )
return ret
@@ -713,7 +714,7 @@
# if (self.value is not None and builder.keywords is not None
# and self.value not in builder.keywords):
# return False
- res = other.codename == self.codename and self.value in [None, other.value]
+ res = other.isKeyword and other.codename == self.codename and self.value in [None, other.value]
#print "matching", self, other, res
return res
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py Wed Nov 21 17:05:03 2007
@@ -62,7 +62,7 @@
SyntaxError.__init__(self, msg, lineno, offset, line)
self.token_stack = token_stack
-def generate_tokens(parser, lines, flags):
+def generate_tokens(parser, lines, flags, keywords):
"""
This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since
the original function is not RPYTHON (uses yield)
@@ -251,6 +251,8 @@
last_comment = ''
elif initial in namechars: # ordinary name
tok = Token(parser.tokens['NAME'], token)
+ if token not in keywords:
+ tok.isKeyword = False
token_list.append((tok, line, lnum, pos))
last_comment = ''
elif initial == '\\': # continued stmt
@@ -312,12 +314,9 @@
class PythonSource(TokenSource):
"""This source uses Jonathan's tokenizer"""
- def __init__(self, parser, strings, flags=0):
- # TokenSource.__init__(self)
- #self.parser = parser
-
+ def __init__(self, parser, strings, keywords, flags=0):
self.input = strings
- tokens = generate_tokens(parser, strings, flags)
+ tokens = generate_tokens(parser, strings, flags, keywords)
self.token_stack = tokens
self._current_line = '' # the current line (as a string)
self._lineno = -1
@@ -393,14 +392,3 @@
Source = PythonSource
-def tokenize_file(filename):
- f = file(filename).read()
- src = Source(f)
- token = src.next()
- while token != ("ENDMARKER", None) and token != (None, None):
- print token
- token = src.next()
-
-if __name__ == '__main__':
- import sys
- tokenize_file(sys.argv[1])
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py Wed Nov 21 17:05:03 2007
@@ -10,7 +10,7 @@
from pypy.interpreter import gateway
from pypy.interpreter.pyparser.error import SyntaxError
from pypy.interpreter.pyparser.pythonlexer import Source, match_encoding_declaration
-#from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT
+from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT
# XXX seems dead
#import pypy.interpreter.pyparser.pysymbol as pysymbol
import pypy.interpreter.pyparser.pytoken as pytoken
@@ -124,12 +124,20 @@
def parse_lines(self, lines, goal, builder, flags=0):
- # builder.keywords = self.keywords.copy()
- # if flags & CO_FUTURE_WITH_STATEMENT:
- # builder.enable_with()
goalnumber = self.symbols[goal]
target = self.root_rules[goalnumber]
- src = Source(self, lines, flags)
+ keywords = {} # dict.fromkeys(self.keywords)
+ for keyword in self.keywords:
+ keywords[keyword] = None
+
+ if flags & CO_FUTURE_WITH_STATEMENT:
+ keywords.update({'with': None,
+ 'as': None })
+ else:
+ keywords.pop('with', None)
+ keywords.pop('as', None)
+ src = Source(self, lines, keywords, flags)
+
if not target.match(src, builder):
line, lineno = src.debug()
# XXX needs better error messages
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py Wed Nov 21 17:05:03 2007
@@ -3,7 +3,7 @@
from pypy.tool import stdlib___future__ as fut
def run(s):
- f = future.FutureAutomaton(s)
+ f = future.FutureAutomaton(future.futureFlags_2_5, s)
try:
f.start()
except IndexError, future.DoneException:
@@ -124,12 +124,12 @@
def test_full_chain():
s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n'
- flags = future.getFutures(s)
+ flags = future.getFutures(future.futureFlags_2_5, s)
assert flags == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED |
fut.CO_FUTURE_WITH_STATEMENT)
def test_intervening_code():
s = 'from __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n'
- flags = future.getFutures(s)
+ flags = future.getFutures(future.futureFlags_2_5, s)
assert flags & fut.CO_FUTURE_WITH_STATEMENT == 0
Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py (original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py Wed Nov 21 17:05:03 2007
@@ -18,7 +18,7 @@
def parse_source(source):
"""returns list of parsed tokens"""
- lexer = Source( P, source.splitlines(True))
+ lexer = Source( P, source.splitlines(True), {})
tokens = []
last_token = Token(NULLTOKEN, None)
while last_token.codename != ENDMARKER:
More information about the pypy-svn
mailing list