diff -Nru jython/org/python/compiler/CodeCompiler.java jython_diff/org/python/compiler/CodeCompiler.java --- jython/org/python/compiler/CodeCompiler.java 2005-10-21 14:18:00.000000000 -0700 +++ jython_diff/org/python/compiler/CodeCompiler.java 2006-03-05 19:48:04.235443200 -0800 @@ -45,6 +45,7 @@ import org.python.parser.ast.Lambda; import org.python.parser.ast.List; import org.python.parser.ast.ListComp; +import org.python.parser.ast.GenExpr; import org.python.parser.ast.Name; import org.python.parser.ast.Num; import org.python.parser.ast.Pass; @@ -68,6 +69,7 @@ import org.python.parser.ast.expr_contextType; import org.python.parser.ast.keywordType; import org.python.parser.ast.listcompType; +import org.python.parser.ast.genexprType; import org.python.parser.ast.modType; import org.python.parser.ast.stmtType; @@ -2022,6 +2024,78 @@ return null; } + public Object visitGenExpr(GenExpr node) throws Exception { + String name=""; + setline(node); + + code.new_(code.pool.Class("org/python/core/PyFunction")); + code.dup(); + loadFrame(); + if (mrefs.f_globals == 0) { + mrefs.f_globals = code.pool.Fieldref( + "org/python/core/PyFrame", "f_globals", $pyObj); + } + code.getfield(mrefs.f_globals); + + ScopeInfo scope = module.getScopeInfo(node); + + makeArray(scope.ac.getDefaults()); + + scope.setup_closure(my_scope); + scope.dump(); + + stmtType stmts[]=new stmtType[1]; + stmtType n = new Yield(node.target , node); + + for (int i = node.generators.length - 1; i >= 0; i--) { + genexprType ge = node.generators[i]; + for (int j = ge.ifs.length - 1; j >= 0; j--) { + n = new If(ge.ifs[j], new stmtType[] { n }, null, ge.ifs[j]); + } + if (i==0) + n = new For(ge.target, new Name("",Name.Load), + new stmtType[] { n }, null, ge); + else + n = new For(ge.target, ge.iter, new stmtType[] { n }, null, ge); + } + stmts[0]=n; + + module.PyCode(new Suite(stmts, node), name, true, + className, false, false, + node.beginLine, scope, cflags).get(code); + + Vector freenames = scope.freevars; + + getDocString(stmts); + + if (!makeClosure(freenames)) { + if (mrefs.PyFunction_init == 0) { + mrefs.PyFunction_init = code.pool.Methodref( + "org/python/core/PyFunction", "", + "(" + $pyObj + $pyObjArr + $pyCode + $pyObj + ")V"); + } + code.invokespecial(mrefs.PyFunction_init); + } else { + if (mrefs.PyFunction_closure_init == 0) { + mrefs.PyFunction_closure_init = code.pool.Methodref( + "org/python/core/PyFunction", "", + "(" + $pyObj + $pyObjArr + $pyCode + $pyObj + $pyObjArr + + ")V"); + } + code.invokespecial(mrefs.PyFunction_closure_init); + } + + if (mrefs.calla1 == 0) { + mrefs.calla1 = code.pool.Methodref( + "org/python/core/PyObject", "__call__", + "(" + $pyObj + ")" + $pyObj); + } + visit(node.generators[0].iter); // visit precomputed iterable + code.invokevirtual(mrefs.calla1); + + return null; + } + public Object visitDict(Dict node) throws Exception { code.new_(code.pool.Class("org/python/core/PyDictionary")); code.dup(); diff -Nru jython/org/python/compiler/ScopesCompiler.java jython_diff/org/python/compiler/ScopesCompiler.java --- jython/org/python/compiler/ScopesCompiler.java 2002-10-15 07:32:28.000000000 -0700 +++ jython_diff/org/python/compiler/ScopesCompiler.java 2006-03-05 20:00:35.365515200 -0800 @@ -4,6 +4,7 @@ import org.python.parser.*; import org.python.parser.ast.*; + import java.util.*; public class ScopesCompiler extends Visitor implements ScopeConstants { @@ -11,37 +12,36 @@ private CompilationContext code_compiler; private Stack scopes; + private ScopeInfo cur = null; + private Hashtable nodeScopes; private int level = 0; private int func_level = 0; - public ScopesCompiler(CompilationContext code_compiler, - Hashtable nodeScopes) - { + public ScopesCompiler(CompilationContext code_compiler, Hashtable nodeScopes) { this.code_compiler = code_compiler; this.nodeScopes = nodeScopes; scopes = new Stack(); } public void beginScope(String name, int kind, SimpleNode node, - ArgListCompiler ac) - { + ArgListCompiler ac) { if (cur != null) { scopes.push(cur); } - if (kind == FUNCSCOPE) func_level++; - cur = new ScopeInfo(name, node, level++, kind, - func_level, ac); + if (kind == FUNCSCOPE) + func_level++; + cur = new ScopeInfo(name, node, level++, kind, func_level, ac); nodeScopes.put(node, cur); } public void endScope() throws Exception { if (cur.kind == FUNCSCOPE) func_level--; level--; - ScopeInfo up = (!scopes.empty())?(ScopeInfo)scopes.pop():null; - cur.cook(up,code_compiler); + ScopeInfo up = (!scopes.empty()) ? (ScopeInfo) scopes.pop() : null; + cur.cook(up, code_compiler); cur.dump(); // dbg cur = up; } @@ -49,7 +49,7 @@ public void parse(SimpleNode node) throws Exception { try { visit(node); - } catch(Throwable t) { + } catch (Throwable t) { throw org.python.core.parser.fixParseError(null, t, code_compiler.getFilename()); } @@ -63,8 +63,7 @@ } public Object visitModule(org.python.parser.ast.Module node) - throws Exception - { + throws Exception { beginScope("", TOPSCOPE, node, null); suite(node.body); endScope(); @@ -120,7 +119,7 @@ beginScope("", FUNCSCOPE, node, ac); int n = ac.names.size(); for (int i = 0; i < n; i++) { - cur.addParam((String)ac.names.elementAt(i)); + cur.addParam((String) ac.names.elementAt(i)); } for (int i = 0; i < ac.init_code.size(); i++) visit((stmtType) ac.init_code.elementAt(i)); @@ -173,14 +172,14 @@ String name = node.names[i]; int prev = cur.addGlobal(name); if (prev >= 0) { - if ((prev&FROM_PARAM) != 0) - code_compiler.error("name '"+name+"' is local and global", - true,node); - if ((prev&GLOBAL) != 0) continue; + if ((prev & FROM_PARAM) != 0) + code_compiler.error("name '" + name + "' is local and global", + true, node); + if ((prev & GLOBAL) != 0) continue; String what; - if ((prev&BOUND) != 0) what = "assignment"; else what = "use"; - code_compiler.error("name '"+name+"' declared global after "+ - what,false,node); + if ((prev & BOUND) != 0) what = "assignment"; else what = "use"; + code_compiler.error("name '" + name + "' declared global after "+ + what, false, node); } } return null; @@ -235,6 +234,56 @@ return null; } + public Object visitGenExpr(GenExpr node) throws Exception { + String name = "GenExpr";// ""; + def(name); + + // XXX FIXME: JIWON: need to capture first argument! + ArgListCompiler ac = new ArgListCompiler(); + + // added for pre-computation + exprType fpargs[]=new exprType[1]; + fpargs[0] = new Name("",Name.Store); + argumentsType args=new argumentsType(fpargs, null, null, new exprType[0]); + ac.visitArgs(args); + // ac.visitName(node.target); + // ac.visitArgs(node.args); + + // exprType[] defaults = ac.getDefaults(); + // int defc = defaults.length; + // for (int i = 0; i < defc; i++) { + // visit(defaults[i]); + // } + + beginScope(name, FUNCSCOPE, node, ac); + int size = ac.names.size(); + for (int i = 0; i < size; i++) { + cur.addParam((String) ac.names.elementAt(i)); + } + for (int i = 0; i < ac.init_code.size(); i++) { + visit((stmtType) ac.init_code.elementAt(i)); + } + cur.markFromParam(); + + stmtType n = new Yield(node.target, node); + + for (int i = node.generators.length - 1; i >= 0; i--) { + genexprType ge = node.generators[i]; + for (int j = ge.ifs.length - 1; j >= 0; j--) { + n = new If(ge.ifs[j], new stmtType[] { n }, null, ge.ifs[j]); + } + if (i==0) + n = new For(ge.target, new Name("",Name.Load), + new stmtType[] { n }, null, ge); + else + n = new For(ge.target, ge.iter, new stmtType[] { n }, null, ge); + } + + suite(new stmtType[] { n }); + + endScope(); + return null; + } public Object visitYield(Yield node) throws Exception { cur.generator = true; diff -Nru jython/org/python/parser/TreeBuilder.java jython_diff/org/python/parser/TreeBuilder.java --- jython/org/python/parser/TreeBuilder.java 2005-07-20 13:25:09.000000000 -0700 +++ jython_diff/org/python/parser/TreeBuilder.java 2006-03-05 20:16:07.515881600 -0800 @@ -1,5 +1,7 @@ package org.python.parser; +import java.util.EmptyStackException; + import org.python.parser.ast.*; import org.python.core.PyObject; @@ -39,7 +41,7 @@ } private exprType makeExpr(SimpleNode node) { - return (exprType) node; + return (exprType)node; } private exprType makeExpr() { @@ -189,7 +191,7 @@ String name = makeIdentifier(); return new FunctionDef(name, arguments, body); case JJTDEFAULTARG: - value = (arity == 1) ? null : makeExpr(); + value = (arity == 1) ? null : makeExpr(); return new DefaultArg(makeExpr(), value); case JJTEXTRAARGLIST: return new ExtraArg(makeIdentifier(), JJTEXTRAARGLIST); @@ -261,7 +263,7 @@ int[] ops = new int[l]; for (int i = l-1; i >= 0; i--) { comparators[i] = makeExpr(); - SimpleNode op = (SimpleNode) stack.popNode(); + SimpleNode op = (SimpleNode) stack.popNode(); switch (op.getId()) { case JJTLESS_CMP: ops[i] = Compare.Lt; break; case JJTGREATER_CMP: ops[i] = Compare.Gt; break; @@ -277,7 +279,8 @@ throw new RuntimeException("Unknown cmp op:" + op.getId()); } } - return new Compare(makeExpr(), ops, comparators); + return new Compare(makeExpr(), ops, comparators); + case JJTLESS_CMP: case JJTGREATER_CMP: case JJTEQUAL_CMP: @@ -370,6 +373,14 @@ name = makeIdentifier(); return new keywordType(name, value); case JJTTUPLE: + if (stack.nodeArity() > 0 && peekNode() instanceof genexprType) { + System.out.println("JJTUPLE as genexpr"); + genexprType[] generators = new genexprType[arity-1]; + for (int i = arity-2; i >= 0; i--) { + generators[i] = (genexprType) popNode(); + } + return new GenExpr(makeExpr(), generators); + } return new Tuple(makeExprs(), Tuple.Load); case JJTLIST: if (stack.nodeArity() > 0 && peekNode() instanceof listcompType) { @@ -459,6 +470,15 @@ target = makeExpr(); ctx.setStore(target); return new listcompType(target, iter, ifs); + case JJTGEN_FOR: + ifs = new exprType[arity-2]; + for (int i = arity-3; i >= 0; i--) { + ifs[i] = makeExpr(); + } + iter = makeExpr(); + target = makeExpr(); + ctx.setStore(target); + return new genexprType(target, iter, ifs); case JJTIMPORTFROM: aliasType[] aliases = makeAliases(arity - 1); String module = makeIdentifier(); @@ -549,7 +569,7 @@ exprType[] newdefs = new exprType[l-startofdefaults]; System.arraycopy(defaults, startofdefaults, newdefs, 0, newdefs.length); - return new argumentsType(fpargs, stararg, kwarg, newdefs); + return new argumentsType(fpargs, stararg, kwarg, newdefs); } } diff -Nru jython/org/python/parser/ast/GenExpr.java jython_diff/org/python/parser/ast/GenExpr.java --- jython/org/python/parser/ast/GenExpr.java 1969-12-31 16:00:00.000000000 -0800 +++ jython_diff/org/python/parser/ast/GenExpr.java 2006-02-21 12:56:07.755046400 -0800 @@ -0,0 +1,55 @@ +// Autogenerated AST node +package org.python.parser.ast; +import org.python.parser.SimpleNode; +import java.io.DataOutputStream; +import java.io.IOException; + +public class GenExpr extends SimpleNode { + public exprType target; + public genexprType[] generators; + + public GenExpr(exprType target, genexprType[] generators) { + this.target = target; + this.generators = generators; + } + + public GenExpr(exprType target, genexprType[] generators, SimpleNode + parent) { + this(target, generators); + this.beginLine = parent.beginLine; + this.beginColumn = parent.beginColumn; + } + + public String toString() { + StringBuffer sb = new StringBuffer("GenExpr["); + sb.append("target="); + sb.append(dumpThis(this.target)); + sb.append(", "); + sb.append("generators="); + sb.append(dumpThis(this.generators)); + sb.append("]"); + return sb.toString(); + } + + public void pickle(DataOutputStream ostream) throws IOException { + pickleThis(34, ostream); + pickleThis(this.target, ostream); + pickleThis(this.generators, ostream); + } + + public Object accept(VisitorIF visitor) throws Exception { + return visitor.visitGenExpr(this); + } + + public void traverse(VisitorIF visitor) throws Exception { + if (target != null) + target.accept(visitor); + if (generators != null) { + for (int i = 0; i < generators.length; i++) { + if (generators[i] != null) + generators[i].accept(visitor); + } + } + } + +} diff -Nru jython/org/python/parser/ast/VisitorBase.java jython_diff/org/python/parser/ast/VisitorBase.java --- jython/org/python/parser/ast/VisitorBase.java 2005-07-20 13:25:09.000000000 -0700 +++ jython_diff/org/python/parser/ast/VisitorBase.java 2005-12-16 18:49:19.530555200 -0800 @@ -201,6 +201,12 @@ return ret; } + public Object visitGenExpr(GenExpr node) throws Exception { + Object ret = unhandled_node(node); + traverse(node); + return ret; + } + public Object visitCompare(Compare node) throws Exception { Object ret = unhandled_node(node); traverse(node); diff -Nru jython/org/python/parser/ast/VisitorIF.java jython_diff/org/python/parser/ast/VisitorIF.java --- jython/org/python/parser/ast/VisitorIF.java 2005-07-20 13:25:09.000000000 -0700 +++ jython_diff/org/python/parser/ast/VisitorIF.java 2005-12-16 18:31:54.177411200 -0800 @@ -35,12 +35,13 @@ public Object visitLambda(Lambda node) throws Exception; public Object visitDict(Dict node) throws Exception; public Object visitListComp(ListComp node) throws Exception; + public Object visitGenExpr(GenExpr node) throws Exception; public Object visitCompare(Compare node) throws Exception; public Object visitCall(Call node) throws Exception; public Object visitRepr(Repr node) throws Exception; public Object visitNum(Num node) throws Exception; public Object visitStr(Str node) throws Exception; - public Object visitUnicode(Unicode node) throws Exception; + public Object visitUnicode(Unicode node) throws Exception; public Object visitAttribute(Attribute node) throws Exception; public Object visitSubscript(Subscript node) throws Exception; public Object visitName(Name node) throws Exception; diff -Nru jython/org/python/parser/ast/genexprType.java jython_diff/org/python/parser/ast/genexprType.java --- jython/org/python/parser/ast/genexprType.java 1969-12-31 16:00:00.000000000 -0800 +++ jython_diff/org/python/parser/ast/genexprType.java 2006-02-21 12:52:03.734161600 -0800 @@ -0,0 +1,65 @@ +// Autogenerated AST node +package org.python.parser.ast; +import org.python.parser.SimpleNode; +import java.io.DataOutputStream; +import java.io.IOException; + +//public class genexprType extends SimpleNode { +public class genexprType extends exprType { + public exprType target; + public exprType iter; + public exprType[] ifs; + + public genexprType(exprType target, exprType iter, exprType[] ifs) { + this.target = target; + this.iter = iter; + this.ifs = ifs; + } + + public genexprType(exprType target, exprType iter, exprType[] ifs, + SimpleNode parent) { + this(target, iter, ifs); + this.beginLine = parent.beginLine; + this.beginColumn = parent.beginColumn; + } + + public String toString() { + StringBuffer sb = new StringBuffer("genexpr["); + sb.append("target="); + sb.append(dumpThis(this.target)); + sb.append(", "); + sb.append("iter="); + sb.append(dumpThis(this.iter)); + sb.append(", "); + sb.append("ifs="); + sb.append(dumpThis(this.ifs)); + sb.append("]"); + return sb.toString(); + } + + public void pickle(DataOutputStream ostream) throws IOException { + pickleThis(50, ostream); + pickleThis(this.target, ostream); + pickleThis(this.iter, ostream); + pickleThis(this.ifs, ostream); + } + + public Object accept(VisitorIF visitor) throws Exception { + traverse(visitor); + return null; + } + + public void traverse(VisitorIF visitor) throws Exception { + if (target != null) + target.accept(visitor); + if (iter != null) + iter.accept(visitor); + if (ifs != null) { + for (int i = 0; i < ifs.length; i++) { + if (ifs[i] != null) + ifs[i].accept(visitor); + } + } + } + +} diff -Nru jython/org/python/parser/python.jjt jython_diff/org/python/parser/python.jjt --- jython/org/python/parser/python.jjt 2005-07-20 13:25:09.000000000 -0700 +++ jython_diff/org/python/parser/python.jjt 2006-02-26 15:03:59.137145600 -0800 @@ -1,5 +1,5 @@ // -*- java -*- -// Copyright © Corporation for National Research Initiatives +// Copyright ? Corpforation for National Research Initiatives options { @@ -914,7 +914,7 @@ void atom() #void: {} { LOOKAHEAD(2) ( ) #tuple -| ( [SmartTestList()] ) +| ( [SmartTestListGexp()] ) | ( [listmaker()] ) #list | ( [dictmaker()] ) #dictionary | "`" SmartTestList() "`" #str_1op(1) @@ -923,7 +923,6 @@ | String() (String() #strjoin(2))* } - //lambdef: 'lambda' [varargslist] ':' test void lambdef():{} { [varargslist()] test() } @@ -963,6 +962,10 @@ void dictmaker() #void: {} {test() test() (LOOKAHEAD(2) test() test())* []} +//testlist_gexp: test (',' test)* [','] +void SmartTestListGexp() #void: {} +{ test() ( (gen_for())+ | (LOOKAHEAD(2) test())* [Comma()] ) #tuple(>1) } + //listmaker: test ( list_for | (',' test)* [','] ) void listmaker() #void: {} { test() ( (list_for())+ | (LOOKAHEAD(2) test())* [Comma()] #tuple(>1)) } @@ -979,6 +982,13 @@ void list_if() #void: {} { test() } +//gen_for: 'for' exprlist 'in' testlist gen_iter +void gen_for(): {} +{ exprlist() SmartTestList() (gen_if())* } + +//gen_if: 'if' test list_iter +void gen_if() #void: {} +{ test() } //classdef: 'class' NAME ['(' testlist ')'] ':' suite void classdef(): {} @@ -1009,7 +1019,6 @@ void argument() #void: {} { ([LOOKAHEAD(2) AnyName() ] test()) #Keyword(>1) } - void Number() #Num : { Token t;