diff -r -u5 -P //c/jython/org/python/compiler/CodeCompiler.java //c/jy-exp/org/python/compiler/CodeCompiler.java --- //c/jython/org/python/compiler/CodeCompiler.java Mon Feb 19 14:44:24 2001 +++ //c/jy-exp/org/python/compiler/CodeCompiler.java Thu Mar 8 04:31:50 2001 @@ -1,10 +1,13 @@ // Copyright © Corporation for National Research Initiatives package org.python.compiler; import org.python.parser.*; +import org.python.core.Py; +import org.python.core.PyObject; +import org.python.core.PyException; import java.io.IOException; import java.util.Stack; import java.util.Hashtable; import java.util.Vector; @@ -28,12 +31,15 @@ int mode; int temporary; public boolean fast_locals, print_results; - public Hashtable locals; - public Hashtable globals; + + public Future futures; + public Hashtable tbl; + public ScopeInfo my_scope; + boolean optimizeGlobals = true; public Vector names; public String className; public Stack continueLabels, breakLabels, finallyLabels; @@ -158,40 +164,53 @@ if (count == 4) return; } + public void error(String msg,boolean err,SimpleNode node) throws Exception { + if (!err) { + try { + Py.warning(Py.SyntaxWarning,msg,(module.sfilename!=null)?module.sfilename:"?", + node.beginLine,null,Py.None); + return; + } catch(PyException e) { + if (!Py.matchException(e,Py.SyntaxWarning)) throw e; + } + } + throw new ParseException(msg,node); + } + public void parse(SimpleNode node, Code code, boolean fast_locals, String className, - boolean classBody, ArgListCompiler ac) + boolean classBody, ScopeInfo scope) throws Exception { this.fast_locals = fast_locals; this.className = className; this.code = code; - //Check out locals if fast_locals is on - LocalsCompiler lc = new LocalsCompiler(); - int n = ac.names.size(); - for (int i=0; i", "(Lorg/python/core/PyObject;[Lorg/python/core/PyObject;Lorg/python/core/PyCode;Lorg/python/core/PyObject;)V"); + } + code.invokespecial(mrefs.PyFunction_init); + } else { + if (mrefs.PyFunction_closure_init == 0) { + mrefs.PyFunction_closure_init = code.pool.Methodref( + "org/python/core/PyFunction", "", + "(Lorg/python/core/PyObject;[Lorg/python/core/PyObject;Lorg/python/core/PyCode;Lorg/python/core/PyObject;[Lorg/python/core/PyObject;)V"); + } + code.invokespecial(mrefs.PyFunction_closure_init); + } - code.invokespecial(mrefs.PyFunction_init); set(node.getChild(0)); return null; } @@ -625,10 +691,11 @@ public int importAll, importFrom; public Object ImportFrom(SimpleNode node) throws Exception { + Future.checkFromFuture(node); // future stmt support setline(node); String name = (String)node.getChild(0).visit(this); code.ldc(name); int n = node.getNumChildren(); if (n > 1) { @@ -674,15 +741,10 @@ } return name; } public Object global_stmt(SimpleNode node) throws Exception { - int n = node.getNumChildren(); - for (int i=0; i 1 && node.getChild(1).id == PythonGrammarTreeConstants.JJTFOR_STMT) return list_comprehension(node); @@ -2001,19 +2063,16 @@ code.invokevirtual("org/python/core/PyObject", "__repr__", "()Lorg/python/core/PyString;"); return null; } - public int PyFunction_init1; + public int PyFunction_init1,PyFunction_closure_init1; public Object lambdef(SimpleNode node) throws Exception { String name = ""; - ArgListCompiler ac = new ArgListCompiler(); SimpleNode suite; if (node.getNumChildren() == 2) { suite = node.getChild(1); - //Parse arguments - node.getChild(0).visit(ac); } else { suite = node.getChild(0); } //Add a return node onto the outside of suite; @@ -2031,21 +2090,35 @@ "f_globals", "Lorg/python/core/PyObject;"); } code.getfield(mrefs.f_globals); - makeArray(ac.getDefaults()); - - module.PyCode(retSuite, name, ac, true, className, - false, false, node.beginLine).get(code); - - if (mrefs.PyFunction_init1 == 0) { - mrefs.PyFunction_init1 = code.pool.Methodref( + makeArray(node.scope.ac.getDefaults()); + + node.scope.setup_closure(my_scope); + node.scope.dump(); + module.PyCode(retSuite, name, true, className, + false, false, node.beginLine, node.scope).get(code); + Vector freenames = node.scope.freevars; + node.scope = null; // release scope info + + if (!makeClosure(freenames)) { + if (mrefs.PyFunction_init1 == 0) { + mrefs.PyFunction_init1 = code.pool.Methodref( "org/python/core/PyFunction", "", "(Lorg/python/core/PyObject;[Lorg/python/core/PyObject;Lorg/python/core/PyCode;)V"); + } + code.invokespecial(mrefs.PyFunction_init1); + } else { + if (mrefs.PyFunction_closure_init1 == 0) { + mrefs.PyFunction_closure_init1 = code.pool.Methodref( + "org/python/core/PyFunction", "", + "(Lorg/python/core/PyObject;[Lorg/python/core/PyObject;Lorg/python/core/PyCode;[Lorg/python/core/PyObject;)V"); + } + code.invokespecial(mrefs.PyFunction_closure_init1); } - code.invokespecial(mrefs.PyFunction_init1); + return null; } public int Ellipsis; @@ -2088,11 +2161,11 @@ } code.invokespecial(mrefs.PySlice_init); return null; } - public int makeClass; + public int makeClass,makeClass_closure; public Object classdef(SimpleNode node) throws Exception { setline(node); //Get class name String name = getName(node.getChild(0)); @@ -2104,26 +2177,38 @@ SimpleNode[] bases = new SimpleNode[n-2]; for (int i=0; i 0) { + if (ac != null && ac.init_code.getNumChildren() > 0) { ac.init_code.jjtAddChild(tree, ac.init_code.getNumChildren()); tree = ac.init_code; } + + if (scope != null) { + int nparamcell = scope.xxx_paramcells.size(); + if (nparamcell > 0) { + if (to_cell == 0) { + to_cell = classfile.pool.Methodref("org/python/core/PyFrame","to_cell","(II)V"); + } + Hashtable tbl = scope.tbl; + Vector paramcells = scope.xxx_paramcells; + for (int i = 0; i < nparamcell; i++) { + c.aload(1); + SymInfo syminf = (SymInfo)tbl.get(paramcells.elementAt(i)); + c.iconst(syminf.locals_index); + c.iconst(syminf.env_index); + c.invokevirtual(to_cell); + } + } + } - compiler.parse(tree, c, fast_locals, className, classBody, ac); + compiler.parse(tree, c, fast_locals, className, classBody, scope); - code.names = new String[compiler.names.size()]; - for(i=0; i nop + + boolean nested_scopes = this.nested_scopes; + boolean func = kind == FUNCSCOPE; + Vector purecells = new Vector(); + cell = 0; + boolean some_inner_free = inner_free.size() > 0; + + for (Enumeration e = inner_free.keys(); e.hasMoreElements(); ) { + String name = (String)e.nextElement(); + SymInfo info = (SymInfo)tbl.get(name); + if (info == null) { + tbl.put(name,new SymInfo(FREE)); + continue; + } + int flags = info.flags; + if (func) { + if ((flags&GLOBAL) == 0 && (flags&BOUND) != 0) { + if (nested_scopes) { + info.flags |= CELL; + if ((info.flags&PARAM) != 0) xxx_paramcells.addElement(name); + cellvars.addElement(name); + info.env_index = cell++; + if ((flags&PARAM) == 0) purecells.addElement(name); + continue; + } + ctxt.error("local name '"+name+"' in '"+scope_name+"' shadows use as global in nested scopes",false,scope_node); + } + } else { + info.flags |= FREE; + } + } + boolean some_free = false; + + boolean nested = up.kind != TOPSCOPE; + for (Enumeration e = tbl.keys(); e.hasMoreElements(); ) { + String name = (String)e.nextElement(); + SymInfo info = (SymInfo)tbl.get(name); + int flags = info.flags; + if (nested && (flags&FREE) != 0) up.inner_free.put(name,PRESENT); + if ((flags&(GLOBAL|PARAM|CELL)) == 0) { + if ((flags&BOUND) != 0) { // ?? only func + // System.err.println("local: "+name); + names.addElement(name); + info.locals_index = local++; + continue; + } + info.flags |= FREE; + some_free = true; + if (nested) up.inner_free.put(name,PRESENT); + } + } + if ((xxx_npurecell = purecells.size()) > 0) { + int sz = purecells.size(); + for (int i = 0; i < sz; i++) { + names.addElement(purecells.elementAt(i)); + } + } + if ((unqual_exec || from_import_star)) { + if(some_inner_free) dynastuff_trouble(true, ctxt); + else if(func_level > 1 && some_free) dynastuff_trouble(false, ctxt); + } + + } + + private void dynastuff_trouble(boolean inner_free,CodeCompiler ctxt) throws Exception { + String illegal; + if (unqual_exec && from_import_star) + illegal = "function '"+scope_name+"' uses import * and bare exec, which are illegal"; + else if (unqual_exec) + illegal = "unqualified exec is not allowed in function '"+scope_name+"'"; + else illegal = "import * is not allowed in function '"+scope_name+"'"; + String why; + if (inner_free) + why = " because it contains a function with free variables"; + else + why = " because it contains free variables"; + ctxt.error(illegal + why ,nested_scopes,scope_node); + } + + public Vector freevars = new Vector(); + + public void setup_closure(ScopeInfo up) { + if (!nested_scopes) return; + int free = cell; // env = cell...,free... + Hashtable up_tbl = up.tbl; + boolean nested = up.kind != TOPSCOPE; + for (Enumeration e = tbl.keys(); e.hasMoreElements(); ) { + String name = (String)e.nextElement(); + SymInfo info = (SymInfo)tbl.get(name); + int flags = info.flags; + if ((flags&FREE) != 0) { + SymInfo up_info = (SymInfo)up_tbl.get(name); + if (up_info != null) { // ?? differs from CPython -- what is the intended behaviour? + int up_flags = up_info.flags; + if ((up_flags&(CELL|FREE)) != 0) { + info.env_index = free++; + freevars.addElement(name); + continue; + } + if (nested && (up_flags&GLOBAL) != 0) { + info.flags = GLOBAL|BOUND; + continue; + } + } + info.flags &= ~FREE; + } + } + + } + +} diff -r -u5 -P //c/jython/org/python/compiler/ScopesCompiler.java //c/jy-exp/org/python/compiler/ScopesCompiler.java --- //c/jython/org/python/compiler/ScopesCompiler.java Thu Jan 1 00:00:00 1970 +++ //c/jy-exp/org/python/compiler/ScopesCompiler.java Thu Mar 8 04:09:36 2001 @@ -0,0 +1,649 @@ +// (C) Copyright 2001 Samuele Pedroni + +package org.python.compiler; + +import org.python.parser.*; +import java.util.*; + +public class ScopesCompiler extends Visitor implements ScopeConstants { + + private CodeCompiler code_compiler; + + private boolean nested_scopes = false; + + private Stack scopes; + private ScopeInfo cur = null; + + private int mode; + private static final int GET=0; + private static final int SET=1; + private static final int DEL=2; + private static final int AUGSET=4; + + private int level = 0; + private int func_level = 0; + + public ScopesCompiler(CodeCompiler code_compiler) { // ?? polish + this.code_compiler = code_compiler; + scopes = new Stack(); + mode = GET; + String nested_scopes_opt = org.python.core.PySystemState.registry.getProperty("python.xfutures.nested_scopes"); + if (nested_scopes_opt != null && nested_scopes_opt.equals("on")) nested_scopes = true; else + nested_scopes = code_compiler.futures.areNestedScopesOn(); + // System.err.println("nested-scopes: "+nested_scopes); + } + + public Object set(SimpleNode node) throws Exception { + return modal(node,SET); + } + + public Object del(SimpleNode node) throws Exception { + return modal(node,DEL); + } + + + public Object augset(SimpleNode node) throws Exception { + return modal(node,AUGSET); + } + + public Object modal(SimpleNode node, int newmode)throws Exception { + mode = newmode; + node.visit(this); + mode = GET; + return null; + } + + public void beginScope(String name,int kind,SimpleNode node,ArgListCompiler ac) { + if (cur != null) { + scopes.push(cur); + } + if (kind == FUNCSCOPE) func_level++; + node.scope = cur = new ScopeInfo(name,node,level++,kind,func_level,ac,nested_scopes); + } + + public void endScope() throws Exception { + if (cur.kind == FUNCSCOPE) func_level--; + level--; + ScopeInfo up = (!scopes.empty())?(ScopeInfo)scopes.pop():null; + cur.cook(up,code_compiler); + cur.dump(); // dbg + cur = up; + } + + public void parse(SimpleNode node) throws Exception { + node.visit(this); + } + + public Object single_input(SimpleNode node) throws Exception { + beginScope("",TOPSCOPE,node,null); + suite(node); + endScope(); + return null; + } + + public Object file_input(SimpleNode node) throws Exception { + beginScope("",TOPSCOPE,node,null); + suite(node); + endScope(); + return null; + } + + public Object eval_input(SimpleNode node) throws Exception { + beginScope("",TOPSCOPE,node,null); + return_stmt(node); + endScope(); + return null; + } + + private String def(SimpleNode node) { + String name = (String)node.getChild(0).getInfo(); + cur.addBound(name); + return name; + } + + public Object funcdef(SimpleNode node) throws Exception { + String my_name = def(node); + ArgListCompiler ac = new ArgListCompiler(); + SimpleNode suite; + if (node.getNumChildren() == 3) { + suite = node.getChild(2); + //Parse arguments + node.getChild(1).visit(ac); + } else { + suite = node.getChild(1); + } + SimpleNode[] defaults = ac.getDefaults(); + int defc = defaults.length; + for(int i=0; i 0 ) { + for (int i=0; i= 0) { + if ((prev&FROM_PARAM) != 0) code_compiler.error("name '"+name+"' is local and global",true,node); + if ((prev&GLOBAL) != 0) continue; + String what; + if ((prev&BOUND) != 0) what = "assignement"; else what = "use"; + code_compiler.error("name '"+name+"' declared global after "+what,false,node); + } + } + return null; + } + + public Object exec_stmt(SimpleNode node) throws Exception { + cur.exec = true; + int n = node.getNumChildren(); + if (n == 1) cur.unqual_exec = true; + for (int i = 0; i < n; i++) node.getChild(i).visit(this); + return null; + } + + public Object assert_stmt(SimpleNode node) throws Exception { + stmt(node); + return null; + } + + public Object if_stmt(SimpleNode node) throws Exception { + stmt(node); + return null; + } + + public Object while_stmt(SimpleNode node) throws Exception { + stmt(node); + return null; + } + + public Object for_stmt(SimpleNode node) throws Exception { + if (mode != GET) illassign(node); + set(node.getChild(0)); + node.getChild(1).visit(this); + node.getChild(2).visit(this); + if (node.getNumChildren()>3) node.getChild(3).visit(this); + return null; + } + + public Object try_stmt(SimpleNode node) throws Exception { + int n = node.getNumChildren(); + for (int i=0; i1) { + SimpleNode args=node.getChild(1); + int n = args.getNumChildren(); + for (int i=0; i 0) { + for (int i=0; i 0) { + for (int i=0; i",FUNCSCOPE,node,ac); + int n = ac.names.size(); + for (int i=0; i 0) { - f_fastlocals = new PyObject[code.co_varnames.length]; + f_fastlocals = new PyObject[code.co_varnames.length-code.xxx_npurecell]; // internal: may change + } + if (code != null) { // reserve space for env + int env_sz = 0; + if (code.co_freevars != null) env_sz += (f_nfreevars = code.co_freevars.length); + if (code.co_cellvars != null) env_sz += (f_ncells = code.co_cellvars.length); + if (env_sz > 0) f_env = new PyCell[env_sz]; } } public PyFrame(PyTableCode code, PyObject globals) { this(code, null, globals, null); @@ -133,18 +142,18 @@ } return getlocal(f_code.co_varnames[index]); } public PyObject getlocal(String index) { - //System.err.println("getlocal: "+index); + // System.err.println("getlocal: "+index); if (f_locals == null) getf_locals(); PyObject ret = f_locals.__finditem__(index); if (ret != null) return ret; - throw Py.NameError("local: '"+index+"'"); + throw Py.UnboundLocalError("local: '"+index+"'"); //return getglobal(index); } public PyObject getname(String index) { if (f_locals == null) getf_locals(); @@ -202,6 +211,30 @@ } public void delglobal(String index) { f_globals.__delitem__(index); } + + // nested scopes helpers + + public PyObject getclosure(int index) { + return f_env[index]; + } + + public PyObject getderef(int index) { + PyObject obj=f_env[index].ob_ref; + if (obj != null) return obj; + String name; + if (index >= f_ncells) name = f_code.co_freevars[index-f_ncells]; + else name = f_code.co_cellvars[index]; + throw Py.UnboundLocalError("local: '"+name+"'"); + } + + public void setderef(int index,PyObject value) { + f_env[index].ob_ref = value; + } + + public void to_cell(int parm_index,int env_index) { + f_env[env_index].ob_ref=f_fastlocals[parm_index]; + } + } diff -r -u5 -P //c/jython/org/python/core/PyFunction.java //c/jy-exp/org/python/core/PyFunction.java --- //c/jython/org/python/core/PyFunction.java Mon Mar 5 17:04:38 2001 +++ //c/jy-exp/org/python/core/PyFunction.java Thu Mar 8 04:23:18 2001 @@ -11,32 +11,49 @@ public PyObject __doc__; public PyObject func_globals; public PyObject[] func_defaults; public PyCode func_code; public PyObject __dict__; + public PyObject func_closure; // nested scopes: closure public PyFunction(PyObject globals, PyObject[] defaults, PyCode code, - PyObject doc) + PyObject doc,PyObject[] closure_cells) { func_globals = globals; __name__ = code.co_name; if (doc == null) __doc__ = Py.None; else __doc__ = doc; func_defaults = defaults; func_code = code; + if (closure_cells != null) { + func_closure = new PyTuple(closure_cells); + } else { + func_closure = null; + } } + public PyFunction(PyObject globals, PyObject[] defaults, PyCode code, + PyObject doc) { + this(globals,defaults,code,doc,null); + } + public PyFunction(PyObject globals, PyObject[] defaults, PyCode code) { - this(globals, defaults, code, null); + this(globals, defaults, code, null,null); } + + public PyFunction(PyObject globals, PyObject[] defaults, PyCode code, PyObject[] closure_cells) { + this(globals, defaults, code, null,closure_cells); + } + private static final String[] __members__ = { "__doc__", "func_doc", "__name__", "func_name", "__dict__", - "func_globals", "func_defaults", "func_code" + "func_globals", "func_defaults", "func_code", + "func_closure" }; public PyObject __dir__() { PyString members[] = new PyString[__members__.length]; for (int i = 0; i < __members__.length; i++) @@ -58,10 +75,16 @@ // TBD: in CPython, func_code, func_defaults, func_doc, __doc__ are // writable. For now, only func_doc, __doc__ are writable in // JPython. if (name == "func_doc" || name == "__doc__") __doc__ = value; + else if (name == "func_closure") { + if (!(value instanceof PyTuple)) { + throw Py.TypeError("func_closure must be set to a tuple"); + } + func_closure = value; + } // not yet implemented: // func_code // func_defaults else if (name == "func_defaults") throwReadonly(name); @@ -108,10 +131,14 @@ // these are special, everything else is findable by reflection if (name == "func_doc") return __doc__; if (name == "func_name") return new PyString(__name__); + if (name == "func_closure") { + if (func_closure != null) return func_closure; + return Py.None; + } if (name == "func_defaults") { if (func_defaults.length == 0) return Py.None; return new PyTuple(func_defaults); } @@ -135,30 +162,30 @@ public PyObject _doget(PyObject container, PyObject wherefound) { return new PyMethod(container, this, wherefound); } public PyObject __call__() { - return func_code.call(func_globals, func_defaults); + return func_code.call(func_globals, func_defaults, func_closure); } public PyObject __call__(PyObject arg) { - return func_code.call(arg, func_globals, func_defaults); + return func_code.call(arg, func_globals, func_defaults, func_closure); } public PyObject __call__(PyObject arg1, PyObject arg2) { - return func_code.call(arg1, arg2, func_globals, func_defaults); + return func_code.call(arg1, arg2, func_globals, func_defaults, func_closure); } public PyObject __call__(PyObject arg1, PyObject arg2, PyObject arg3) { - return func_code.call(arg1, arg2, arg3, func_globals, func_defaults); + return func_code.call(arg1, arg2, arg3, func_globals, func_defaults, func_closure); } public PyObject __call__(PyObject[] args, String[] keywords) { - return func_code.call(args, keywords, func_globals, func_defaults); + return func_code.call(args, keywords, func_globals, func_defaults, func_closure); } public PyObject __call__(PyObject arg1, PyObject[] args, String[] keywords) { return func_code.call(arg1, args, keywords, func_globals, - func_defaults); + func_defaults, func_closure); } public String toString() { return ""; } } diff -r -u5 -P //c/jython/org/python/core/PyTableCode.java //c/jy-exp/org/python/core/PyTableCode.java --- //c/jython/org/python/core/PyTableCode.java Mon Feb 26 16:25:02 2001 +++ //c/jy-exp/org/python/core/PyTableCode.java Thu Mar 8 04:25:38 2001 @@ -10,26 +10,43 @@ { public int co_argcount; int nargs; public int co_firstlineno = -1; public String co_varnames[]; + public String co_cellvars[]; + public int xxx_npurecell; // internal: may change + public String co_freevars[]; public String co_filename; public int co_flags; public boolean args, keywords; PyFunctionTable funcs; int func_id; - + + public PyTableCode(int argcount, String varnames[], String filename, String name, int firstlineno, boolean args, boolean keywords, PyFunctionTable funcs, int func_id) { + this(argcount,varnames,filename,name,firstlineno,args,keywords,funcs,func_id,null,null,0); + } + + public PyTableCode(int argcount, String varnames[], + String filename, String name, + int firstlineno, + boolean args, boolean keywords, + PyFunctionTable funcs, int func_id, + String[] cellvars,String[] freevars,int xxx_npurecell) // may change + { co_argcount = nargs = argcount; co_varnames = varnames; co_filename = filename; co_firstlineno = firstlineno; + co_cellvars = cellvars; + co_freevars = freevars; + this.xxx_npurecell = xxx_npurecell; this.args = args; co_name = name; if (args) { co_argcount -= 1; co_flags |= 0x04; @@ -44,11 +61,11 @@ } private static final String[] __members__ = { "co_name", "co_argcount", "co_varnames", "co_filename", "co_firstlineno", - "co_flags" + "co_flags","co_cellvars","co_freevars" // not supported: co_nlocals, co_code, co_consts, co_names, // co_lnotab, co_stacksize }; public PyObject __dir__() { @@ -72,22 +89,29 @@ public void __delattr__(String name) { throwReadonly(name); } + private static PyTuple toPyStringTuple(String[] ar) { + if (ar == null) return Py.EmptyTuple; + int sz = ar.length; + PyString[] pystr = new PyString[sz]; + for (int i = 0; i < sz; i++) { + pystr[i] = new PyString(ar[i]); + } + return new PyTuple(pystr); + } + public PyObject __findattr__(String name) { // have to craft co_varnames specially - if (name == "co_varnames") { - PyString varnames[] = new PyString[co_varnames.length]; - for (int i = 0; i < co_varnames.length; i++) - varnames[i] = new PyString(co_varnames[i]); - return new PyTuple(varnames); - } + if (name == "co_varnames") return toPyStringTuple(co_varnames); + if (name == "co_cellvars") return toPyStringTuple(co_cellvars); + if (name == "co_freevars") return toPyStringTuple(co_freevars); return super.__findattr__(name); } - public PyObject call(PyFrame frame) { + public PyObject call(PyFrame frame, PyObject closure) { // System.err.println("tablecode call: "+co_name); ThreadState ts = Py.getThreadState(); if (ts.systemState == null) { ts.systemState = Py.defaultSystemState; } @@ -105,10 +129,24 @@ //System.err.println("ts: "+ts); //System.err.println("ss: "+ts.systemState); frame.f_builtins = ts.systemState.builtins; } } + // nested scopes: setup env with closure + int env_j = 0; + int ncells = frame.f_ncells; + int nfreevars = frame.f_nfreevars; + PyCell[] env = frame.f_env; + PyTuple freevars = (PyTuple)closure; + for (int i = 0; i < ncells; i++,env_j++) { + env[env_j] = new PyCell(); + } + for (int i=0; i