Index: org/python/compiler/Code.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/compiler/Code.java,v retrieving revision 2.7 diff -u -r2.7 Code.java --- org/python/compiler/Code.java 15 Oct 2002 14:32:27 -0000 2.7 +++ org/python/compiler/Code.java 31 Dec 2003 04:09:20 -0000 @@ -41,6 +41,12 @@ return l; } + public Label getLabelAtPosition() { + Label l = getLabel(); + l.setPosition(); + return l; + } + public void addLabel(Label l) { labels.addElement(l); } @@ -336,6 +342,16 @@ public void goto_(Label label) throws IOException { branch(167, label); + } + + public void iaload() throws IOException { + code.writeByte(46); + push(-1); + } + + public void iastore() throws IOException { + code.writeByte(79); + push(-3); } public void iconst(int i) throws IOException { Index: org/python/compiler/CodeCompiler.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/compiler/CodeCompiler.java,v retrieving revision 2.32 diff -u -r2.32 CodeCompiler.java --- org/python/compiler/CodeCompiler.java 25 Aug 2003 12:24:14 -0000 2.32 +++ org/python/compiler/CodeCompiler.java 31 Dec 2003 04:09:21 -0000 @@ -53,8 +53,20 @@ public String className; public Stack continueLabels, breakLabels, finallyLabels; - public Stack inFinallyLabels; - public Vector yields = new Vector(); + + /** + * Which subroutine are we compiling? Maintained as a stack + * for nesting. There's an entry in the stack for the main routine + * of the method. + * + * Subroutines are used to compile finally blocks. + */ + public Stack subroutines; + + /** + * Stores the maximum nesting depth of the subroutines in this method. + */ + public int subroutineDepth = 0; /* break/continue finally's level. * This is the lowest level in finallyLabels which should @@ -74,7 +86,7 @@ continueLabels = new Stack(); breakLabels = new Stack(); finallyLabels = new Stack(); - inFinallyLabels = new Stack(); + subroutines = new Stack(); this.print_results = print_results; } @@ -94,14 +106,42 @@ int f_lasti; - public void setLastI(int idx) throws Exception { + void prepareLastI() throws Exception { if (mrefs.f_lasti == 0) { mrefs.f_lasti = code.pool.Fieldref( - "org/python/core/PyFrame", "f_lasti", "I"); + "org/python/core/PyFrame", "f_lasti", "[I"); } + } + + public void setLastI(int idx) throws Exception { + prepareLastI(); loadFrame(); + code.getfield(mrefs.f_lasti); + int tmp = storeTop(); + code.aload(tmp); + code.iconst(currentSubroutineDepth()); code.iconst(idx); - code.putfield(mrefs.f_lasti); + code.iastore(); + // clear out all lower levels. + for (int i = currentSubroutineDepth() + 1; i < subroutineDepth; ++i) + { + code.aload(tmp); + code.iconst(i); + code.iconst(0); + code.iastore(); + } + code.freeLocal(tmp); + } + + /** + * leave lasti on the stack + */ + public void getLastI() throws Exception { + prepareLastI(); + loadFrame(); + code.getfield(mrefs.f_lasti); + code.iconst(currentSubroutineDepth()); + code.iaload(); } public int storeTop() throws Exception { @@ -192,7 +232,6 @@ code.freeLocal(augtmp4); } - public void parse(modType node, Code code, boolean fast_locals, String className, boolean classBody, ScopeInfo scope,CompilerFlags cflags) @@ -205,6 +244,9 @@ my_scope = scope; names = scope.names; + subroutines.push(new Subroutine(true, my_scope.generator)); + subroutineDepth = getFinallyDepth(node) + 1; + tbl = scope.tbl; optimizeGlobals = fast_locals&&!scope.exec&&!scope.from_import_star; @@ -219,7 +261,8 @@ } else { if (exit == null) { //System.out.println("no exit"); - setLastI(-1); + if (my_scope.generator) + setLastI(-1); getNone(); code.areturn(); @@ -519,7 +562,7 @@ } for (int i = finallyLabels.size() - 1; i >= bcfLevel; i--) { - doFinallyPart((InFinally)finallyLabels.elementAt(i)); + doFinallyPart((Subroutine)finallyLabels.elementAt(i)); } code.goto_((Label)breakLabels.peek()); @@ -533,14 +576,12 @@ } for (int i = finallyLabels.size() - 1; i >= bcfLevel; i--) { - doFinallyPart((InFinally)finallyLabels.elementAt(i)); + doFinallyPart((Subroutine)finallyLabels.elementAt(i)); } code.goto_((Label)continueLabels.peek()); return Exit; } - int yield_count = 0; - int f_savedlocals; public Object visitYield(Yield node) throws Exception { @@ -555,35 +596,69 @@ "block with a 'finally' clause", node); } + saveLocals(); + prepareReentry(); - InFinally inFinally = null; - if (inFinallyLabels.size() > 0) - inFinally = (InFinally) inFinallyLabels.peek(); + visit(node.value); + code.areturn(); + addReentryLabel(null); + + restoreLocals(); - if (inFinally == null) { - saveLocals(); - visit(node.value); - setLastI(++yield_count); - code.areturn(); - Label restart = code.getLabel(); - yields.addElement(restart); - restart.setPosition(); - restoreLocals(); - } else { - saveLocals(); - visit(node.value); - code.areturn(); - code.ret(inFinally.retLocal); - inFinally.labels[inFinally.cnt++].setPosition(); - code.stack = 1; - code.astore(inFinally.retLocal); - restoreLocals(); - } return null; } + public int currentSubroutineDepth() + { + return subroutines.size() - 1; + } + + public Subroutine getCurrentSubroutine() + { + return (Subroutine) subroutines.peek(); + } + + /** + * Get the list of places the current subroutine is reentered after + * a yield or a jump to a routine that yields. + */ + public Vector getReentryPoints() + { + return getCurrentSubroutine().reentryPoints; + } + + public void prepareReentry() throws Exception + { + Subroutine sub = getCurrentSubroutine(); + setLastI(++sub.entryCount); + } + + /** + * Add a label to the current subroutine so that the control flow + * can jump to here after a yield. This stores the reentry point + * for use in generating the jump table later. If we are jumping + * to a subroutine from here, we need to allow the subroutine + * to generate a stub that clears variables correctly. + */ + public void addReentryLabel(Subroutine reentryTo) throws Exception + { + Label here = code.getLabelAtPosition(); + Label jumpTo = here; + + Subroutine subroutine = getCurrentSubroutine(); + + if (reentryTo != null) { + jumpTo = code.getLabel(); + subroutine.reentryStubs.addElement(jumpTo); + subroutine.reentryStubEnds.addElement(here); + subroutine.reentryVars.addElement(reentryTo.varsLiveOnEntry); + } + + subroutine.reentryPoints.addElement(jumpTo); + } + private void restoreLocals() throws Exception { Vector v = code.getActiveLocals(); @@ -669,9 +744,12 @@ code.astore(tmp); } for (int i = finallyLabels.size() - 1; i >= 0; i--) { - doFinallyPart((InFinally) finallyLabels.elementAt(i)); + doFinallyPart((Subroutine) finallyLabels.elementAt(i)); + } + + if (my_scope.generator) { + setLastI(-1); } - setLastI(-1); if (node.value != null) { code.aload(tmp); @@ -1094,31 +1172,30 @@ Object ret; - // Do protected suite int yieldCount = 0; - if (my_scope.generator) { - YieldChecker checker = new YieldChecker(); - checker.visit(node.finalbody); - yieldCount = checker.yieldCount; - } - if (yieldCount > 0) { - throw new ParseException("'yield' in finally not yet supported", - node); + if (my_scope.generator) + { + YieldChecker checker = new YieldChecker(); + checker.visit(node.finalbody); + yieldCount = checker.yieldCount; } - InFinally inFinally = new InFinally(yieldCount + 1); + Subroutine finallyRoutine = new Subroutine(false, yieldCount != 0); - finallyLabels.push(inFinally); + // Do protected suite + finallyLabels.push(finallyRoutine); int excLocal = code.getLocal("java/lang/Throwable"); code.aconst_null(); code.astore(excLocal); + finallyRoutine.varsLiveOnEntry = code.getActiveLocals(); + start.setPosition(); ret = suite(node.body); end.setPosition(); if (ret == null) { - doFinallyPart(inFinally); + doFinallyPart(finallyRoutine); code.goto_(finallyEnd); } @@ -1139,34 +1216,47 @@ } code.invokestatic(mrefs.add_traceback); - doFinallyPart(inFinally); + doFinallyPart(finallyRoutine); code.aload(excLocal); code.checkcast(code.pool.Class("java/lang/Throwable")); code.athrow(); + finallyRoutine.retLocal = code.getFinallyLocal("ret"); + // Do finally suite - inFinally.labels[0].setPosition(); - code.stack = 1; - inFinally.retLocal = code.getFinallyLocal("ret"); - code.astore(inFinally.retLocal); + finallyRoutine.entry.setPosition(); + Label genswitch = null; + if (finallyRoutine.isReentered) { + genswitch = code.getLabel(); + code.goto_(genswitch); + } - // Trick the JVM verifier into thinking this code might not - // be executed - //code.iconst(1); - //code.ifeq(skipSuite); + finallyRoutine.start.setPosition(); + + // if reentered, we'll do this at the genswitch + if (! finallyRoutine.isReentered) { + finallyRoutine.storeRet(); + } - inFinallyLabels.push(inFinally); + subroutines.push(finallyRoutine); - // The actual finally suite is always executed (since 1 != 0) ret = suite(node.finalbody); - inFinallyLabels.pop(); + if (finallyRoutine.isReentered) { + setLastI(0); // return from finally resets lasti at this level + } + + code.ret(finallyRoutine.retLocal); - // Fake jump to here to pretend this could always happen - //skipSuite.setPosition(); + if (finallyRoutine.isReentered) { + generateReentryTable(genswitch, + finallyRoutine.start, + finallyRoutine.reentryPoints); + } - code.ret(inFinally.retLocal); - code.freeFinallyLocal(inFinally.retLocal); + subroutines.pop(); + + code.freeFinallyLocal(finallyRoutine.retLocal); finallyEnd.setPosition(); @@ -1178,25 +1268,41 @@ return null; } - private void doFinallyPart(InFinally inFinally) throws Exception { - if (inFinally.labels.length == 1) { - code.jsr(inFinally.labels[0]); - } else { - Label endOfFinal = code.getLabel(); - for (int i = 0; i < inFinally.labels.length; i++) { - setLastI(++yield_count); - code.jsr(inFinally.labels[i]); - - //code.areturn(); - if (i < inFinally.labels.length-1) { - code.goto_(endOfFinal); - Label restart = code.getLabel(); - yields.addElement(restart); - restart.setPosition(); - } - } - endOfFinal.setPosition(); + /** + * Generate the table that jumps back into this subroutine after a + * yeild. + */ + public void generateReentryTable(Label genswitch, + Label start, + Vector reentries) throws Exception + { + Subroutine sub = getCurrentSubroutine(); + sub.generateReentryStubs(); + + genswitch.setPosition(); + + sub.storeRet(); + + getLastI(); + + Label[] entries = new Label[reentries.size()+1]; + + entries[0] = start; + for (int i = 1; i < entries.length; i++) { + entries[i] = (Label) reentries.elementAt(i-1); } + code.tableswitch(start, 0, entries); + } + + /** + * Generate a call to the finally: suite from within the try: suite. + */ + private void doFinallyPart(Subroutine finallyState) throws Exception { + if (finallyState.isReentered) { + prepareReentry(); + addReentryLabel(finallyState); + } + code.jsr(finallyState.entry); } class YieldChecker extends Visitor { @@ -1208,6 +1314,28 @@ } } + class FinallyChecker extends Visitor { + public int depth = 0; + public int maxDepth = 0; + + public Object visitTryFinally(TryFinally node) throws Exception + { + visit(node.body); + depth++; + maxDepth = Math.max(depth, maxDepth); + visit(node.finalbody); + depth--; + return null; + } + } + + public int getFinallyDepth(SimpleNode node) throws Exception + { + FinallyChecker fc = new FinallyChecker(); + fc.visit(node); + return fc.maxDepth; + } + public int set_exception; public Object visitTryExcept(TryExcept node) throws Exception { Label start = code.getLabel(); @@ -2327,15 +2455,107 @@ } - class InFinally { + /** + * Keeps track of the entry points, return address, and live variables + * of a subroutine within a method. + */ + class Subroutine { + /** + * is this the main routine of the method? + */ + public boolean isMain; + /** + * does this routine contain yields that couse it to need reentry? + */ + public boolean isReentered; + /** + * which local variable to we return to? + */ public int retLocal; - public Label[] labels; - public int cnt = 1; - - public InFinally(int labelcnt) { - labels = new Label[labelcnt]; - for (int i = 0; i < labelcnt; i++) { - labels[i] = code.getLabel(); + /** + * which entry are we on now? + */ + public int entryCount = 0; + /** + * the entry point for this routine. + */ + public Label entry = code.getLabel(); + /** + * the start of the main block + */ + public Label start = code.getLabel(); + /** + * reentry points within the main block. + */ + public Vector reentryPoints = new Vector(); + + /** + * these variables have to be cleared on a reentry, or the verifier + * can get unhappy. + */ + public Vector varsLiveOnEntry; + + /** + * A vector of stubs for reentry which clear the variables live in + * a nested block before jumping into the flow. These stubs are used + * for the jump table of this subroutine, and they end + * with a jump to a jsr to a nested block's jump table; each stub + * clears vars that have to be empty before the start of the nested + * block. + */ + public Vector reentryStubs = new Vector(); + public Vector reentryStubEnds = new Vector(); + public Vector reentryVars = new Vector(); + + public Subroutine(boolean mainRoutine, boolean reentered) + { + isMain = mainRoutine; + isReentered = reentered; + } + + public void clearVars(Vector vars) throws IOException + { + for (int i = 0; i < vars.size(); ++i) { + String var = (String)vars.elementAt(i); + if (var == null) { + continue; + } + code.aconst_null(); + code.astore(i); + } + } + + /** + * Generate a set of stubs that can clear live variables before + * reentering this block. Really, we're clearing live variables for + * entry to a nested block (see addReentryLabel) -- we need to do this + * relatively far from the nested block. This makes the verifier happy + * -- all the variables are clear -- even though it's not strictly + * necessary. This can't be done inside the try: block for a + * try:finally: because the verifier doesn't like it inside the + * exception catch scope. We thus do it just before generating the + * tableswitch that will jump to these stubs. + */ + public void generateReentryStubs() throws IOException + { + for (int i = 0; i < reentryStubs.size(); ++i) { + Label stub = (Label)reentryStubs.elementAt(i); + stub.setPosition(); + Vector vars = (Vector)reentryVars.elementAt(i); + clearVars(vars); + Label stubEnd = (Label)reentryStubEnds.elementAt(i); + code.goto_(stubEnd); + } + } + + /** + * Store the return address of a subroutine into a local variable + * on entry (or reentry). + */ + public void storeRet() throws Exception { + if (! isMain) { + code.stack = 1; + code.astore(retLocal); } } } Index: org/python/compiler/Module.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/compiler/Module.java,v retrieving revision 2.13 diff -u -r2.13 Module.java --- org/python/compiler/Module.java 15 Oct 2002 14:32:28 -0000 2.13 +++ org/python/compiler/Module.java 31 Dec 2003 04:09:21 -0000 @@ -197,6 +197,9 @@ public String[] freevars; public int jy_npurecell; + // for generators & finallys + public int jy_subroutineDepth = 0; + public int moreflags; public PyCodeConstant() { ; @@ -245,11 +248,13 @@ c.iconst(moreflags); + c.iconst(jy_subroutineDepth); + int mref_newCode = c.pool.Methodref( "org/python/core/Py", "newCode", "(I" + $strArr + $str + $str + "IZZ" + $pyFuncTbl + "I" + - $strArr + $strArr + "II)" + $pyCode); + $strArr + $strArr + "III)" + $pyCode); c.invokestatic(mref_newCode); //c.aconst_null(); @@ -453,22 +458,10 @@ scope, cflags); if (scope.generator) { - genswitch.setPosition(); - c.aload(1); - if (compiler.f_lasti == 0) { - compiler.f_lasti = c.pool.Fieldref( - "org/python/core/PyFrame", "f_lasti", "I"); - } - c.getfield(compiler.f_lasti); - - Label[] yields = new Label[compiler.yields.size()+1]; - - yields[0] = start; - for (int i = 1; i < yields.length; i++) { - yields[i] = (Label) compiler.yields.elementAt(i-1); - } - c.tableswitch(start, 0, yields); - // XXX: Generate an error + compiler.generateReentryTable(genswitch, + start, + compiler.getReentryPoints()); + code.jy_subroutineDepth = compiler.subroutineDepth; } // !classdef only Index: org/python/core/Py.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/Py.java,v retrieving revision 2.70 diff -u -r2.70 Py.java --- org/python/core/Py.java 6 Aug 2003 11:46:26 -0000 2.70 +++ org/python/core/Py.java 31 Dec 2003 04:09:22 -0000 @@ -488,6 +488,21 @@ boolean args, boolean keywords, PyFunctionTable funcs, int func_id, String[] cellvars,String[] freevars, + int npurecell, int moreflags, + int subroutineDepth) + { + return new PyTableCode(argcount, varnames, + filename, name, firstlineno, args, keywords, + funcs, func_id, cellvars, freevars, npurecell, + moreflags, subroutineDepth); + } + + public static PyCode newCode(int argcount, String varnames[], + String filename, String name, + int firstlineno, + boolean args, boolean keywords, + PyFunctionTable funcs, int func_id, + String[] cellvars,String[] freevars, int npurecell, int moreflags) { Index: org/python/core/PyFrame.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/PyFrame.java,v retrieving revision 2.14 diff -u -r2.14 PyFrame.java --- org/python/core/PyFrame.java 15 Oct 2002 14:32:28 -0000 2.14 +++ org/python/core/PyFrame.java 31 Dec 2003 04:09:22 -0000 @@ -18,7 +18,7 @@ public PyCell[] f_env; // nested scopes: cell + free env public int f_ncells; public int f_nfreevars; - public int f_lasti; + public int[] f_lasti; public Object[] f_savedlocals; // an interface to functions suitable for tracing, e.g. via sys.settrace() @@ -60,7 +60,9 @@ env_sz += (f_ncells = code.co_cellvars.length); if (env_sz > 0) f_env = new PyCell[env_sz]; - } + f_lasti = new int[code.jy_subroutineDepth + 1]; + } else + f_lasti = null; } public PyFrame(PyTableCode code, PyObject globals) { Index: org/python/core/PyGenerator.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/PyGenerator.java,v retrieving revision 2.1 diff -u -r2.1 PyGenerator.java --- org/python/core/PyGenerator.java 15 Oct 2002 14:32:28 -0000 2.1 +++ org/python/core/PyGenerator.java 31 Dec 2003 04:09:22 -0000 @@ -31,7 +31,7 @@ public PyObject __iternext__() { if (gi_running) throw Py.ValueError("generator already executing"); - if (gi_frame.f_lasti == -1) + if (gi_frame.f_lasti[0] == -1) return null; gi_running = true; PyObject result = null; @@ -40,10 +40,10 @@ } finally { gi_running = false; } -// System.out.println("lasti:" + gi_frame.f_lasti); +// System.out.println("lasti:" + gi_frame.f_lasti[0]); //if (result == Py.None) // new Exception().printStackTrace(); - if (result == Py.None && gi_frame.f_lasti == -1) + if (result == Py.None && gi_frame.f_lasti[0] == -1) return null; return result; } Index: org/python/core/PyTableCode.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/PyTableCode.java,v retrieving revision 2.22 diff -u -r2.22 PyTableCode.java --- org/python/core/PyTableCode.java 6 Aug 2003 11:46:26 -0000 2.22 +++ org/python/core/PyTableCode.java 31 Dec 2003 04:09:23 -0000 @@ -21,6 +21,7 @@ public boolean args, keywords; PyFunctionTable funcs; int func_id; + int jy_subroutineDepth; // internal: jython: holds the nesting level of jsrs final public static int CO_OPTIMIZED = 0x0001; //final public static int CO_NEWLOCALS = 0x0002 @@ -50,7 +51,19 @@ boolean args, boolean keywords, PyFunctionTable funcs, int func_id, String[] cellvars, String[] freevars, int npurecell, - int moreflags) // may change + int moreflags) + { + this(argcount, varnames, filename, name, firstlineno, args, + keywords, funcs, func_id, null, null, 0, 0, 0); + } + + public PyTableCode(int argcount, String varnames[], + String filename, String name, + int firstlineno, + boolean args, boolean keywords, + PyFunctionTable funcs, int func_id, + String[] cellvars, String[] freevars, int npurecell, + int moreflags, int subroutineDepth) // may change { co_argcount = nargs = argcount; co_varnames = varnames; @@ -74,6 +87,7 @@ co_flags |= moreflags; this.funcs = funcs; this.func_id = func_id; + this.jy_subroutineDepth = subroutineDepth; } private static final String[] __members__ = { @@ -223,7 +237,9 @@ e.traceback = tb; } - frame.f_lasti = -1; + if ((co_flags & CO_GENERATOR) != 0) { + frame.f_lasti[0] = -1; + } if (frame.tracefunc != null) { frame.tracefunc.traceException(frame, e); Index: org/python/core/imp.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/imp.java,v retrieving revision 2.64 diff -u -r2.64 imp.java --- org/python/core/imp.java 3 Jan 2003 00:49:34 -0000 2.64 +++ org/python/core/imp.java 31 Dec 2003 04:09:26 -0000 @@ -12,7 +12,7 @@ */ public class imp { - public static final int APIVersion = 12; + public static final int APIVersion = 13; private imp() { ; }