diff -r 06cfd92ad6a0 src/org/python/core/Py.java --- a/src/org/python/core/Py.java Mon Dec 14 18:06:57 2015 -0700 +++ b/src/org/python/core/Py.java Tue Dec 22 17:28:16 2015 -0700 @@ -1019,7 +1019,7 @@ } JyAttribute.setAttr(instance, JyAttribute.JAVA_PROXY_ATTR, proxy); proxy._setPyInstance(instance); - proxy._setPySystemState(ts.systemState); + proxy._setPySystemState(ts.getSystemState()); return; } @@ -1041,7 +1041,7 @@ instance = pyc.__call__(pargs); JyAttribute.setAttr(instance, JyAttribute.JAVA_PROXY_ATTR, proxy); proxy._setPyInstance(instance); - proxy._setPySystemState(ts.systemState); + proxy._setPySystemState(ts.getSystemState()); } /** @@ -1145,11 +1145,11 @@ ThreadState ts = getThreadState(); - ts.systemState.last_value = exc.value; - ts.systemState.last_type = exc.type; - ts.systemState.last_traceback = exc.traceback; + ts.getSystemState().last_value = exc.value; + ts.getSystemState().last_type = exc.type; + ts.getSystemState().last_traceback = exc.traceback; - PyObject exceptHook = ts.systemState.__findattr__("excepthook"); + PyObject exceptHook = ts.getSystemState().__findattr__("excepthook"); if (exceptHook != null) { try { exceptHook.__call__(exc.type, exc.value, exc.traceback); @@ -1457,18 +1457,18 @@ public static final PySystemState setSystemState(PySystemState newSystemState) { ThreadState ts = getThreadState(newSystemState); - PySystemState oldSystemState = ts.systemState; + PySystemState oldSystemState = ts.getSystemState(); if (oldSystemState != newSystemState) { //XXX: should we make this a real warning? //System.err.println("Warning: changing systemState "+ // "for same thread!"); - ts.systemState = newSystemState; + ts.setSystemState(newSystemState); } return oldSystemState; } public static final PySystemState getSystemState() { - return getThreadState().systemState; + return getThreadState().getSystemState(); //defaultSystemState; } @@ -2051,7 +2051,7 @@ } public static void printResult(PyObject ret) { - Py.getThreadState().systemState.invoke("displayhook", ret); + Py.getThreadState().getSystemState().invoke("displayhook", ret); } public static final int ERROR = -1; public static final int WARNING = 0; diff -r 06cfd92ad6a0 src/org/python/core/PyBaseCode.java --- a/src/org/python/core/PyBaseCode.java Mon Dec 14 18:06:57 2015 -0700 +++ b/src/org/python/core/PyBaseCode.java Tue Dec 22 17:28:16 2015 -0700 @@ -28,8 +28,8 @@ public PyObject call(ThreadState ts, PyFrame frame, PyObject closure) { // System.err.println("tablecode call: "+co_name); - if (ts.systemState == null) { - ts.systemState = Py.defaultSystemState; + if (ts.getSystemState() == null) { // TODO: is this check still useful?? + ts.setSystemState(Py.defaultSystemState); } //System.err.println("got ts: "+ts+", "+ts.systemState); @@ -44,7 +44,7 @@ } else { //System.err.println("ts: "+ts); //System.err.println("ss: "+ts.systemState); - frame.f_builtins = ts.systemState.builtins; + frame.f_builtins = ts.getSystemState().builtins; } } // nested scopes: setup env with closure @@ -105,7 +105,7 @@ // Check for interruption, which is used for restarting the interpreter // on Jython - if (ts.systemState._systemRestart && Thread.currentThread().isInterrupted()) { + if (ts.getSystemState()._systemRestart && Thread.currentThread().isInterrupted()) { throw new PyException(_systemrestart.SystemRestart); } return ret; diff -r 06cfd92ad6a0 src/org/python/core/PyFrame.java --- a/src/org/python/core/PyFrame.java Mon Dec 14 18:06:57 2015 -0700 +++ b/src/org/python/core/PyFrame.java Tue Dec 22 17:28:16 2015 -0700 @@ -273,7 +273,7 @@ // Set up f_builtins if not already set if (f_builtins == null) { - f_builtins = Py.getThreadState().systemState.builtins; + f_builtins = Py.getThreadState().getSystemState().builtins; } return f_builtins.__finditem__(index); } diff -r 06cfd92ad6a0 src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java Mon Dec 14 18:06:57 2015 -0700 +++ b/src/org/python/core/PySystemState.java Tue Dec 22 17:28:16 2015 -0700 @@ -14,8 +14,6 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLDecoder; import java.nio.charset.Charset; import java.nio.charset.UnsupportedCharsetException; import java.security.AccessControlException; diff -r 06cfd92ad6a0 src/org/python/core/PyTableCode.java --- a/src/org/python/core/PyTableCode.java Mon Dec 14 18:06:57 2015 -0700 +++ b/src/org/python/core/PyTableCode.java Tue Dec 22 17:28:16 2015 -0700 @@ -125,8 +125,8 @@ @Override public PyObject call(ThreadState ts, PyFrame frame, PyObject closure) { // System.err.println("tablecode call: "+co_name); - if (ts.systemState == null) { - ts.systemState = Py.defaultSystemState; + if (ts.getSystemState() == null) { // TODO: is this check still useful?? + ts.setSystemState(Py.defaultSystemState); } //System.err.println("got ts: "+ts+", "+ts.systemState); @@ -141,7 +141,7 @@ } else { //System.err.println("ts: "+ts); //System.err.println("ss: "+ts.systemState); - frame.f_builtins = ts.systemState.builtins;; + frame.f_builtins = ts.getSystemState().builtins;; } } // nested scopes: setup env with closure @@ -202,7 +202,7 @@ // Check for interruption, which is used for restarting the interpreter // on Jython - if (ts.systemState._systemRestart && Thread.currentThread().isInterrupted()) { + if (ts.getSystemState()._systemRestart && Thread.currentThread().isInterrupted()) { throw new PyException(_systemrestart.SystemRestart); } return ret; diff -r 06cfd92ad6a0 src/org/python/core/ThreadState.java --- a/src/org/python/core/ThreadState.java Mon Dec 14 18:06:57 2015 -0700 +++ b/src/org/python/core/ThreadState.java Tue Dec 22 17:28:16 2015 -0700 @@ -1,12 +1,11 @@ // Copyright (c) Corporation for National Research Initiatives package org.python.core; -import java.util.LinkedList; +// a ThreadState refers to one PySystemState; this weak ref allows for tracking all ThreadState objects +// that refer to a given PySystemState public class ThreadState { - public PySystemState systemState; - public PyFrame frame; public PyException exception; @@ -25,10 +24,26 @@ private PyDictionary compareStateDict; + private PySystemStateRef systemStateRef; + public ThreadState(PySystemState systemState) { - this.systemState = systemState; + setSystemState(systemState); } + public void setSystemState(PySystemState systemState) { + if (systemState == null) { + systemState = Py.defaultSystemState; + } + if (systemStateRef == null || systemStateRef.get() != systemState) { + systemStateRef = new PySystemStateRef(systemState, this); + } + } + + public PySystemState getSystemState() { + PySystemState systemState = systemStateRef == null ? null : systemStateRef.get(); + return systemState == null ? Py.defaultSystemState : systemState; + } + public boolean enterRepr(PyObject obj) { if (reprStack == null) { reprStack = new PyList(new PyObject[] {obj}); diff -r 06cfd92ad6a0 src/org/python/core/ThreadStateMapping.java --- a/src/org/python/core/ThreadStateMapping.java Mon Dec 14 18:06:57 2015 -0700 +++ b/src/org/python/core/ThreadStateMapping.java Tue Dec 22 17:28:16 2015 -0700 @@ -1,27 +1,51 @@ package org.python.core; import com.google.common.collect.MapMaker; - import java.util.Map; +/** + * A ThreadState augments a standard java.lang.Thread to support Python semantics. The ThreadStateMapping utility class + * ensures that the runtime can look up a ThreadState at any time for a given Thread, while also ensuring that it is + * properly cleaned up. + * + * A ThreadState to Thread relation must be maintained over the entirety of the Python call stack, including any + * interleaving with Java code. This relationship is maintained by inCallThreadState; the use of ThreadLocal + * ensures that the Thread has no static class loader dependency on the Jython runtime or any classes that are loaded. + * Because ThreadState itself has a weak reference to PySystemState, we also need to ensure a hard reference to it is + * separately maintained in the call stack. + * + * The ThreadState to Thread relationship also need to be maintained so long as the PySystemState for a ThreadState (shared by + * N ThreadState objects) is referenced. This relationship is maintained by the bijective mapping provided by + * globalThreadStates and inverseGlobalThreadStates. + * + * See discussion here: http://bugs.jython.org/issue2321 + * and: http://bugs.jython.org/issue1327 + * + * NOTE and possible FIXME: + * inCallThreadState does not currently track changes rebinding a ThreadState to a new PySystemState in the Python + * call stack. But not certain if this actually would actually work - changing sys in this way for some thread of + * execution likely would cause lots of issues in the Python code!!! + */ + class ThreadStateMapping { - private static final Map cachedThreadState = - new MapMaker().weakKeys().weakValues().makeMap(); - - private static ThreadLocal scopedThreadState= new ThreadLocal() { + private static final Map globalThreadStates = + new MapMaker().weakKeys().makeMap(); + private static final Map inverseGlobalThreadStates = + new MapMaker().weakValues().makeMap(); + private static final ThreadLocal inCallThreadState = new ThreadLocal() { @Override protected Object[] initialValue() { - return new Object[1]; + return new Object[2]; // ThreadState, hard ref to the ThreadState's PySystemState } }; public ThreadState getThreadState(PySystemState newSystemState) { - Object scoped = scopedThreadState.get()[0]; - if (scoped != null) { - return (ThreadState)scoped; + Object[] scoped = inCallThreadState.get(); + if (scoped[0] != null) { + return (ThreadState)scoped[0]; } Thread currentThread = Thread.currentThread(); - ThreadState ts = cachedThreadState.get(currentThread); + ThreadState ts = globalThreadStates.get(currentThread); if (ts != null) { return ts; } @@ -34,16 +58,23 @@ newSystemState = Py.defaultSystemState; } + PySystemStateRef freedRef = (PySystemStateRef)PySystemStateRef.referenceQueue.poll(); + while (freedRef != null ) { + globalThreadStates.remove(inverseGlobalThreadStates.remove(freedRef.getThreadState())); + freedRef = (PySystemStateRef)PySystemStateRef.referenceQueue.poll(); + } ts = new ThreadState(newSystemState); - cachedThreadState.put(currentThread, ts); + globalThreadStates.put(currentThread, ts); + inverseGlobalThreadStates.put(ts, currentThread); return ts; } public static void enterCall(ThreadState ts) { if (ts.call_depth == 0) { - scopedThreadState.get()[0] = ts; -// Thread.currentThread().setContextClassLoader(imp.getSyspathJavaLoader()); - } else if (ts.call_depth > ts.systemState.getrecursionlimit()) { + Object[] scoped = inCallThreadState.get(); + scoped[0] = ts; + scoped[1] = ts.getSystemState(); + } else if (ts.call_depth > ts.getSystemState().getrecursionlimit()) { throw Py.RuntimeError("maximum recursion depth exceeded"); } ts.call_depth++; @@ -52,8 +83,9 @@ public static void exitCall(ThreadState ts) { ts.call_depth--; if (ts.call_depth == 0) { - scopedThreadState.get()[0] = null; -// Thread.currentThread().setContextClassLoader(null); + Object[] scoped = inCallThreadState.get(); + scoped[0] = null; // allow ThreadState to be GCed + scoped[1] = null; // allow corresponding PySystemState to be GCed } } } diff -r 06cfd92ad6a0 src/org/python/core/exceptions.java --- a/src/org/python/core/exceptions.java Mon Dec 14 18:06:57 2015 -0700 +++ b/src/org/python/core/exceptions.java Tue Dec 22 17:28:16 2015 -0700 @@ -27,8 +27,8 @@ dict.__setitem__("__doc__", new PyString(__doc__)); ThreadState ts = Py.getThreadState(); - if (ts.systemState == null) { - ts.systemState = Py.defaultSystemState; + if (ts.getSystemState() == null) { // TODO: is this check still useful?? + ts.setSystemState(Py.defaultSystemState); } // Push frame PyFrame frame = new PyFrame(null, new PyStringMap());