? core/l ? modules/stringBuffer.java2 ? util/JythonBSFEngine.javax ? util/RunJar.javax Index: parser/python.jjt =================================================================== RCS file: /cvsroot/jython/jython/org/python/parser/python.jjt,v retrieving revision 2.13 diff -u -r2.13 python.jjt --- parser/python.jjt 2001/04/25 18:47:19 2.13 +++ parser/python.jjt 2001/07/19 21:37:56 @@ -6,6 +6,7 @@ // only non-defaults are given here. NODE_SCOPE_HOOK = true; // call jjtree*NodeScope() STATIC = false; // multiple parsers + NODE_USES_PARSER = true; COMMON_TOKEN_ACTION = true; // CommonTokenAction(Token) // DEBUG_PARSER = true; // DEBUG_TOKEN_MANAGER = true; @@ -17,6 +18,8 @@ public class PythonGrammar { + public String encoding; + void jjtreeOpenNodeScope(Node n) { ((SimpleNode)n).beginLine = getToken(1).beginLine; ((SimpleNode)n).beginColumn = getToken(1).beginColumn; Index: parser/SimpleNode.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/parser/SimpleNode.java,v retrieving revision 2.13 diff -u -r2.13 SimpleNode.java --- parser/SimpleNode.java 2001/03/08 23:43:51 2.13 +++ parser/SimpleNode.java 2001/07/19 21:37:57 @@ -3,6 +3,7 @@ import org.python.core.Py; import org.python.core.PyString; +import java.io.*; public class SimpleNode implements Node { @@ -81,7 +82,7 @@ info = parseString(s, quotes, beginLine, beginColumn); } - public static String parseString(String s, int quotes, + public String parseString(String s, int quotes, int beginLine, int beginColumn) { //System.out.println("string: "+s); char quoteChar = s.charAt(0); @@ -101,10 +102,60 @@ int i=quotes+start; int last_i=i; - return PyString.decode_UnicodeEscape(s, i, n, "strict", ustring); + if (i == n) + return ""; + InputStream instr = new StringInputStream(s, i, n); + Reader r = null; + if (parser.encoding != null) { + try { + r = new InputStreamReader(instr, parser.encoding); + } catch (UnsupportedEncodingException exc) { ; } + } + if (r == null) { + r = new InputStreamReader(instr); + } + + char buf[] = new char[n*2]; + int slen; + try { + slen = r.read(buf); + } catch (IOException e) { + throw Py.UnicodeError("decoding failed:" + e.getMessage()); + } + s = new String(buf, 0, slen); + + return PyString.decode_UnicodeEscape(s, 0, slen, "strict", ustring); } } + private static class StringInputStream extends InputStream { + private String str; + private int idx; + private int strlen; + + StringInputStream(String str, int off, int len) { + this.str = str; + this.idx = off; + this.strlen = len; + } + + public int read() { + return idx < strlen ? str.charAt(idx++) : -1; + } + + public int read(byte[] buf, int off, int len) { + if (idx >= strlen) { + return -1; + } + if (len > strlen - idx) { + len = strlen - idx; + } + for (int i = 0; i < len; i++) { + buf[off++] = (byte) str.charAt(idx++); + } + return len; + } + } public void setInteger(String s, int radix) { cvs server: Diffing core Index: core/CompilerFlags.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/CompilerFlags.java,v retrieving revision 2.1 diff -u -r2.1 CompilerFlags.java --- core/CompilerFlags.java 2001/05/27 18:49:15 2.1 +++ core/CompilerFlags.java 2001/07/19 21:37:57 @@ -11,4 +11,5 @@ public boolean nested_scopes; + public String encoding; } Index: core/Py.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/Py.java,v retrieving revision 2.49 diff -u -r2.49 Py.java --- core/Py.java 2001/07/19 07:49:50 2.49 +++ core/Py.java 2001/07/19 21:38:03 @@ -1516,7 +1516,7 @@ public static PyCode compile_flags(InputStream istream, String filename, String type,CompilerFlags cflags) { - SimpleNode node = parser.parse(istream, type, filename); + SimpleNode node = parser.parse(istream, type, filename, cflags); boolean printResults = false; if (type.equals("single")) printResults = true; Index: core/imp.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/imp.java,v retrieving revision 2.48 diff -u -r2.48 imp.java --- core/imp.java 2001/07/17 20:34:35 2.48 +++ core/imp.java 2001/07/19 21:38:10 @@ -120,7 +120,7 @@ filename = ""; org.python.parser.SimpleNode node = null; //*Forte* try { - node = parser.parse(fp, "exec", filename); + node = parser.parse(fp, "exec", filename, null); } finally { fp.close(); } Index: core/parser.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/core/parser.java,v retrieving revision 2.9 diff -u -r2.9 parser.java --- core/parser.java 2001/05/15 18:53:35 2.9 +++ core/parser.java 2001/07/19 21:38:10 @@ -67,11 +67,12 @@ public static Node parse(String string, String kind) { - return parse(new StringBufferInputStream(string), kind, ""); + return parse(new StringBufferInputStream(string), kind, + "", null); } public static SimpleNode parse(InputStream istream, String kind, - String filename) + String filename, CompilerFlags cflags) { int nbytes; try { @@ -85,7 +86,7 @@ if (nbytes > 100000) nbytes = 100000; - Reader reader = new InputStreamReader(istream); + Reader reader = new RawInputStreamReader(istream); //if (Options.fixMacReaderBug); reader = new FixMacReaderBug(reader); @@ -96,6 +97,9 @@ } catch (IOException exc) { } PythonGrammar g = new PythonGrammar(bufreader); + if (cflags != null) { + g.encoding = cflags.encoding; + } SimpleNode node = null; try { if (kind.equals("eval")) { @@ -129,19 +133,21 @@ } public static SimpleNode partialParse(String string, String kind, - String filename) + String filename, + CompilerFlags cflags) { SimpleNode node = null; //System.err.println(new PyString(string).__repr__().toString()); try { - node = parse(new StringBufferInputStream(string), kind, filename); + node = parse(new StringBufferInputStream(string), + kind, filename, cflags); } catch (PySyntaxError e) { //System.out.println("e: "+e.lineno+", "+e.column+", "+ // e.forceNewline); try { node = parse(new StringBufferInputStream(string+"\n"), - kind, filename); + kind, filename, cflags); } catch (PySyntaxError e1) { //System.out.println("e1: "+e1.lineno+", "+e1.column+ @@ -151,6 +157,51 @@ return null; } return node; + } +} + +/** + * A InputStreamReader which turns bytes directly into chars. + * It is needed because JavaCC takes input from a Reader but expect the + * data to be bytes. The actual byte-<char transformation occur in + * SimpleNode.parseString. + */ +class RawInputStreamReader extends Reader { + private InputStream is; + private byte[] buf; + + public RawInputStreamReader(InputStream is) { + this.is = is; + this.buf = new byte[1024]; + } + + public int read() throws IOException { + int ret = is.read(); + if (ret == -1) + return ret; + return ret & 0xFF; + } + + public int read(char[] buf, int off, int len) throws IOException { + int cnt = 0; + while (len > 0) { + int l = Math.min(this.buf.length, len); + int n = is.read(this.buf, 0, l); + if (n <= 0) { + if (cnt == 0) return n; + break; + } + for (int i = 0; i < n; i++) { + buf[off++] = (char) (this.buf[i] & 0xFF); + } + len -= n; + cnt += n; + } + return cnt; + } + + public void close() throws IOException { + is.close(); } } cvs server: Diffing modules Index: modules/codeop.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/modules/codeop.java,v retrieving revision 2.5 diff -u -r2.5 codeop.java --- modules/codeop.java 2001/07/16 16:25:04 2.5 +++ modules/codeop.java 2001/07/19 21:38:10 @@ -35,7 +35,7 @@ public static PyObject compile_command_flags(String string,String filename,String kind,CompilerFlags cflags) { org.python.parser.SimpleNode node = - parser.partialParse(string+"\n", kind, filename); + parser.partialParse(string+"\n", kind, filename, cflags); if (node == null) return Py.None; cvs server: Diffing modules/sre cvs server: Diffing util Index: util/jython.java =================================================================== RCS file: /cvsroot/jython/jython/org/python/util/jython.java,v retrieving revision 2.18 diff -u -r2.18 jython.java --- util/jython.java 2001/02/25 17:10:12 2.18 +++ util/jython.java 2001/07/19 21:38:11 @@ -18,6 +18,7 @@ "-jar jar : program read from __run__.py in jar file\n"+ "-c cmd : program passed in as string (terminates option list)\n"+ "-W arg : warning control (arg is action:message:category:module:lineno)\n"+ + "-E codec : Use a different codec the reading from the console.\n"+ "file : program read from script file\n"+ "- : program read from stdin (default; interactive mode if a "+ "tty)\n"+ @@ -180,6 +181,13 @@ if (opts.interactive) { try { + if (opts.encoding == null) { + opts.encoding = PySystemState.registry.getProperty( + "python.console.encoding", null); + } + if (opts.encoding != null) { + interp.cflags.encoding = opts.encoding; + } interp.interact(null); } catch (Throwable t) { Py.printException(t); @@ -200,6 +208,7 @@ public java.util.Properties properties; public String command; public java.util.Vector warnoptions = new java.util.Vector(); + public String encoding; public CommandLineOptions() { filename = null; @@ -264,7 +273,9 @@ } else if (arg.equals("-W")) { warnoptions.addElement(args[++index]); - break; + } + else if (arg.equals("-E")) { + encoding = args[++index]; } else if (arg.startsWith("-D")) { String key = null;