### Eclipse Workspace Patch 1.0 #P jython-trunk Index: src/org/python/indexer/ast/NComprehension.java =================================================================== --- src/org/python/indexer/ast/NComprehension.java (revision 0) +++ src/org/python/indexer/ast/NComprehension.java (revision 0) @@ -0,0 +1,75 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +public class NComprehension extends NNode { + + static final long serialVersionUID = -598250664243757218L; + + public NNode target; + public NNode iter; + public List ifs; + + public NComprehension(NNode target, NNode iter, List ifs) { + this(target, iter, ifs, 0, 1); + } + + public NComprehension(NNode target, NNode iter, List ifs, int start, int end) { + super(start, end); + this.target = target; + this.iter = iter; + this.ifs = ifs; + addChildren(target, iter); + addChildren(ifs); + } + + @Override + public boolean bindsName() { + return true; + } + + @Override + protected void bindNames(Scope s) throws Exception { + bindNames(s, target, NameBinder.make()); + } + + private void bindNames(Scope s, NNode target, NameBinder binder) throws Exception { + if (target instanceof NName) { + binder.bind(s, (NName)target, new NUnknownType()); + return; + } + if (target instanceof NSequence) { + for (NNode n : ((NSequence)target).getElements()) { + bindNames(s, n, binder); + } + } + } + + @Override + public NType resolve(Scope s) throws Exception { + NameBinder.make().bindIter(s, target, iter); + resolveList(ifs, s); + return setType(target.getType()); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(target, v); + visitNode(iter, v); + visitNodeList(ifs, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/__init__.py =================================================================== --- tests/java/org/python/indexer/data/pkg/__init__.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/__init__.py (revision 0) @@ -0,0 +1 @@ +myvalue = [1, 2, 3] Index: src/org/python/antlr/ast/Attribute.java =================================================================== --- src/org/python/antlr/ast/Attribute.java (revision 6695) +++ src/org/python/antlr/ast/Attribute.java (working copy) @@ -45,6 +45,10 @@ public String getInternalAttr() { return attr; } + private Name attrName; + public Name getInternalAttrName() { + return attrName; + } @ExposedGet(name = "attr") public PyObject getAttr() { if (attr == null) return Py.None; @@ -119,6 +123,24 @@ this.ctx = ctx; } + public Attribute(Token token, expr value, Name attr, expr_contextType ctx) { + super(token); + this.value = value; + addChild(value); + this.attr = attr.getText(); + this.attrName = attr; + this.ctx = ctx; + } + + public Attribute(Integer ttype, Token token, expr value, Name attr, expr_contextType ctx) { + super(ttype, token); + this.value = value; + addChild(value); + this.attr = attr.getText(); + this.attrName = attr; + this.ctx = ctx; + } + public Attribute(Integer ttype, Token token, expr value, String attr, expr_contextType ctx) { super(ttype, token); this.value = value; Index: tests/java/org/python/indexer/data/yinw/yinw-17.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-17.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-17.py (revision 0) @@ -0,0 +1,8 @@ +class A: + a = 1 + +class B: + b = A() + +o = B() +print o. b .a Index: src/org/python/indexer/ast/NEllipsis.java =================================================================== --- src/org/python/indexer/ast/NEllipsis.java (revision 0) +++ src/org/python/indexer/ast/NEllipsis.java (revision 0) @@ -0,0 +1,26 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +public class NEllipsis extends NNode { + + static final long serialVersionUID = 4148534089952252511L; + + public NEllipsis() { + } + + public NEllipsis(int start, int end) { + super(start, end); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: src/org/python/indexer/Outliner.java =================================================================== --- src/org/python/indexer/Outliner.java (revision 0) +++ src/org/python/indexer/Outliner.java (revision 0) @@ -0,0 +1,202 @@ +package org.python.indexer; + +import org.python.indexer.NBinding; +import org.python.indexer.Util; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +/** + * Generates a file outline from the index: a structure representing the + * variable and attribute definitions in a file. + */ +public class Outliner { + + public static abstract class Entry { + protected String qname; // entry qualified name + protected int offset; // file offset of referenced declaration + protected NBinding.Kind kind; // binding kind of outline entry + + public Entry() { + } + + public Entry(String qname, int offset, NBinding.Kind kind) { + this.qname = qname; + this.offset = offset; + this.kind = kind; + } + + public abstract boolean isLeaf(); + public Leaf asLeaf() { + return (Leaf)this; + } + + public abstract boolean isBranch(); + public Branch asBranch() { + return (Branch)this; + } + + public abstract boolean hasChildren(); + public abstract List getChildren(); + public abstract void setChildren(List children); + + public String getQname() { + return qname; + } + public void setQname(String qname) { + if (qname == null) { + throw new IllegalArgumentException("qname param cannot be null"); + } + this.qname = qname; + } + + public int getOffset() { + return offset; + } + public void setOffset(int offset) { + this.offset = offset; + } + + public NBinding.Kind getKind() { + return kind; + } + public void setKind(NBinding.Kind kind) { + if (kind == null) { + throw new IllegalArgumentException("kind param cannot be null"); + } + this.kind = kind; + } + /** + * Return simple name. + */ + public String getName() { + String[] parts = qname.split("[.&@%]"); + return parts[parts.length-1]; + } + } + + /** + * An outline entry with children. + */ + public static class Branch extends Entry { + private List children = new ArrayList(); + + public Branch() { + } + public Branch(String qname, int start, NBinding.Kind kind) { + super(qname, start, kind); + } + public boolean isLeaf() { + return false; + } + public boolean isBranch() { + return true; + } + public boolean hasChildren() { + return children != null && !children.isEmpty(); + } + public List getChildren() { + return children; + } + public void setChildren(List children) { + this.children = children; + } + } + + /** + * An entry with no children. + */ + public static class Leaf extends Entry { + public boolean isLeaf() { + return true; + } + public boolean isBranch() { + return false; + } + + public Leaf() { + } + + public Leaf(String qname, int start, NBinding.Kind kind) { + super(qname, start, kind); + } + public boolean hasChildren() { + return false; + } + public List getChildren() { + return new ArrayList(); + } + public void setChildren(List children) { + throw new UnsupportedOperationException("Leaf nodes cannot have children."); + } + } + + /** + * Create an outline for a file in the index. + * @param scope the file scope + * @param path the file for which to build the outline + * @return a list of entries constituting the file outline. + * Returns an empty list if the indexer hasn't indexed that path. + */ + public List generate(Indexer idx, String abspath) throws Exception { + NModuleType mt = idx.getModuleForFile(abspath); + if (mt == null) { + return new ArrayList(); + } + return generate(mt.getTable(), abspath); + } + + /** + * Create an outline for a symbol table. + * @param scope the file scope + * @param path the file for which we're building the outline + * @return a list of entries constituting the outline + */ + public List generate(Scope scope, String path) { + List result = new ArrayList(); + + Set entries = new TreeSet(); + for (NBinding nb : scope.values()) { + if (!nb.isSynthetic() + && !nb.isBuiltin() + && !nb.getDefs().isEmpty() + && path.equals(nb.getSignatureNode().getFile())) { + entries.add(nb); + } + } + + for (NBinding nb : entries) { + Def signode = nb.getSignatureNode(); + List kids = null; + + if (nb.getKind() == NBinding.Kind.CLASS) { + NType realType = nb.followType(); + if (realType.isUnionType()) { + for (NType t : realType.asUnionType().getTypes()) { + if (t.isClassType()) { + realType = t; + break; + } + } + } + kids = generate(realType.getTable(), path); + } + + Entry kid = kids != null ? new Branch() : new Leaf(); + kid.setOffset(signode.start()); + kid.setQname(nb.getQname()); + kid.setKind(nb.getKind()); + + if (kids != null) { + kid.setChildren(kids); + } + result.add(kid); + } + return result; + } +} Index: src/org/python/indexer/ast/NDelete.java =================================================================== --- src/org/python/indexer/ast/NDelete.java (revision 0) +++ src/org/python/indexer/ast/NDelete.java (revision 0) @@ -0,0 +1,49 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +import java.util.List; + +public class NDelete extends NNode { + + static final long serialVersionUID = -2223255555054110766L; + + public List targets; + + public NDelete(List elts) { + this(elts, 0, 1); + } + + public NDelete(List elts, int start, int end) { + super(start, end); + this.targets = elts; + addChildren(elts); + } + + @Override + public NType resolve(Scope s) throws Exception { + for (NNode n : targets) { + resolveExpr(n, s); + if (n instanceof NName) { + s.remove(((NName)n).id); + } + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(targets, v); + } + } +} Index: src/org/python/antlr/ast/arguments.java =================================================================== --- src/org/python/antlr/ast/arguments.java (revision 6695) +++ src/org/python/antlr/ast/arguments.java (working copy) @@ -45,6 +45,12 @@ public String getInternalVararg() { return vararg; } + + private Name varargName; + public Name getInternalVarargName() { + return varargName; + } + @ExposedGet(name = "vararg") public PyObject getVararg() { if (vararg == null) return Py.None; @@ -59,6 +65,12 @@ public String getInternalKwarg() { return kwarg; } + + private Name kwargName; + public Name getInternalKwargName() { + return kwargName; + } + @ExposedGet(name = "kwarg") public PyObject getKwarg() { if (kwarg == null) return Py.None; @@ -117,8 +129,29 @@ setDefaults(defaults); } - public arguments(Token token, java.util.List args, String vararg, String kwarg, - java.util.List defaults) { + // public arguments(Token token, java.util.List args, String vararg, String kwarg, + // java.util.List defaults) { + // super(token); + // this.args = args; + // if (args == null) { + // this.args = new ArrayList(); + // } + // for(PythonTree t : this.args) { + // addChild(t); + // } + // this.vararg = vararg; + // this.kwarg = kwarg; + // this.defaults = defaults; + // if (defaults == null) { + // this.defaults = new ArrayList(); + // } + // for(PythonTree t : this.defaults) { + // addChild(t); + // } + // } + + public arguments(Token token, java.util.List args, Name vararg, Name kwarg, + java.util.List defaults) { super(token); this.args = args; if (args == null) { @@ -127,8 +160,10 @@ for(PythonTree t : this.args) { addChild(t); } - this.vararg = vararg; - this.kwarg = kwarg; + this.vararg = vararg == null ? null : vararg.getText(); + this.varargName = vararg; + this.kwarg = kwarg == null ? null : kwarg.getText(); + this.kwargName = kwarg; this.defaults = defaults; if (defaults == null) { this.defaults = new ArrayList(); Index: src/org/python/indexer/ast/NIndex.java =================================================================== --- src/org/python/indexer/ast/NIndex.java (revision 0) +++ src/org/python/indexer/ast/NIndex.java (revision 0) @@ -0,0 +1,41 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NIndex extends NNode { + + static final long serialVersionUID = -8920941673115420849L; + + public NNode value; + + public NIndex(NNode n) { + this(n, 0, 1); + } + + public NIndex(NNode n, int start, int end) { + super(start, end); + this.value = n; + addChildren(n); + } + + @Override + public NType resolve(Scope s) throws Exception { + return setType(resolveExpr(value, s)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(value, v); + } + } +} Index: src/org/python/indexer/ast/NUnaryOp.java =================================================================== --- src/org/python/indexer/ast/NUnaryOp.java (revision 0) +++ src/org/python/indexer/ast/NUnaryOp.java (revision 0) @@ -0,0 +1,44 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NUnaryOp extends NNode { + + static final long serialVersionUID = 4877088513200468108L; + + public NNode op; + public NNode operand; + + public NUnaryOp(NNode op, NNode n) { + this(op, n, 0, 1); + } + + public NUnaryOp(NNode op, NNode n, int start, int end) { + super(start, end); + this.op = op; + this.operand = n; + addChildren(op, n); + } + + @Override + public NType resolve(Scope s) throws Exception { + return setType(resolveExpr(operand, s)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(op, v); + visitNode(operand, v); + } + } +} Index: src/org/python/indexer/ast/NExceptHandler.java =================================================================== --- src/org/python/indexer/ast/NExceptHandler.java (revision 0) +++ src/org/python/indexer/ast/NExceptHandler.java (revision 0) @@ -0,0 +1,71 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +public class NExceptHandler extends NNode { + + static final long serialVersionUID = 6215262228266158119L; + + public NNode name; + public NNode exceptionType; + public NBlock body; + + public NExceptHandler(NNode name, NNode exceptionType, NBlock body) { + this(name, exceptionType, body, 0, 1); + } + + public NExceptHandler(NNode name, NNode exceptionType, NBlock body, int start, int end) { + super(start, end); + this.name = name; + this.exceptionType = exceptionType; + this.body = body; + addChildren(name, exceptionType, body); + } + + @Override + public boolean bindsName() { + return true; + } + + @Override + protected void bindNames(Scope s) throws Exception { + if (name != null) { + NameBinder.make().bind(s, name, new NUnknownType()); + } + } + + @Override + public NType resolve(Scope s) throws Exception { + NType typeval = new NUnknownType(); + if (exceptionType != null) { + typeval = resolveExpr(exceptionType, s); + } + if (name != null) { + NameBinder.make().bind(s, name, typeval); + } + if (body != null) { + return setType(resolveExpr(body, s)); + } else { + return setType(new NUnknownType()); + } + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(name, v); + visitNode(exceptionType, v); + visitNode(body, v); + } + } +} Index: src/org/python/antlr/RecordingErrorHandler.java =================================================================== --- src/org/python/antlr/RecordingErrorHandler.java (revision 0) +++ src/org/python/antlr/RecordingErrorHandler.java (revision 0) @@ -0,0 +1,65 @@ +package org.python.antlr; + +import org.antlr.runtime.BaseRecognizer; +import org.antlr.runtime.BitSet; +import org.antlr.runtime.IntStream; +import org.antlr.runtime.Lexer; +import org.antlr.runtime.RecognitionException; +import org.python.antlr.ast.ErrorExpr; +import org.python.antlr.ast.ErrorMod; +import org.python.antlr.ast.ErrorSlice; +import org.python.antlr.ast.ErrorStmt; +import org.python.antlr.base.expr; +import org.python.antlr.base.mod; +import org.python.antlr.base.slice; +import org.python.antlr.base.stmt; + +import java.util.ArrayList; +import java.util.List; + +public class RecordingErrorHandler implements ErrorHandler { + + public List errs = new ArrayList(); + + public void reportError(BaseRecognizer br, RecognitionException re) { + br.reportError(re); + errs.add(re); + } + + public void recover(Lexer lex, RecognitionException re) { + lex.recover(re); + } + + public void recover(BaseRecognizer br, IntStream input, RecognitionException re) { + br.recover(input, re); + } + + public boolean mismatch(BaseRecognizer br, IntStream input, int ttype, BitSet follow) { + return true; + } + + public Object recoverFromMismatchedToken(BaseRecognizer br, IntStream input, + int ttype, BitSet follow) { + return null; + } + + public expr errorExpr(PythonTree t) { + return new ErrorExpr(t); + } + + public mod errorMod(PythonTree t) { + return new ErrorMod(t); + } + + public slice errorSlice(PythonTree t) { + return new ErrorSlice(t); + } + + public stmt errorStmt(PythonTree t) { + return new ErrorStmt(t); + } + + public void error(String message, PythonTree t) { + System.err.println(message); + } +} Index: src/org/python/indexer/ast/NQname.java =================================================================== --- src/org/python/indexer/ast/NQname.java (revision 0) +++ src/org/python/indexer/ast/NQname.java (revision 0) @@ -0,0 +1,259 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.Util; +import org.python.indexer.types.NType; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NUnknownType; + +import java.io.File; + +/** + * Recursive doubly-linked list representation of a qualified module name, + * either absolute or relative. Similar to {@link NAttribute}, but handles + * leading dots and other import-specific special cases.

+ * + * Qualified names with leading dots are given {@code NQname} elements for each + * leading dot. Dots separating simple names are not given {@code NQname} + * elements.

+ */ +public class NQname extends NNode { + + static final long serialVersionUID = -5892553606852895686L; + + private NQname next; + private NName name; + + public NQname(NQname next, NName name) { + this(next, name, 0, 1); + } + + public NQname(NQname next, NName name, int start, int end) { + super(start, end); + if (name == null) + throw new IllegalArgumentException("null name"); + this.name = name; + this.next = next; + addChildren(name, next); + } + + /** + * Returns this component of the qname chain. + */ + public NName getName() { + return name; + } + + /** + * Returns the previous component of this qname chain, or {@code null} if + * this is the first component. + */ + public NQname getPrevious() { + NNode parent = getParent(); + if (parent instanceof NQname) { + return (NQname)parent; + } + return null; + } + + /** + * Returns the next component of the chain, or {@code null} if this is + * the last component. + */ + public NQname getNext() { + return next; + } + + /** + * Returns the last/bottom component of the chain. + */ + public NQname getBottom() { + return next == null ? this : next.getBottom(); + } + + /** + * Returns {@code true} if this is the first/top component of the chain, + * or if the name is unqualified (i.e. has only one component, no dots). + */ + public boolean isTop() { + return getPrevious() == null; + } + + /** + * Returns {@code true} if this is the last/bottom component of the chain, + * or if the name is unqualified (i.e. has only one component, no dots). + */ + public boolean isBottom() { + return next == null; + } + + /** + * Returns {@code true} if this qname represents a simple, non-dotted module + * name such as "os", "random" or "foo". + */ + public boolean isUnqualified() { + return isTop() && isBottom(); + } + + /** + * Joins all components in this qname chain, beginning with the + * current component. + */ + public String toQname() { + return isBottom() ? name.id : name.id + "." + next.toQname(); + } + + /** + * Returns the qname down to (and including) this component, ending + * with this component's name. For instance, if this {@code NQname} + * instance represents the {@code foo} in {@code org.foo.bar}, this + * method will return {@code org.foo}. + */ + public String thisQname() { + NQname n = getTop(); + StringBuilder sb = new StringBuilder(); + sb.append(n.name.id); + while (n != this) { + sb.append("."); + n = n.next; + sb.append(n.name.id); + } + return sb.toString(); + } + + /** + * Returns the top (first) component in the chain. + */ + public NQname getTop() { + return isTop() ? this : getPrevious().getTop(); + } + + /** + * Returns {@code true} if this qname component is a leading dot. + */ + public boolean isDot() { + return ".".equals(name.id); + } + + /** + * Resolves and loads the module named by this qname. + * @return the module represented by the qname up to this point. + */ + @Override + public NType resolve(Scope s) throws Exception { + setType(name.setType(new NUnknownType())); + + // Check for top-level native or standard module. + if (isUnqualified()) { + NModuleType mt = Indexer.idx.loadModule(name.id); + if (mt != null) { + return setType(name.setType(mt)); + } + } else { + // Check for second-level builtin such as "os.path". + NModuleType mt = Indexer.idx.getBuiltinModule(thisQname()); + if (mt != null) { + setType(name.setType(mt)); + resolveExpr(next, s); + return mt; + } + } + + return resolveInFilesystem(s); + } + + private NType resolveInFilesystem(Scope s) throws Exception { + NModuleType start = getStartModule(s); + if (start == null) { + reportUnresolvedModule(); + return getType(); + } + + String qname = start.getTable().getPath(); + String relQname; + if (isDot()) { + relQname = Util.getQnameParent(qname); + } else if (!isTop()) { + relQname = qname + "." + name.id; + } else { + // top name: first look in current dir, then sys.path + String dirQname = isInitPy() ? qname : Util.getQnameParent(qname); + relQname = dirQname + "." + name.id; + if (Indexer.idx.loadModule(relQname) == null) { + relQname = name.id; + } + } + + NModuleType mod = Indexer.idx.loadModule(relQname); + if (mod == null) { + reportUnresolvedModule(); + return getType(); + } + setType(name.setType(mod)); + + if (!isTop() && mod.getFile() != null) { + Scope parentPkg = getPrevious().getTable(); + NBinding mb = Indexer.idx.moduleTable.lookup(mod.getFile()); + parentPkg.put(name.id, mb); + } + + resolveExpr(next, s); + return getType(); + } + + private boolean isInitPy() { + String path = getFile(); + if (path == null) { + return false; + } + return new File(path).getName().equals("__init__.py"); + } + + private NModuleType getStartModule(Scope s) throws Exception { + if (!isTop()) { + return getPrevious().getType().asModuleType(); + } + + // Start with module for current file (i.e. containing directory). + + NModuleType start = null; + Scope mtable = s.getSymtabOfType(Scope.Type.MODULE); + if (mtable != null) { + start = Indexer.idx.loadModule(mtable.getPath()); + if (start != null) { + return start; + } + } + + String dir = new File(getFile()).getParent(); + if (dir == null) { + Indexer.idx.warn("Unable to find parent dir for " + getFile()); + return null; + } + + return Indexer.idx.loadModule(dir); + } + + private void reportUnresolvedModule() { + addError("module not found: " + name.id); + Indexer.idx.recordUnresolvedModule(thisQname(), getFile()); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(next, v); + visitNode(name, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/test.py =================================================================== --- tests/java/org/python/indexer/data/pkg/test.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/test.py (revision 0) @@ -0,0 +1,4 @@ +import animal.mammal.dog as dog + +def hi(): + print dog.bark Index: src/org/python/indexer/demos/Styler.java =================================================================== --- src/org/python/indexer/demos/Styler.java (revision 0) +++ src/org/python/indexer/demos/Styler.java (revision 0) @@ -0,0 +1,238 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.demos; + +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.BaseRecognizer; +import org.antlr.runtime.CommonToken; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.RecognizerSharedState; +import org.antlr.runtime.Token; +import org.python.antlr.PythonLexer; +import org.python.antlr.PythonTree; +import org.python.antlr.RecordingErrorHandler; +import org.python.indexer.Def; +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.StyleRun; +import org.python.indexer.ast.DefaultNodeVisitor; +import org.python.indexer.ast.NAssign; +import org.python.indexer.ast.NFunctionDef; +import org.python.indexer.ast.NModule; +import org.python.indexer.ast.NName; +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NNum; +import org.python.indexer.ast.NStr; +import org.python.indexer.ast.NUrl; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Decorates Python source with style runs from the index. + */ +class Styler extends DefaultNodeVisitor { + + static final Pattern BUILTIN = + Pattern.compile("None|True|False|NotImplemented|Ellipsis|__debug__"); + + /** + * Matches the start of a triple-quote string. + */ + private static final Pattern TRISTRING_PREFIX = + Pattern.compile("^[ruRU]{0,2}['\"]{3}"); + + private Indexer indexer; + private String source; + private String path; + private List styles = new ArrayList(); + private Linker linker; + + /** Offsets of doc strings found by node visitor. */ + private Set docOffsets = new HashSet(); + + public Styler(Indexer idx, Linker linker) { + this.indexer = idx; + this.linker = linker; + } + + /** + * @param path absolute file path + * @param src file contents + */ + public List addStyles(String path, String src) throws Exception { + this.path = path; + source = src; + NModule m = indexer.getAstForFile(path); + m.visit(this); + highlightLexicalTokens(); + return styles; + } + + @Override + public boolean visit(NName n) { + NNode parent = n.getParent(); + if (parent instanceof NFunctionDef) { + NFunctionDef fn = (NFunctionDef)parent; + if (n == fn.name) { + addStyle(n, StyleRun.Type.FUNCTION); + } else if (n == fn.kwargs || n == fn.varargs) { + addStyle(n, StyleRun.Type.PARAMETER); + } + return true; + } + + if (BUILTIN.matcher(n.id).matches()) { + addStyle(n, StyleRun.Type.BUILTIN); + return true; + } + + return true; + } + + @Override + public boolean visit(NNum n) { + addStyle(n, StyleRun.Type.NUMBER); + return true; + } + + @Override + public boolean visit(NStr n) { + String s = sourceString(n.start(), n.end()); + if (TRISTRING_PREFIX.matcher(s).lookingAt()) { + addStyle(n.start(), n.end() - n.start(), StyleRun.Type.DOC_STRING); + docOffsets.add(n.start()); // don't re-highlight as a string + highlightDocString(n); + } + return true; + } + + private void highlightDocString(NStr node) { + String s = sourceString(node.start(), node.end()); + DocStringParser dsp = new DocStringParser(s, node, linker); + dsp.setResolveReferences(true); + styles.addAll(dsp.highlight()); + } + + /** + * Use scanner to find keywords, strings and comments. + */ + private void highlightLexicalTokens() { + RecognizerSharedState state = new RecognizerSharedState(); + state.errorRecovery = true; // don't issue 10 billion errors + + PythonLexer lex = new PythonLexer( + new ANTLRStringStream(source) { + @Override + public String getSourceName() { + return path; + } + }, + state); + + lex.setErrorHandler(new RecordingErrorHandler() { + @Override + public void error(String message, PythonTree t) { + // don't println + } + @Override + public void reportError(BaseRecognizer br, RecognitionException re) { + // don't println + } + }); + + Token tok; + while ((tok = lex.nextToken()) != Token.EOF_TOKEN) { + switch (tok.getType()) { + case PythonLexer.STRING: { + int beg = ((CommonToken)tok).getStartIndex(); + int end = ((CommonToken)tok).getStopIndex(); + if (!docOffsets.contains(beg)) { + addStyle(beg, end - beg + 1, StyleRun.Type.STRING); + } + break; + } + case PythonLexer.COMMENT: { + int beg = ((CommonToken)tok).getStartIndex(); + int end = ((CommonToken)tok).getStopIndex(); + String comment = tok.getText(); + addStyle(beg, end - beg + 1, StyleRun.Type.COMMENT); + break; + } + case PythonLexer.AND: + case PythonLexer.AS: + case PythonLexer.ASSERT: + case PythonLexer.BREAK: + case PythonLexer.CLASS: + case PythonLexer.CONTINUE: + case PythonLexer.DEF: + case PythonLexer.DELETE: + case PythonLexer.ELIF: + case PythonLexer.EXCEPT: + case PythonLexer.FINALLY: + case PythonLexer.FOR: + case PythonLexer.FROM: + case PythonLexer.GLOBAL: + case PythonLexer.IF: + case PythonLexer.IMPORT: + case PythonLexer.IN: + case PythonLexer.IS: + case PythonLexer.LAMBDA: + case PythonLexer.NOT: + case PythonLexer.OR: + case PythonLexer.ORELSE: + case PythonLexer.PASS: + case PythonLexer.PRINT: + case PythonLexer.RAISE: + case PythonLexer.RETURN: + case PythonLexer.TRY: + case PythonLexer.WHILE: + case PythonLexer.WITH: + case PythonLexer.YIELD: { + int beg = ((CommonToken)tok).getStartIndex(); + int end = ((CommonToken)tok).getStopIndex(); + addStyle(beg, end - beg + 1, StyleRun.Type.KEYWORD); + break; + } + } + } + } + + private void addStyle(NNode e, int start, int len, StyleRun.Type type) { + if (e == null || e.getFile() == null) { // if it's an NUrl, for instance + return; + } + addStyle(start, len, type); + } + + private void addStyle(NNode e, StyleRun.Type type) { + if (e != null) { + addStyle(e, e.start(), e.end() - e.start(), type); + } + } + + private void addStyle(int beg, int len, StyleRun.Type type) { + styles.add(new StyleRun(type, beg, len)); + } + + private String sourceString(NNode e) { + return sourceString(e.start(), e.end()); + } + + private String sourceString(int beg, int end) { + int a = Math.max(beg, 0); + int b = Math.min(end, source.length()); + b = Math.max(b, 0); + try { + return source.substring(a, b); + } catch (StringIndexOutOfBoundsException sx) { + System.out.println("whoops: beg=" + a + ", end=" + b + ", len=" + source.length()); + return ""; + } + } +} Index: tests/java/org/python/indexer/data/pkg/misc/moduleA.py =================================================================== --- tests/java/org/python/indexer/data/pkg/misc/moduleA.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/misc/moduleA.py (revision 0) @@ -0,0 +1,5 @@ +__all__ = ['a', 'b', 'c'] +a = "1" +b = "2" +c = "3" +d = "4" Index: tests/java/org/python/indexer/data/pkg/other/color/red.py =================================================================== --- tests/java/org/python/indexer/data/pkg/other/color/red.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/other/color/red.py (revision 0) @@ -0,0 +1,3 @@ +r = 255 +g = 0 +b = 0 Index: tests/java/org/python/indexer/data/yinw/yinw-15.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-15.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-15.py (revision 0) @@ -0,0 +1,30 @@ +def purge(): + "Clear the regular expression cache" + _cache.clear() + +_cache = {} +_cache_repl = {} + +_pattern_type = type(sre_compile.compile("", 0)) + +_MAXCACHE = 100 + +def _compile(*key): + # internal: compile pattern + cachekey = (type(key[0]),) + key + p = _cache.get(cachekey) + if p is not None: + return p + pattern, flags = key + if isinstance(pattern, _pattern_type): + return pattern + if not sre_compile.isstring(pattern): + raise TypeError, "first argument must be string or compiled pattern" + try: + p = sre_compile.compile(pattern, flags) + except error, v: + raise error, v # invalid expression + if len(_cache) >= _MAXCACHE: + _cache.clear() + _cache[cachekey] = p + return p Index: src/org/python/indexer/ast/NAlias.java =================================================================== --- src/org/python/indexer/ast/NAlias.java (revision 0) +++ src/org/python/indexer/ast/NAlias.java (revision 0) @@ -0,0 +1,90 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +/** + * A name alias. Used for the components of import and import-from statements. + */ +public class NAlias extends NNode { + + static final long serialVersionUID = 4127878954298987559L; + + /** + * Represents one of three possibilities: + *

    + *
  1. import stmt: qname of one of the imported modules
  2. + *
  3. import-from stmt: qname of the referenced module
  4. + *
  5. import-from stmt: simple name imported from referenced module
  6. + *
+ */ + public NQname qname; // ["." ["." ["." ["a" ["b" ["c"]]]]]] + + /** + * The un-parsed qname or simple name corresponding to {@link #qname}. + */ + public String name; // "...a.b.c" + + /** + * The alias name, if an "as" clause was specified. + */ + public NName aname; // "bar" in "... foo as bar" + + public NAlias(String name, NQname qname, NName aname) { + this(name, qname, aname, 0, 1); + } + + public NAlias(String name, NQname qname, NName aname, int start, int end) { + super(start, end); + this.qname = qname; + this.name = name; + this.aname = aname; + addChildren(qname, aname); + } + + /** + * Returns the alias, if non-{@code null}, else the simple name. + */ + public String getBoundName() { + return aname != null ? aname.id : name; + } + + /** + * Resolves and returns the referenced + * {@link org.python.indexer.types.NModuleType} in an import or + * or import-from statement. NImportFrom statements manually + * resolve their child NAliases. + */ + @Override + public NType resolve(Scope s) throws Exception { + setType(resolveExpr(qname, s)); + + // "import a.b.c" defines 'a' (the top module) in the scope, whereas + // "import a.b.c as x" defines 'x', which refers to the bottom module. + if (aname != null && qname != null) { + setType(qname.getBottom().getType()); + aname.setType(getType()); + } + + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(qname, v); + visitNode(aname, v); + } + } +} Index: src/org/python/antlr/ast/ImportFrom.java =================================================================== --- src/org/python/antlr/ast/ImportFrom.java (revision 6695) +++ src/org/python/antlr/ast/ImportFrom.java (working copy) @@ -1,4 +1,4 @@ -// Autogenerated AST node +// Autogenerated AST node -*- c-basic-offset:4 -*- package org.python.antlr.ast; import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; @@ -24,6 +24,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.List; @ExposedType(name = "_ast.ImportFrom", base = AST.class) public class ImportFrom extends stmt { @@ -42,8 +43,8 @@ this.module = AstAdapters.py2identifier(module); } - private java.util.List names; - public java.util.List getInternalNames() { + private List names; + public List getInternalNames() { return names; } @ExposedGet(name = "names") @@ -55,6 +56,11 @@ this.names = AstAdapters.py2aliasList(names); } + private List moduleNames; + public List getInternalModuleNames() { + return moduleNames; + } + private Integer level; public Integer getInternalLevel() { return level; @@ -111,8 +117,10 @@ setLevel(level); } - public ImportFrom(Token token, String module, java.util.List names, Integer level) { - super(token); + public ImportFrom(int ttype, Token token, + String module, List moduleNames, + List names, Integer level) { + super(ttype, token); this.module = module; this.names = names; if (names == null) { @@ -121,10 +129,17 @@ for(PythonTree t : this.names) { addChild(t); } + this.moduleNames = moduleNames; + if (moduleNames == null) { + this.moduleNames = new ArrayList(); + } + for(PythonTree t : this.moduleNames) { + addChild(t); + } this.level = level; } - public ImportFrom(Integer ttype, Token token, String module, java.util.List names, + public ImportFrom(Integer ttype, Token token, String module, List names, Integer level) { super(ttype, token); this.module = module; @@ -138,7 +153,7 @@ this.level = level; } - public ImportFrom(PythonTree tree, String module, java.util.List names, Integer level) { + public ImportFrom(PythonTree tree, String module, List names, Integer level) { super(tree); this.module = module; this.names = names; Index: src/org/python/indexer/Ref.java =================================================================== --- src/org/python/indexer/Ref.java (revision 0) +++ src/org/python/indexer/Ref.java (revision 0) @@ -0,0 +1,215 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import org.python.indexer.ast.NAttribute; +import org.python.indexer.ast.NName; +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NStr; + +/** + * Encapsulates information about a binding reference. + */ +public class Ref { + + private static final int ATTRIBUTE = 0x1; + private static final int CALL = 0x2; // function/method call + private static final int NEW = 0x4; // instantiation + private static final int STRING = 0x8; // source node is a String + + private int start; + private String file; + private String name; + private int flags; + + public Ref(NNode node) { + if (node == null) { + throw new IllegalArgumentException("null node"); + } + file = node.getFile(); + start = node.start(); + + if (node instanceof NName) { + NName nn = ((NName)node); + name = nn.id; + if (nn.isCall()) { + // We don't always have enough information at this point to know + // if it's a constructor call or a regular function/method call, + // so we just determine if it looks like a call or not, and the + // indexer will convert constructor-calls to NEW in a later pass. + markAsCall(); + } + } else if (node instanceof NStr) { + markAsString(); + name = ((NStr)node).n.toString(); + } else { + throw new IllegalArgumentException("I don't know what " + node + " is."); + } + + NNode parent = node.getParent(); + if ((parent instanceof NAttribute) + && node == ((NAttribute)parent).attr) { + markAsAttribute(); + } + } + + /** + * Constructor that provides a way for clients to add additional references + * not associated with an AST node (e.g. inside a comment or doc string). + * @param path absolute path to the file containing the reference + * @param offset the 0-indexed file offset of the reference + * @param text the text of the reference + */ + public Ref(String path, int offset, String text) { + if (path == null) { + throw new IllegalArgumentException("'path' cannot be null"); + } + if (text == null) { + throw new IllegalArgumentException("'text' cannot be null"); + } + file = path; + start = offset; + name = text; + } + + /** + * Returns the file containing the reference. + */ + public String getFile() { + return file; + } + + /** + * Returns the text of the reference. + */ + public String getName() { + return name; + } + + /** + * Returns the starting file offset of the reference. + */ + public int start() { + return start; + } + + /** + * Returns the ending file offset of the reference. + */ + public int end() { + return start + length(); + } + + /** + * Returns the length of the reference text. + */ + public int length() { + return isString() ? name.length() + 2 : name.length(); + } + + /** + * Returns {@code true} if this reference was unquoted name. + */ + public boolean isName() { + return !isString(); + } + + /** + * Returns {@code true} if this reference was an attribute + * of some other node. + */ + public boolean isAttribute() { + return (flags & ATTRIBUTE) != 0; + } + + public void markAsAttribute() { + flags |= ATTRIBUTE; + } + + /** + * Returns {@code true} if this reference was a quoted name. + * If so, the {@link #start} and {@link #length} include the positions + * of the opening and closing quotes, but {@link #isName} returns the + * text within the quotes. + */ + public boolean isString() { + return (flags & STRING) != 0; + } + + public void markAsString() { + flags |= STRING; + } + + /** + * Returns {@code true} if this reference is a function or method call. + */ + public boolean isCall() { + return (flags & CALL) != 0; + } + + /** + * Returns {@code true} if this reference is a class instantiation. + */ + public void markAsCall() { + flags |= CALL; + flags &= ~NEW; + } + + public boolean isNew() { + return (flags & NEW) != 0; + } + + public void markAsNew() { + flags |= NEW; + flags &= ~CALL; + } + + public boolean isRef() { + return !(isCall() || isNew()); + } + + @Override + public String toString() { + return ""; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Ref)) { + return false; + } + Ref ref = (Ref)obj; + if (start != ref.start) { + return false; + } + if (name != null) { + if (!name.equals(ref.name)) { + return false; + } + } else { + if (ref.name != null) { + return false; + } + } + if (file != null) { + if (!file.equals(ref.file)) { + return false; + } + } else { + if (ref.file != null) { + return false; + } + } + // This should never happen, but checking it here can help surface bugs. + if (flags != ref.flags) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return ("" + file + name + start).hashCode(); + } +} Index: src/org/python/antlr/ast/FunctionDef.java =================================================================== --- src/org/python/antlr/ast/FunctionDef.java (revision 6695) +++ src/org/python/antlr/ast/FunctionDef.java (working copy) @@ -32,6 +32,10 @@ public String getInternalName() { return name; } + private Name nameNode; + public Name getInternalNameNode() { + return nameNode; + } @ExposedGet(name = "name") public PyObject getName() { if (name == null) return Py.None; @@ -148,6 +152,28 @@ } } + public FunctionDef(Token token, Name name, arguments args, java.util.List body, + java.util.List decorator_list) { + super(token); + this.name = name.getText(); + this.nameNode = name; + this.args = args; + this.body = body; + if (body == null) { + this.body = new ArrayList(); + } + for(PythonTree t : this.body) { + addChild(t); + } + this.decorator_list = decorator_list; + if (decorator_list == null) { + this.decorator_list = new ArrayList(); + } + for(PythonTree t : this.decorator_list) { + addChild(t); + } + } + public FunctionDef(Integer ttype, Token token, String name, arguments args, java.util.List body, java.util.List decorator_list) { super(ttype, token); Index: tests/java/org/python/indexer/data/yinw/yinw-2.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-2.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-2.py (revision 0) @@ -0,0 +1,17 @@ +class A: + a = 1 + +class B: + a = 2 + +l = [A(), A(), B()] + +print l[0].a + +# locations: +# = [] +# = [] +# = [] +# = [] +# :>>.a:9:6> = [, ] + Index: tests/java/org/python/indexer/data/refs2.py =================================================================== --- tests/java/org/python/indexer/data/refs2.py (revision 0) +++ tests/java/org/python/indexer/data/refs2.py (revision 0) @@ -0,0 +1,10 @@ +class Foo: + a = 1 + +class Bar(Foo): + b = 2 + +Foo() + +x = Bar +x() Index: src/org/python/indexer/ast/NPass.java =================================================================== --- src/org/python/indexer/ast/NPass.java (revision 0) +++ src/org/python/indexer/ast/NPass.java (revision 0) @@ -0,0 +1,26 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +public class NPass extends NNode { + + static final long serialVersionUID = 3668786487029793620L; + + public NPass() { + } + + public NPass(int start, int end) { + super(start, end); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: .classpath =================================================================== --- .classpath (revision 6976) +++ .classpath (working copy) @@ -12,7 +12,6 @@ - Index: tests/java/org/python/indexer/data/yinw/yinw-9.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-9.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-9.py (revision 0) @@ -0,0 +1,19 @@ +class A: + a = 1 + +def f(x): + if (x>1): return f(x-1) + else: return A() + +print f(2).a + + +# test of recursive functions + +# locations: +# = [] +# = [] +# = [] +# = [] +# = [] +# :[]:8:6>.a:8:6> = [] Index: src/org/python/indexer/types/NDictType.java =================================================================== --- src/org/python/indexer/types/NDictType.java (revision 0) +++ src/org/python/indexer/types/NDictType.java (revision 0) @@ -0,0 +1,61 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.ast.NUrl; + +public class NDictType extends NType { + + private NType keyType; + private NType valueType; + + public NDictType() { + this(new NUnknownType(), new NUnknownType()); + } + + public NDictType(NType key0, NType val0) { + keyType = key0; + valueType = val0; + getTable().addSuper(Indexer.idx.builtins.BaseDict.getTable()); + getTable().setPath(Indexer.idx.builtins.BaseDict.getTable().getPath()); + } + + public void setKeyType(NType keyType) { + this.keyType = keyType; + } + + public NType getKeyType() { + return keyType; + } + + public void setValueType(NType valType) { + this.valueType = valType; + } + + public NType getValueType() { + return valueType; + } + + public void add(NType key, NType val) { + keyType = NUnionType.union(keyType, key); + valueType = NUnionType.union(valueType, val); + } + + public NTupleType toTupleType(int n) { + NTupleType ret = new NTupleType(); + for (int i = 0; i < n; i++) { + ret.add(keyType); + } + return ret; + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + keyType.print(ctr, sb); + sb.append(":"); + valueType.print(ctr, sb); + } +} Index: src/org/python/antlr/PythonTree.java =================================================================== --- src/org/python/antlr/PythonTree.java (revision 6823) +++ src/org/python/antlr/PythonTree.java (working copy) @@ -5,6 +5,7 @@ import org.antlr.runtime.tree.CommonTree; import org.python.core.PyType; +import org.python.antlr.ast.Name; import org.python.antlr.ast.VisitorIF; import java.util.ArrayList; @@ -165,6 +166,35 @@ node.setChildIndex(index); } + /** + * Converts a list of Name to a dotted-name string. + * Because leading dots are indexable identifiers (referring + * to parent directories in relative imports), a Name list + * may include leading dots, but not dots between names. + */ + public static String dottedNameListToString(List names) { + if (names == null) { + return ""; + } + StringBuilder sb = new StringBuilder(); + boolean leadingDot = true; + for (int i = 0, len = names.size(); i < len; i++) { + Name name = names.get(i); + String id = name.getInternalId(); + if (id == null) { + continue; + } + if (!".".equals(id)) { + leadingDot = false; + } + sb.append(id); + if (i < len - 1 && !leadingDot) { + sb.append("."); + } + } + return sb.toString(); + } + @Override public String toString() { if (isNil()) { Index: tests/java/org/python/indexer/data/mod3.py =================================================================== --- tests/java/org/python/indexer/data/mod3.py (revision 0) +++ tests/java/org/python/indexer/data/mod3.py (revision 0) @@ -0,0 +1,7 @@ + +import distutils + +distutils.foo = "bar" + +def mod3test(): + return dir(distutils) Index: tests/java/org/python/indexer/data/pkg/plant/garden/rose.py =================================================================== Index: tests/java/org/python/indexer/data/yinw/yinw-7.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-7.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-7.py (revision 0) @@ -0,0 +1,9 @@ +def commonprefix(m): + "Given a list of pathnames, returns the longest common leading component" + if not m: return '' + s1 = min(m) + s2 = max(m) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 Index: src/org/python/antlr/GrammarActions.java =================================================================== --- src/org/python/antlr/GrammarActions.java (revision 6959) +++ src/org/python/antlr/GrammarActions.java (working copy) @@ -61,19 +61,41 @@ this.errorHandler = eh; } - String makeFromText(List dots, String name) { - StringBuffer d = new StringBuffer(); + String makeFromText(List dots, List names) { + StringBuilder d = new StringBuilder(); if (dots != null) { for (int i=0;i makeModuleNameNode(List dots, List names) { + List result = new ArrayList(); + if (dots != null) { + for (Object o : dots) { + Token tok = (Token)o; + result.add(new Name(tok, tok.getText(), expr_contextType.Load)); + } + } + result.addAll(names); + return result; + } + + List makeDottedName(Token top, List attrs) { + List result = new ArrayList(); + result.add(new Name(top, top.getText(), expr_contextType.Load)); + if (attrs != null) { + for (PythonTree attr : attrs) { + Token token = attr.getToken(); + result.add(new Name(token, token.getText(), expr_contextType.Load)); + } + } + return result; + } + int makeLevel(List lev) { if (lev == null) { return 0; @@ -113,6 +135,21 @@ return s; } + Name makeNameNode(Token t) { + if (t == null) { + return null; + } + return new Name(t, t.getText(), expr_contextType.Load); + } + + List makeNameNodes(List names) { + List s = new ArrayList(); + for (int i=0; i f = castStmts(finBody); return new TryFinally(t, b, f); } - + stmt makeFuncdef(Token t, Token nameToken, arguments args, List funcStatements, List decorators) { if (nameToken == null) { return errorHandler.errorStmt(new PythonTree(t)); } - cantBeNone(nameToken); + Name n = cantBeNoneName(nameToken); arguments a; if (args != null) { a = args; } else { - a = new arguments(t, new ArrayList(), null, null, new ArrayList()); + a = new arguments(t, new ArrayList(), (Name)null, null, new ArrayList()); } List s = castStmts(funcStatements); List d = castExprs(decorators); - return new FunctionDef(t, nameToken.getText(), a, s, d); + return new FunctionDef(t, n, a, s, d); } List makeAssignTargets(expr lhs, List rhs) { @@ -293,17 +330,17 @@ List p = castExprs(params); List d = castExprs(defaults); - String s; - String k; + Name s; + Name k; if (snameToken == null) { s = null; } else { - s = cantBeNone(snameToken); + s = cantBeNoneName(snameToken); } if (knameToken == null) { k = null; } else { - k = cantBeNone(knameToken); + k = cantBeNoneName(knameToken); } return new arguments(t, p, s, k, d); } @@ -516,6 +553,13 @@ return t.getText(); } + Name cantBeNoneName(Token t) { + if (t == null || t.getText().equals("None")) { + errorHandler.error("can't be None", new PythonTree(t)); + } + return new Name(t, t.getText(), expr_contextType.Load); + } + void cantBeNone(PythonTree e) { if (e.getText().equals("None")) { errorHandler.error("can't be None", e); @@ -722,18 +766,4 @@ } return s; } - - public String makeDottedText(Token name, List c) { - final String dot = "."; - if (c == null || c.isEmpty()) { - return name.getText(); - } - StringBuilder b = new StringBuilder(name.getText()); - for (PythonTree t : c) { - b.append(dot); - b.append(t.getToken().getText()); - } - return b.toString(); - } - } Index: src/org/python/indexer/types/NUnionType.java =================================================================== --- src/org/python/indexer/types/NUnionType.java (revision 0) +++ src/org/python/indexer/types/NUnionType.java (revision 0) @@ -0,0 +1,235 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; + +import java.util.HashSet; +import java.util.Set; + +/** + * A union type is a set of several other types. During a union operation, + * destructuring happens and unknown types are unified. + */ +public class NUnionType extends NType { + + /** + * Union types can lead to infinite recursion in the occurs check. Until + * we've got a handle on these cases, I'm limiting the recursion depth. + * + * @see http://www.cs.kuleuven.ac.be/~dtai/projects/ALP/newsletter/archive_93_96/net/impl/occur.html + * for an interesting and highly relevant discussion. + */ + private static final int MAX_RECURSION_DEPTH = 15; + + private Set types; + + public NUnionType() { + this.types = new HashSet(); + } + + public NUnionType(NType... initialTypes) { + this(); + for (NType nt : initialTypes) { + addType(nt); + } + } + + public void setTypes(Set types) { + this.types = types; + } + + public Set getTypes() { + return types; + } + + public void addType(NType t) { + if (t == null) { + throw new IllegalArgumentException("null type"); + } + if (t.isUnionType()) { + types.addAll(t.asUnionType().types); + } else { + types.add(t); + } + } + + public boolean contains(NType t) { + return types.contains(t); + } + + public static NType union(NType u, NType v) { + NType wu = NUnknownType.follow(u); + NType wv = NUnknownType.follow(v); + if (wu == wv) { + return u; + } + + // This is a bit unconventional, as most type inferencers try to + // determine whether a given name can ever take a null value. However, + // doing so complicates the logic and proliferates union types, arguably + // with little benefit for Python. So for now, X|None => X. + if (wu == Indexer.idx.builtins.None) { + return v; + } + if (wv == Indexer.idx.builtins.None) { + return u; + } + + if (wu.isUnknownType() && !occurs(wu, wv, 0)) { + NUnknownType.point(wu, wv); + return u; + } + + if (wv.isUnknownType() && !occurs(wv, wu, 0)) { + NUnknownType.point(wv, wu); + return v; + } + if (wu.isTupleType() && wv.isTupleType()) { + NTupleType tu = (NTupleType)wu; + NTupleType tv = (NTupleType)wv; + if (tu.getElementTypes().size() == tv.getElementTypes().size()) { + NTupleType ret = new NTupleType(); + for (int i = 0; i < tu.getElementTypes().size(); i++) { + ret.add(union(tu.getElementTypes().get(i), tv.getElementTypes().get(i))); + } + return ret; + } + return newUnion(wu, wv); + } + if (wu.isListType() && wv.isListType()) { + return new NListType(union(wu.asListType().getElementType(), + wv.asListType().getElementType())); + } + if (wu.isDictType() && wv.isDictType()) { + NDictType du = (NDictType)wu; + NDictType dv = (NDictType)wv; + return new NDictType(union(du.getKeyType(), dv.getKeyType()), + union(du.getValueType(), dv.getValueType())); + } + if (wu.isFuncType() && wv.isFuncType()) { + return new NFuncType(NUnionType.union(wu.asFuncType().getReturnType(), + wv.asFuncType().getReturnType())); + } + + // XXX: see comments in NInstanceType + if (wu.isFuncType() && wv.isClassType()) { + // NUnknownType.point(wu.asFuncType().getReturnType(), new NInstanceType(wv)); + NUnknownType.point(wu.asFuncType().getReturnType(), wv); + NUnknownType.point(u, wv); + return u; + } + if (wu.isClassType() && wv.isFuncType()) { + // NUnknownType.point(wv.asFuncType().getReturnType(), new NInstanceType(wu)); + NUnknownType.point(wv.asFuncType().getReturnType(), wu); + NUnknownType.point(v, wu); + return v; + } + + return newUnion(wu, wv); + } + + /** + * @see http://en.wikipedia.org/wiki/Occurs_check + */ + private static boolean occurs(NType u, NType v, int depth) { + if (depth++ > MAX_RECURSION_DEPTH) { + return true; + } + + u = NUnknownType.follow(u); + v = NUnknownType.follow(v); + if (u == v) { + return true; + } + + if (v.isTupleType()) { + for (NType vv : v.asTupleType().getElementTypes()) { + if (occurs(u, vv, depth)) { + return true; + } + } + return false; + } + + if (v.isListType()) { + return occurs(u, v.asListType().getElementType(), depth); + } + + if (v.isDictType()) { + return occurs(u, v.asDictType().getKeyType(), depth) + || occurs(u, v.asDictType().getValueType(), depth); + } + + if (v.isFuncType()) { + // A function type appearing in its own return type can happen + // (e.g. def foo(): return [foo]), and causes infinite recursion if + // we don't check for it + NType ret = v.asFuncType().getReturnType(); + if (occurs(v, ret, depth)) { + return true; + } + return occurs(u, ret, depth); + } + + if (v.isUnionType()) { + for (NType vv : v.asUnionType().types) { + if (occurs(u, vv, depth)) { + return true; + } + } + return false; + } + + return false; + } + + public static NUnionType newUnion(NType... types) { + NUnionType ret = new NUnionType(); + for (NType type : types) { + ret.addType(type); + } + return ret; + } + + /** + * Returns the first alternate whose type is not unknown. + * @return the first non-unknown alternate, or {@code null} if none found + */ + public NType firstKnownAlternate() { + for (NType type : types) { + if (!type.follow().isUnknownType()) { + return type; + } + } + return null; + } + + /** + * Returns the first alternate whose type is not unknown and + * is not {@link Indexer.idx.builtins.None}. + * @return the first non-unknown, non-{@code None} alternate, or {@code null} if none found + */ + public NType firstKnownNonNullAlternate() { + for (NType type : types) { + NType tt = type.follow(); + if (!tt.isUnknownType() && tt != Indexer.idx.builtins.None) { + return type; + } + } + return null; + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + sb.append("["); + for (NType u : types) { + u.print(ctr, sb); + sb.append(","); + } + sb.setLength(sb.length() - 1); // pop last comma + sb.append("]"); + } +} Index: tests/java/org/python/indexer/data/class1.py =================================================================== --- tests/java/org/python/indexer/data/class1.py (revision 0) +++ tests/java/org/python/indexer/data/class1.py (revision 0) @@ -0,0 +1,12 @@ +class A: + a = 1 + + def __init__(self): + self.b = 2 + + def hi(self, msg): + print "%s: %s" % (msg, A.a) + +x = A +y = A() +z = y.b Index: src/org/python/indexer/AstCache.java =================================================================== --- src/org/python/indexer/AstCache.java (revision 0) +++ src/org/python/indexer/AstCache.java (revision 0) @@ -0,0 +1,324 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import org.antlr.runtime.ANTLRFileStream; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.RecognitionException; +import org.python.antlr.AnalyzingParser; +import org.python.antlr.base.mod; +import org.python.indexer.ast.NModule; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Provides a factory for python source ASTs. Maintains configurable on-disk and + * in-memory caches to avoid re-parsing files during analysis. + */ +public class AstCache { + + public static final String CACHE_DIR = Util.getSystemTempDir() + "jython/ast_cache/"; + + private static final Logger LOG = Logger.getLogger(AstCache.class.getCanonicalName()); + + private Map cache = new HashMap(); + + private static AstCache INSTANCE; + + private AstCache() throws Exception { + File f = new File(CACHE_DIR); + if (!f.exists()) { + f.mkdirs(); + } + } + + public static AstCache get() throws Exception { + if (INSTANCE == null) { + INSTANCE = new AstCache(); + } + return INSTANCE; + } + + /** + * Clears the memory cache. + */ + public void clear() { + cache.clear(); + } + + /** + * Removes all serialized ASTs from the on-disk cache. + * @return {@code true} if all cached AST files were removed + */ + public boolean clearDiskCache() { + try { + File dir = new File(CACHE_DIR); + for (File f : dir.listFiles()) { + if (f.isFile()) { + f.delete(); + } + } + return true; + } catch (Exception x) { + severe("Failed to clear disk cache: " + x); + return false; + } + } + + /** + * Returns the syntax tree for {@code path}. May find and/or create a + * cached copy in the mem cache or the disk cache. + * @param path absolute path to a source file + * @return the AST, or {@code null} if the parse failed for any reason + * @throws Exception if anything unexpected occurs + */ + public NModule getAST(String path) throws Exception { + if (path == null) throw new IllegalArgumentException("null path"); + return fetch(path); + } + + /** + * Returns the syntax tree for {@code path} with {@code contents}. + * Uses the memory cache but not the disk cache. + * This method exists primarily for unit testing. + * @param path a name for the file. Can be relative. + * @param contents the source to parse + */ + public NModule getAST(String path, String contents) throws Exception { + if (path == null) throw new IllegalArgumentException("null path"); + if (contents == null) throw new IllegalArgumentException("null contents"); + + // Cache stores null value if the parse failed. + if (cache.containsKey(path)) { + return cache.get(path); + } + + NModule mod = null; + try { + mod = parse(path, contents); + if (mod != null) { + mod.setFileAndMD5(path, Util.getMD5(contents.getBytes("UTF-8"))); + } + } finally { + cache.put(path, mod); // may be null + } + return mod; + } + + /** + * Get or create an AST for {@code path}, checking and if necessary updating + * the disk and memory caches. + * @param path absolute source path + */ + private NModule fetch(String path) throws Exception { + // Cache stores null value if the parse failed. + if (cache.containsKey(path)) { + return cache.get(path); + } + + // Might be cached on disk but not in memory. + NModule mod = getSerializedModule(path); + if (mod != null) { + fine("reusing " + path); + cache.put(path, mod); + return mod; + } + + try { + mod = parse(path); + } finally { + cache.put(path, mod); // may be null + } + + if (mod != null) { + serialize(mod); + } + + return mod; + } + + /** + * Parse a file. Does not look in the cache or cache the result. + */ + private NModule parse(String path) throws Exception { + fine("parsing " + path); + mod ast = invokeANTLR(path); + return generateAST(ast, path); + } + + /** + * Parse a string. Does not look in the cache or cache the result. + */ + private NModule parse(String path, String contents) throws Exception { + fine("parsing " + path); + mod ast = invokeANTLR(path, contents); + return generateAST(ast, path); + } + + private NModule generateAST(mod ast, String path) throws Exception { + if (ast == null) { + Indexer.idx.reportFailedAssertion("ANTLR returned NULL for " + path); + return null; + } + + // Convert to indexer's AST. Type conversion warnings are harmless here. + Object obj = ast.accept(new AstConverter()); + if (!(obj instanceof NModule)) { + warn("\n[warning] converted AST is not a module: " + obj); + return null; + } + + NModule module = (NModule)obj; + if (new File(path).canRead()) { + module.setFile(path); + } + return module; + } + + private mod invokeANTLR(String filename) { + CharStream text = null; + try { + text = new ANTLRFileStream(filename); + } catch (IOException iox) { + fine(filename + ": " + iox); + return null; + } + return invokeANTLR(text, filename); + } + + private mod invokeANTLR(String filename, String contents) { + CharStream text = new ANTLRStringStream(contents); + return invokeANTLR(text, filename); + } + + private mod invokeANTLR(CharStream text, String filename) { + AnalyzingParser p = new AnalyzingParser(text, filename, null); + mod ast = null; + try { + ast = p.parseModule(); + } catch (Exception x) { + fine("parse for " + filename + " failed: " + x); + } + recordParseErrors(filename, p.getRecognitionErrors()); + return ast; + } + + private void recordParseErrors(String path, List errs) { + if (errs.isEmpty()) { + return; + } + List diags = Indexer.idx.getParseErrs(path); + for (RecognitionException rx : errs) { + String msg = rx.line + ":" + rx.charPositionInLine + ":" + rx; + diags.add(new Diagnostic(path, Diagnostic.Type.ERROR, -1, -1, msg)); + } + } + + /** + * Each source file's AST is saved in an object file named for the MD5 + * checksum of the source file. All that is needed is the MD5, but the + * file's base name is included for ease of debugging. + */ + public String getCachePath(File sourcePath) throws Exception { + return getCachePath(Util.getMD5(sourcePath), sourcePath.getName()); + } + + public String getCachePath(String md5, String name) { + return CACHE_DIR + name + md5 + ".ast"; + } + + // package-private for testing + void serialize(NModule ast) throws Exception { + String path = getCachePath(ast.getMD5(), new File(ast.getFile()).getName()); + ObjectOutputStream oos = null; + FileOutputStream fos = null; + try { + fos = new FileOutputStream(path); + oos = new ObjectOutputStream(fos); + oos.writeObject(ast); + } finally { + if (oos != null) { + oos.close(); + } else if (fos != null) { + fos.close(); + } + } + } + + // package-private for testing + NModule getSerializedModule(String sourcePath) { + try { + File sourceFile = new File(sourcePath); + if (sourceFile == null || !sourceFile.canRead()) { + return null; + } + File cached = new File(getCachePath(sourceFile)); + if (!cached.canRead()) { + return null; + } + return deserialize(sourceFile); + } catch (Exception x) { + severe("Failed to deserialize " + sourcePath + ": " + x); + return null; + } + } + + // package-private for testing + NModule deserialize(File sourcePath) throws Exception { + String cachePath = getCachePath(sourcePath); + FileInputStream fis = null; + ObjectInputStream ois = null; + try { + fis = new FileInputStream(cachePath); + ois = new ObjectInputStream(fis); + NModule mod = (NModule)ois.readObject(); + // Files in different dirs may have the same base name and contents. + mod.setFile(sourcePath); + return mod; + } finally { + if (ois != null) { + ois.close(); + } else if (fis != null) { + fis.close(); + } + } + } + + private void log(Level level, String msg) { + if (LOG.isLoggable(level)) { + LOG.log(level, msg); + } + } + + private void severe(String msg) { + log(Level.SEVERE, msg); + } + + private void warn(String msg) { + log(Level.WARNING, msg); + } + + private void info(String msg) { + log(Level.INFO, msg); + } + + private void fine(String msg) { + log(Level.FINE, msg); + } + + private void finer(String msg) { + log(Level.FINER, msg); + } +} Index: src/org/python/indexer/ast/NList.java =================================================================== --- src/org/python/indexer/ast/NList.java (revision 0) +++ src/org/python/indexer/ast/NList.java (revision 0) @@ -0,0 +1,56 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.ast.NNode; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NType; + +import java.util.List; + +public class NList extends NSequence { + + static final long serialVersionUID = 6623743056841822992L; + + public NList(List elts) { + this(elts, 0, 1); + } + + public NList(List elts, int start, int end) { + super(elts, start, end); + } + + @Override + public NType resolve(Scope s) throws Exception { + if (elts.size() == 0) { + return setType(new NListType()); // list + } + + NListType listType = null; + for (NNode elt : elts) { + if (listType == null) { + listType = new NListType(resolveExpr(elt, s)); + } else { + listType.add(resolveExpr(elt, s)); + } + } + if (listType != null) { + setType(listType); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(elts, v); + } + } +} Index: src/org/python/indexer/ast/NGeneratorExp.java =================================================================== --- src/org/python/indexer/ast/NGeneratorExp.java (revision 0) +++ src/org/python/indexer/ast/NGeneratorExp.java (revision 0) @@ -0,0 +1,53 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NType; + +import java.util.List; + +public class NGeneratorExp extends NNode { + + static final long serialVersionUID = -8614142736962193509L; + + public NNode elt; + public List generators; + + public NGeneratorExp(NNode elt, List generators) { + this(elt, generators, 0, 1); + } + + public NGeneratorExp(NNode elt, List generators, int start, int end) { + super(start, end); + this.elt = elt; + this.generators = generators; + addChildren(elt); + addChildren(generators); + } + + /** + * Python's list comprehension will erase any variable used in generators. + * This is wrong, but we "respect" this bug here. + */ + @Override + public NType resolve(Scope s) throws Exception { + resolveList(generators, s); + return setType(new NListType(resolveExpr(elt, s))); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(elt, v); + visitNodeList(generators, v); + } + } +} Index: src/org/python/indexer/ast/NExprStmt.java =================================================================== --- src/org/python/indexer/ast/NExprStmt.java (revision 0) +++ src/org/python/indexer/ast/NExprStmt.java (revision 0) @@ -0,0 +1,45 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +/** + * Expression statement. + */ +public class NExprStmt extends NNode { + + static final long serialVersionUID = 7366113211576923188L; + + public NNode value; + + public NExprStmt(NNode n) { + this(n, 0, 1); + } + + public NExprStmt(NNode n, int start, int end) { + super(start, end); + this.value = n; + addChildren(n); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(value, s); + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(value, v); + } + } +} Index: src/org/python/indexer/ast/NAugAssign.java =================================================================== --- src/org/python/indexer/ast/NAugAssign.java (revision 0) +++ src/org/python/indexer/ast/NAugAssign.java (revision 0) @@ -0,0 +1,47 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NAugAssign extends NNode { + + static final long serialVersionUID = -6479618862099506199L; + + public NNode target; + public NNode value; + public String op; + + public NAugAssign(NNode target, NNode value, String op) { + this(target, value, op, 0, 1); + } + + public NAugAssign(NNode target, NNode value, String op, int start, int end) { + super(start, end); + this.target = target; + this.value = value; + this.op = op; + addChildren(target, value); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(target, s); + return setType(resolveExpr(value, s)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(target, v); + visitNode(value, v); + } + } +} Index: src/org/python/indexer/ast/NListComp.java =================================================================== --- src/org/python/indexer/ast/NListComp.java (revision 0) +++ src/org/python/indexer/ast/NListComp.java (revision 0) @@ -0,0 +1,55 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NType; + +import java.util.List; + +public class NListComp extends NNode { + + static final long serialVersionUID = -150205687457446323L; + + public NNode elt; + public List generators; + + public NListComp(NNode elt, List generators) { + this(elt, generators, 0, 1); + } + + public NListComp(NNode elt, List generators, int start, int end) { + super(start, end); + this.elt = elt; + this.generators = generators; + addChildren(elt); + addChildren(generators); + } + + /** + * Python's list comprehension will bind the variables used in generators. + * This will erase the original values of the variables even after the + * comprehension. + */ + @Override + public NType resolve(Scope s) throws Exception { + NameBinder binder = NameBinder.make(); + resolveList(generators, s); + return setType(new NListType(resolveExpr(elt, s))); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(elt, v); + visitNodeList(generators, v); + } + } +} Index: src/org/python/indexer/types/NType.java =================================================================== --- src/org/python/indexer/types/NType.java (revision 0) +++ src/org/python/indexer/types/NType.java (revision 0) @@ -0,0 +1,202 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Builtins; +import org.python.indexer.Indexer; +import org.python.indexer.Scope; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class NType { + + private Scope table; + + protected static final String LIBRARY_URL = Builtins.LIBRARY_URL; + protected static final String TUTORIAL_URL = Builtins.TUTORIAL_URL; + protected static final String REFERENCE_URL = Builtins.REFERENCE_URL; + + private static Pattern INSTANCE_TAG = Pattern.compile("(.+?)=#([0-9]+)"); + + public NType() { + } + + public void setTable(Scope table) { + this.table = table; + } + + public Scope getTable() { + if (table == null) { + table = new Scope(null, Scope.Type.SCOPE); + } + return table; + } + + /** + * Returns {@link NUnknownType#follow} of this type. + */ + public NType follow() { + return NUnknownType.follow(this); + } + + /** + * Returns {@code true} if this Python type is implemented in native code + * (i.e., C, Java, C# or some other host language.) + */ + public boolean isNative() { + return Indexer.idx.builtins.isNative(this); + } + + public boolean isClassType() { + return this instanceof NClassType; + } + + public boolean isDictType() { + return this instanceof NDictType; + } + + public boolean isFuncType() { + return this instanceof NFuncType; + } + + public boolean isInstanceType() { + return this instanceof NInstanceType; + } + + public boolean isListType() { + return this instanceof NListType; + } + + public boolean isModuleType() { + return this instanceof NModuleType; + } + + public boolean isNumType() { + return this == Indexer.idx.builtins.BaseNum; + } + + public boolean isStrType() { + return this == Indexer.idx.builtins.BaseStr; + } + + public boolean isTupleType() { + return this instanceof NTupleType; + } + + public boolean isUnionType() { + return this instanceof NUnionType; + } + + public boolean isUnknownType() { + return this instanceof NUnknownType; + } + + public NClassType asClassType() { + return (NClassType)this; + } + + public NDictType asDictType() { + return (NDictType)this; + } + + public NFuncType asFuncType() { + return (NFuncType)this; + } + + public NInstanceType asInstanceType() { + return (NInstanceType)this; + } + + public NListType asListType() { + return (NListType)this; + } + + public NModuleType asModuleType() { + return (NModuleType)this; + } + + public NTupleType asTupleType() { + return (NTupleType)this; + } + + public NUnionType asUnionType() { + return (NUnionType)this; + } + + public NUnknownType asUnknownType() { + return (NUnknownType)this; + } + + @Override + public String toString() { + StringBuilder input = new StringBuilder(); + print(new CyclicTypeRecorder(), input); + + // Postprocess to remove unused instance reference numbers. + StringBuilder sb = new StringBuilder(input.length()); + Matcher m = INSTANCE_TAG.matcher(input.toString()); + int end = -1; + while (m.find()) { + end = m.end(); + int num = Integer.parseInt(m.group(2)); + if (input.indexOf("<#" + num + ">") == -1) { // referenced? + sb.append(m.group(1)); // skip tag + } else { + sb.append(m.group()); // whole thing + } + } + if (end != -1) { + sb.append(input.substring(end)); + } + + return sb.toString(); + } + + /** + * Internal class to support printing in the presence of type-graph cycles. + */ + protected class CyclicTypeRecorder { + int count = 0; + private Map elements = new HashMap(); + + /** + * Get the instance number for the specified type. + * @return the instance number: positive if the type was already recorded, + * or its negative if the type was just recorded and assigned a number. + */ + public int fetch(NType t) { + Integer i = elements.get(t); + if (i != null) { + return i; + } + i = ++count; + elements.put(t, i); + return -i; + } + } + + /** + * Internal method to support printing in the presence of type-graph cycles. + */ + protected void print(CyclicTypeRecorder ctr, StringBuilder sb) { + int num = ctr.fetch(this); + if (num > 0) { + sb.append("<#").append(num).append(">"); + } else { + String tag = getClass().getName(); + tag = tag.substring(tag.lastIndexOf(".") + 2); + sb.append("<").append(tag).append("=#").append(-num).append(":"); + printKids(ctr, sb); + sb.append(">"); + } + } + + /** + * Internal method to support printing in the presence of type-graph cycles. + */ + protected abstract void printKids(CyclicTypeRecorder ctr, StringBuilder sb); +} Index: tests/java/org/python/indexer/data/pkg/animal/reptile/__init__.py =================================================================== Index: src/org/python/indexer/Scope.java =================================================================== --- src/org/python/indexer/Scope.java (revision 0) +++ src/org/python/indexer/Scope.java (revision 0) @@ -0,0 +1,805 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import org.python.indexer.ast.NName; +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NUrl; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map.Entry; +import java.util.Map; +import java.util.Set; + +/** + * Symbol table. + */ +public class Scope { + + /** + * For preventing circular inheritance from recursing. + */ + private static Set looked = new HashSet(); + + public enum Type { + CLASS, + INSTANCE, + FUNCTION, + MODULE, + GLOBAL, + SCOPE + } + + /** + * XXX: This table is incorrectly overloaded to contain both object + * attributes and lexical-ish scope names, when they are in some cases + * separate namespaces. (In particular, they're effectively the same + * namespace for module scope and class scope, and they're different for + * function scope, which uses the {@code func_dict} namespace for storing + * attributes.) + */ + private Map table; // stays null for most scopes (mem opt) + + private Scope parent; + private List supers; + private Set globalNames; + private Type scopeType; + private String path = ""; + private int lambdaCounter = 0; + private boolean isBindingPhase = false; + + public Scope(Scope parent, Type type) { + if (type == null) { + throw new IllegalArgumentException("'type' param cannot be null"); + } + setParent(parent); + setScopeType(type); + } + + public void setTable(Map table) { + this.table = table; + } + + /** + * Returns an immutable view of the table. + */ + public Map getTable() { + if (table != null) { + return Collections.unmodifiableMap(table); + } + Map map = Collections.emptyMap(); + return map; + } + + public void setParent(Scope parent) { + this.parent = parent; + } + + public Scope getParent() { + return parent; + } + + public void addSuper(Scope sup) { + if (supers == null) { + supers = new ArrayList(); + } + supers.add(sup); + } + + public void setSupers(List supers) { + this.supers = supers; + } + + public List getSupers() { + if (supers != null) { + return Collections.unmodifiableList(supers); + } + List list = Collections.emptyList(); + return list; + } + + public void setScopeType(Type type) { + this.scopeType = type; + } + + public Type getScopeType() { + return scopeType; + } + + public boolean isFunctionScope() { + return scopeType == Type.FUNCTION; + } + + /** + * Mark a name as being global (i.e. module scoped) for name-binding and + * name-lookup operations in this code block and any nested scopes. + */ + public void addGlobalName(String name) { + if (name == null) { + return; + } + if (globalNames == null) { + globalNames = new HashSet(); + } + globalNames.add(name); + } + + /** + * Returns {@code true} if {@code name} appears in a {@code global} + * statement in this scope or any enclosing scope. + */ + public boolean isGlobalName(String name) { + if (globalNames != null) { + return globalNames.contains(name); + } + return parent == null ? false : parent.isGlobalName(name); + } + + /** + * Directly assigns a binding to a name in this table. Does not add a new + * definition or reference to the binding. This form of {@code put} is + * often followed by a call to {@link putLocation} to create a reference to + * the binding. When there is no code location associated with {@code id}, + * or it is otherwise undesirable to create a reference, the + * {@link putLocation} call is omitted. + */ + public void put(String id, NBinding b) { + putBinding(id, b); + } + + /** + * Adds a definition and/or reference to the table. + * If there is no binding for {@code id}, creates one and gives it + * {@code type} and {@code kind}.

+ * + * If a binding already exists, then add either a definition or a reference + * at {@code loc} to the binding. By convention we consider it a definition + * if the type changes. If the passed type is different from the binding's + * current type, set the binding's type to the union of the old and new + * types, and add a definition. If the new type is the same, just add a + * reference.

+ * + * If the binding already exists, {@code kind} is only updated if a + * definition was added and the binding's type was previously the + * unknown type. + */ + public NBinding put(String id, NNode loc, NType type, NBinding.Kind kind) { + if (type == null) { + throw new IllegalArgumentException("Null type: id=" + id + ", loc=" + loc); + } + NBinding b = lookupScope(id); + return insertOrUpdate(b, id, loc, type, kind); + } + + /** + * Same as {@link #put}, but adds the name as an attribute of this scope. + * Looks up the superclass chain to see if the attribute exists, rather + * than looking in the lexical scope chain. + * + * @return the new binding, or {@code null} if the current scope does + * not have a properly initialized path. + */ + public NBinding putAttr(String id, NNode loc, NType type, NBinding.Kind kind) { + if (type == null) { + throw new IllegalArgumentException("Null type: id=" + id + ", loc=" + loc); + } + + // Attributes are always part of a qualified name. If there is no qname + // on the target type, it's a bug (we forgot to set the path somewhere.) + if ("".equals(path)) { + Indexer.idx.reportFailedAssertion( + "Attempting to set attr '" + id + "' at location " + loc + + (loc != null ? loc.getFile() : "") + + " in scope with no path (qname) set: " + this.toShortString()); + return null; + } + + NBinding b = lookupAttr(id); + return insertOrUpdate(b, id, loc, type, kind); + } + + private NBinding insertOrUpdate(NBinding b, String id, NNode loc, NType t, NBinding.Kind k) { + if (b == null) { + b = insertBinding(new NBinding(id, loc, t, k)); + } else { + updateType(b, loc, t, k); + } + return b; + } + + /** + * Adds a new binding for {@code id}. If a binding already existed, + * replaces its previous definitions, if any, with {@code loc}. Sets the + * binding's type to {@code type} (not a union with the previous type). + */ + public NBinding update(String id, NNode loc, NType type, NBinding.Kind kind) { + if (type == null) { + throw new IllegalArgumentException("Null type: id=" + id + ", loc=" + loc); + } + return update(id, new Def(loc), type, kind); + } + + /** + * Adds a new binding for {@code id}. If a binding already existed, + * replaces its previous definitions, if any, with {@code loc}. Sets the + * binding's type to {@code type} (not a union with the previous type). + */ + public NBinding update(String id, Def loc, NType type, NBinding.Kind kind) { + if (type == null) { + throw new IllegalArgumentException("Null type: id=" + id + ", loc=" + loc); + } + NBinding b = lookupScope(id); + if (b == null) { + return insertBinding(new NBinding(id, loc, type, kind)); + } + + b.getDefs().clear(); // XXX: what about updating refs & idx.locations? + b.addDef(loc); + b.setType(type); + + // XXX: is this a bug? I think he meant to do this check before the + // line above that sets b.type, if it's supposed to be like put(). + if (b.getType().isUnknownType()) { + b.setKind(kind); + } + return b; + } + + private NBinding insertBinding(NBinding b) { + switch (b.getKind()) { + case MODULE: + b.setQname(b.getType().getTable().path); + break; + case PARAMETER: + b.setQname(extendPathForParam(b.getName())); + break; + default: + b.setQname(extendPath(b.getName())); + break; + } + + b = Indexer.idx.putBinding(b); + putBinding(b.getName(), b); + return b; + } + + private void putBinding(String id, NBinding b) { + ensureTable(); + table.put(id, b); + } + + private void updateType(NBinding b, NNode loc, NType type, NBinding.Kind kind) { + NType curType = b.followType(); + if (!isNewType(curType, type)) { + if (loc != null + && !(loc instanceof NUrl) + && !b.getDefs().contains(loc)) { + Indexer.idx.putLocation(loc, b); + } + return; + } + + if (loc != null && !b.getRefs().contains(loc)) { + b.addDef(loc); + b.setProvisional(false); + } + + // The union ordering matters here. If they're two different unknown + // types, union() points the first one to the second one. We want to + // keep the binding's existing type iff its table contains provisional + // attribute bindings that we need to look up later. + NType btype = b.getType(); + NType t1, t2; + if (btype.isUnknownType() && !btype.getTable().isEmpty()) { + t1 = type; + t2 = btype; + } else { + t1 = btype; + t2 = type; + } + NType newType = NUnionType.union(t1, t2); + b.setType(newType); + + if (curType.isUnknownType()) { + b.setKind(kind); + } + + retargetReferences(b, curType); + } + + /** + * If the current type had a provisional binding, retarget its refs to the + * new type. It probably only works one level deep: need dataflow analysis + * in the general case. However, it does pick up some extra references, + * so it's reasonable for now. + */ + private void retargetReferences(NBinding b, NType curType) { + Scope newScope = b.followType().getTable(); + for (Map.Entry e : curType.getTable().entrySet()) { + String attr = e.getKey(); + NBinding oldBinding = e.getValue(); + if (!oldBinding.isProvisional()) { + continue; + } + Indexer.idx.removeBinding(oldBinding); + NBinding newBinding = newScope.lookupAttr(attr); + if (newBinding != null) { + List refs = new ArrayList(); // avoid ConcurrentModificationException + refs.addAll(oldBinding.getRefs()); + for (Ref ref : refs) { + Indexer.idx.updateLocation(ref, newBinding); + } + } + } + } + + /** + * Returns {@code true} if the binding is being assigned a new type. + */ + private boolean isNewType(NType curType, NType type) { + // In the bindNames() phase we want all places where a given name + // is bound in the same scope to share the same binding, because + // we haven't resolved the types yet. This takes care of that case. + if (isBindingPhase) { + return false; + } + + if (curType.isUnionType()) { + return !curType.asUnionType().contains(type); + } + + return curType != type; + } + + public void remove(String id) { + if (table != null) { + table.remove(id); + } + } + + /** + * Create a copy of the symbol table but without the links to parent, supers + * and children. Useful for creating instances. + * + * @return the symbol table for use by the instance. + */ + public Scope copy(Type tableType) { + Scope ret = new Scope(null, tableType); + if (table != null) { + ret.ensureTable(); + ret.table.putAll(table); + } + return ret; + } + + public void setPath(String path) { + if (path == null) { + throw new IllegalArgumentException("'path' param cannot be null"); + } + this.path = path; + } + + public String getPath() { + return path; + } + + public void setPath(String a, String b) { + NBinding b1 = lookup(a); + NBinding b2 = lookup(b); + if (b1 != null && b2 != null) { + b1.setQname(b2.getQname()); + } + } + + /** + * Look up a name (String) in the current symbol table. If not found, + * recurse on the parent table. + */ + public NBinding lookup(String name) { + NBinding b = getModuleBindingIfGlobal(name); + if (b != null) { + return b; + } + if (table != null) { + NBinding ent = table.get(name); + if (ent != null) { + return ent; + } + } + if (getParent() == null) { + return null; + } + return getParent().lookup(name); + } + + /** + * Specialized version for the convenience of looking up {@code Name}s. + * For all other types return {@code null}. + */ + public NBinding lookup(NNode n) { + if (n instanceof NName) { + return lookup(((NName)n).id); + } + return null; + } + + /** + * Look up a name, but only in the current scope. + * @return the local binding for {@code name}, or {@code null}. + */ + public NBinding lookupLocal(String name) { + NBinding b = getModuleBindingIfGlobal(name); + if (b != null) { + return b; + } + return table == null ? null : table.get(name); + } + + /** + * Look up an attribute in the type hierarchy. Don't look at parent link, + * because the enclosing scope may not be a super class. The search is + * "depth first, left to right" as in Python's (old) multiple inheritance + * rule. The new MRO can be implemented, but will probably not introduce + * much difference. + * @param supersOnly search only in the supers' scopes, not in local table. + */ + public NBinding lookupAttr(String name, boolean supersOnly) { + if (looked.contains(this)) { + return null; + } + if (table != null && !supersOnly) { + NBinding b = table.get(name); + if (b != null) { + return b; + } + } + if (supers == null || supers.isEmpty()) { + return null; + } + looked.add(this); + try { + for (Scope p : supers) { + NBinding b = p.lookupAttr(name); + if (b != null) { + return b; + } + } + return null; + } finally { + looked.remove(this); + } + } + + /** + * Look up an attribute in the local scope and superclass scopes. + * @see lookupAttr(String,boolean) + */ + public NBinding lookupAttr(String name) { + return lookupAttr(name, false); + } + + /** + * Look up the scope chain for a binding named {@code name} + * and if found, return its type. + */ + public NType lookupType(String name) { + return lookupType(name, false); + } + + /** + * Look for a binding named {@code name} and if found, return its type. + * @param localOnly {@code true} to look only in the current scope; + * if {@code false}, follows the scope chain. + */ + public NType lookupType(String name, boolean localOnly) { + NBinding b = localOnly ? lookupLocal(name) : lookup(name); + if (b == null) { + return null; + } + NType ret = b.followType(); + // XXX: really need to make ModuleTable polymorphic... + if (this == Indexer.idx.moduleTable) { + if (ret.isModuleType()) { + return ret; + } + if (ret.isUnionType()) { + for (NType t : ret.asUnionType().getTypes()) { + NType realType = t.follow(); + if (realType.isModuleType()) { + return realType; + } + } + } + Indexer.idx.warn("Found non-module type in module table: " + b); + return null; + } + return ret; + } + + public NType lookupTypeAttr(String name) { + NBinding b = lookupAttr(name); + if (b != null) { + return b.followType(); + } + return null; + } + + /** + * Look up a name, but the search is bounded by a type and will not proceed + * to an outer scope when reaching a certain type of symbol table. + * + * @param name the name to be looked up + * @param typebound the type we wish the search to be bounded at + * @return a binding, or {@code null} if not found + */ + public NBinding lookupBounded(String name, Type typebound) { + if (scopeType == typebound) { + return table == null ? null : table.get(name); + } + if (getParent() == null) { + return null; + } + return getParent().lookupBounded(name, typebound); + } + + /** + * Returns {@code true} if this is a scope in which names may be bound. + */ + public boolean isScope() { + switch (scopeType) { + case CLASS: + case INSTANCE: + case FUNCTION: + case MODULE: + case GLOBAL: + return true; + default: + return false; + } + } + + /** + * Find the enclosing scope-defining symbol table.

+ * + * More precisely, if a form introduces a new name in the "current scope", + * resolving the form needs to search up the symbol-table chain until it + * finds the table representing the scope to which the name should be added. + * Used by {@link org.python.indexer.ast.NameBinder} to create new name + * bindings in the appropriate enclosing table with the appropriate binding + * type. + */ + public Scope getScopeSymtab() { + if (this.isScope()) { + return this; + } + if (getParent() == null) { + Indexer.idx.reportFailedAssertion("No binding scope found for " + this.toShortString()); + return this; + } + return getParent().getScopeSymtab(); + } + + /** + * Look up a name, but bounded by a scope defining construct. Those scopes + * are of type module, class, instance or function. This is used in + * determining the locations of a variable's definition. + */ + public NBinding lookupScope(String name) { + NBinding b = getModuleBindingIfGlobal(name); + if (b != null) { + return b; + } + Scope st = getScopeSymtab(); + if (st != null) { + return st.lookupLocal(name); + } + return null; + } + + /** + * Find a symbol table of a certain type in the enclosing scopes. + */ + public Scope getSymtabOfType(Type type) { + if (scopeType == type) { + return this; + } + if (parent == null) { + return null; + } + return parent.getSymtabOfType(type); + } + + /** + * Returns the global scope (i.e. the module scope for the current module). + */ + public Scope getGlobalTable() { + Scope result = getSymtabOfType(Type.MODULE); + if (result == null) { + Indexer.idx.reportFailedAssertion("No module table found for " + this); + result = this; + } + return result; + } + + /** + * Returns the containing lexical scope (which may be this scope) + * for lexical name lookups. In particular, it skips class scopes. + */ + public Scope getEnclosingLexicalScope() { + if (scopeType == Scope.Type.FUNCTION + || scopeType == Scope.Type.MODULE) { + return this; + } + if (parent == null) { + Indexer.idx.reportFailedAssertion("No lexical scope found for " + this); + return this; + } + return parent.getEnclosingLexicalScope(); + } + + /** + * If {@code name} is declared as a global, return the module binding. + */ + private NBinding getModuleBindingIfGlobal(String name) { + if (isGlobalName(name)) { + Scope module = getGlobalTable(); + if (module != null && module != this) { + return module.lookupLocal(name); + } + } + return null; + } + + /** + * Name binding occurs in a separate pass before the name resolution pass, + * building out the scope tree and binding names in the correct scopes. + * In this pass, the name binding and lookup rules are slightly different. + * This condition is transient: no scopes will be in the name-binding phase + * in a completed index (or module). + */ + public boolean isNameBindingPhase() { + return isBindingPhase; + } + + public void setNameBindingPhase(boolean isBindingPhase) { + this.isBindingPhase = isBindingPhase; + } + + /** + * Merge all records from another symbol table. Used by {@code import from *}. + */ + public void merge(Scope other) { + ensureTable(); + this.table.putAll(other.table); + } + + public Set keySet() { + if (table != null) { + return table.keySet(); + } + Set result = Collections.emptySet(); + return result; + } + + public Collection values() { + if (table != null) { + return table.values(); + } + Collection result = Collections.emptySet(); + return result; + } + + public Set> entrySet() { + if (table != null) { + return table.entrySet(); + } + Set> result = Collections.emptySet(); + return result; + } + + public boolean isEmpty() { + return table == null ? true : table.isEmpty(); + } + + /** + * Dismantles all resources allocated by this scope. + */ + public void clear() { + if (table != null) { + table.clear(); + table = null; + } + parent = null; + if (supers != null) { + supers.clear(); + supers = null; + } + if (globalNames != null) { + globalNames.clear(); + globalNames = null; + } + } + + public String newLambdaName() { + return "lambda%" + (++lambdaCounter); + } + + /** + * Generates a qname for a parameter of a function or method. + * There is not enough context for {@link #extendPath} to differentiate + * params from locals, so callers must use this method when the name is + * known to be a parameter name. + */ + public String extendPathForParam(String name) { + if (path.equals("")) { + throw new IllegalStateException("Not inside a function"); + } + return path + "@" + name; + } + + /** + * Constructs a qualified name by appending {@code name} to this scope's qname.

+ * + * The indexer uses globally unique fully qualified names to address + * identifier definition sites. Many Python identifiers are already + * globally addressable using dot-separated package, class and attribute + * names.

+ * + * Function variables and parameters are not globally addressable in the + * language, so the indexer uses a special path syntax for creating globally + * unique qualified names for them. By convention the syntax is "@" for + * parameters and "&" for local variables. + * + * @param name a name to append to the current qname + * @return the qname for {@code name}. Does not change this scope's path. + */ + public String extendPath(String name) { + if (name.endsWith(".py")) { + name = Util.moduleNameFor(name); + } + if (path.equals("")) { + return name; + } + String sep = null; + switch (scopeType) { + case MODULE: + case CLASS: + case INSTANCE: + case SCOPE: + sep = "."; + break; + case FUNCTION: + sep = "&"; + break; + default: + System.err.println("unsupported context for extendPath: " + scopeType); + return path; + } + return path + sep + name; + } + + private void ensureTable() { + if (table == null) { + table = new LinkedHashMap(); + } + } + + @Override + public String toString() { + return ""; + } + + public String toShortString() { + return ""; + } +} Index: tests/java/org/python/indexer/data/pkg/other/color/__init__.py =================================================================== Index: src/org/python/antlr/BaseParser.java =================================================================== --- src/org/python/antlr/BaseParser.java (revision 6819) +++ src/org/python/antlr/BaseParser.java (working copy) @@ -31,7 +31,7 @@ this.errorHandler = eh; } - private PythonParser setupParser(boolean single) { + protected PythonParser setupParser(boolean single) { PythonLexer lexer = new PythonLexer(charStream); lexer.setErrorHandler(errorHandler); lexer.single = single; Index: tests/java/org/python/indexer/data/pkg/other/force/__init__.py =================================================================== Index: src/org/python/indexer/ast/NImportFrom.java =================================================================== --- src/org/python/indexer/ast/NImportFrom.java (revision 0) +++ src/org/python/indexer/ast/NImportFrom.java (revision 0) @@ -0,0 +1,222 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Def; +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; + +/** + * Handles import from statements such as {@code from moduleA import a, b as c, d} + * and {@code from foo.bar.moduleB import *}.

+ * + * The indexer's implementation of import * uses different semantics from + * all the other forms of import. It's basically a bug, although the jury + * is still out as to which implementation is better.

+ * + * For the others we define name bindings anywhere an actual name is + * introduced into the scope containing the import statement, and references + * to the imported module or name everywhere else. This mimics the behavior + * of Python at runtime, but it may be confusing to anyone with only a casual + * understanding of Python's data model, who might think it works more like + * Java.

+ * + * For import * we just enter the imported names into the symbol table, + * which lets other code reference them, but the references "pass through" + * automatically to the module from which the names were imported.

+ * + * To illustate the difference, consider the following four modules: + *

+ *  moduleA.py:
+ *     a = 1
+ *     b = 2
+ *
+ *  moduleB.py:
+ *     c = 3
+ *     d = 4
+ *
+ *  moduleC.py:
+ *     from moduleA import a, b
+ *     from moduleB import *
+ *     print a  # indexer finds definition of 'a' 2 lines up
+ *     print b  # indexer finds definition of 'b' 3 lines up
+ *     print c  # indexer finds definition of 'c' in moduleB
+ *     print d  # indexer finds definition of 'd' in moduleB
+ *
+ *  moduleD.py:
+ *     import moduleC
+ *     print moduleC.a  # indexer finds definition of 'a' in moduleC
+ *     print moduleC.b  # indexer finds definition of 'b' in moduleC
+ *     print moduleC.c  # indexer finds definition of 'c' in moduleB
+ *     print moduleC.c  # indexer finds definition of 'd' in moduleB
+ * 
+ * To make import * work like the others, we need only create bindings + * for the imported names. But where would the bindings be located? + * Assuming that we were to co-locate them all at the "*" name node, + * clicking on a reference to any of the names would jump to the "*". + * It's not clear that this is a better user experience.

+ * + * We could make the other import statement forms work like {@code import *}, + * but that path is even more fraught with confusing inconsistencies. + */ +public class NImportFrom extends NNode { + + static final long serialVersionUID = 5070549408963950138L; + + // from ...a.b.c import x, foo as bar, y + // from y.z import * + public String module; // "...a.b.c" + public NQname qname; // ".", ".", ".", "a", "b", "c" + public List aliases; // "x", "foo" [as] "bar", "y" + + public NImportFrom(String module, NQname qname, List aliases) { + this(module, qname, aliases, 0, 1); + } + + public NImportFrom(String module, NQname qname, List aliases, + int start, int end) { + super(start, end); + this.module = module; + this.qname = qname; + this.aliases = aliases; + addChildren(qname); + addChildren(aliases); + } + + @Override + public boolean bindsName() { + return true; + } + + @Override + protected void bindNames(Scope s) throws Exception { + // XXX: we can support this by resolving the qname now. + if (isImportStar()) { + return; + } + NImport.bindAliases(s, aliases); + } + + @Override + public NType resolve(Scope s) throws Exception { + Scope scope = s.getScopeSymtab(); + resolveExpr(qname, s); + + NType bottomType = qname.getBottom().getType(); + if (!bottomType.isModuleType()) { + return setType(new NUnknownType()); + } + NModuleType mt = (NModuleType)bottomType; + setType(mt); + + NImport.addReferences(s, qname, false /* don't put top name in scope */); + + if (isImportStar()) { + importStar(s, mt); + return getType(); + } + + for (NAlias a : aliases) { + resolveAlias(scope, mt, a); + } + return getType(); + } + + public boolean isImportStar() { + return aliases.size() == 1 && "*".equals(aliases.get(0).name); + } + + /** + * Resolve "foo [as bar]" in "from x[.y] import foo [as bar]". + * There are several possibilities for "foo": it could be a file + * in the directory "x[.y]", or it could be a name exported by + * the module "x[.y]" (from its __init__.py), or it could be a + * public name in the file "x/y.py". + * + * @param scope the scope into which names should be imported + * @param mt the non-{@code null} module "x[.y]". Could refer to + * either x/y.py or x/y/__init__.py. + * @param a the "foo [as bar]" component of the import statement + */ + private void resolveAlias(Scope scope, NModuleType mt, NAlias a) throws Exception { + // Possibilities 1 & 2: x/y.py or x/y/__init__.py + NBinding entry = mt.getTable().lookup(a.name); + + if (entry == null) { + // Possibility 3: try looking for x/y/foo.py + String mqname = qname.toQname() + "." + a.qname.toQname(); + NModuleType mt2 = Indexer.idx.loadModule(mqname); + if (mt2 != null) { + entry = Indexer.idx.lookupQname(mt2.getTable().getPath()); + } + } + if (entry == null) { + addError(a, "name " + a.qname.getName().id + + " not found in module " + this.module); + return; + } + String qname = a.qname.getName().id; + String aname = a.aname != null ? a.aname.id : null; + + // Create references for both the name and the alias (if present). + // Then if "foo", add "foo" to scope. If "foo as bar", add "bar". + Indexer.idx.putLocation(a.qname.getName(), entry); + if (aname != null) { + Indexer.idx.putLocation(a.aname, entry); + scope.put(aname, entry); + } else { + scope.put(qname, entry); + } + } + + private void importStar(Scope s, NModuleType mt) throws Exception { + if (mt == null || mt.getFile() == null) { + return; + } + + NModule mod = Indexer.idx.getAstForFile(mt.getFile()); + if (mod == null) { + return; + } + + List names = mod.getExportedNames(); + if (!names.isEmpty()) { + for (String name : names) { + NBinding nb = mt.getTable().lookupLocal(name); + if (nb != null) { + s.put(name, nb); + } + } + } else { + // Fall back to importing all names not starting with "_". + for (Entry e : mt.getTable().entrySet()) { + if (!e.getKey().startsWith("_")) { + s.put(e.getKey(), e.getValue()); + } + } + } + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(qname, v); + visitNodeList(aliases, v); + } + } +} Index: src/org/python/indexer/ast/NameBinder.java =================================================================== --- src/org/python/indexer/ast/NameBinder.java (revision 0) +++ src/org/python/indexer/ast/NameBinder.java (revision 0) @@ -0,0 +1,215 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NTupleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; +import org.python.indexer.Scope; + +import java.util.List; + +import static org.python.indexer.NBinding.Kind.ATTRIBUTE; +import static org.python.indexer.NBinding.Kind.CLASS; +import static org.python.indexer.NBinding.Kind.CONSTRUCTOR; +import static org.python.indexer.NBinding.Kind.FUNCTION; +import static org.python.indexer.NBinding.Kind.METHOD; +import static org.python.indexer.NBinding.Kind.MODULE; +import static org.python.indexer.NBinding.Kind.PARAMETER; +import static org.python.indexer.NBinding.Kind.SCOPE; +import static org.python.indexer.NBinding.Kind.VARIABLE; + +/** + * Handles binding names to scopes, including destructuring assignment. + */ +public class NameBinder { + + static private final NameBinder DEFAULT_BINDER = new NameBinder(); + static private final NameBinder ATTRIBUTE_BINDER = new NameBinder(ATTRIBUTE); + static private final NameBinder CLASS_BINDER = new NameBinder(CLASS); + static private final NameBinder CONSTRUCTOR_BINDER = new NameBinder(CONSTRUCTOR); + static private final NameBinder FUNCTION_BINDER = new NameBinder(FUNCTION); + static private final NameBinder METHOD_BINDER = new NameBinder(METHOD); + static private final NameBinder MODULE_BINDER = new NameBinder(MODULE); + static private final NameBinder PARAMETER_BINDER = new NameBinder(PARAMETER); + static private final NameBinder VARIABLE_BINDER = new NameBinder(VARIABLE); + + private NBinding.Kind kind; + + /** + * Factory method for creating instances. + */ + public static NameBinder make() { + return DEFAULT_BINDER; + } + + /** + * Factory method for creating instances. + * @return a {@code NameBinder} that will create bindings of {@code kind}, + * overriding the default choices. + */ + public static NameBinder make(NBinding.Kind kind) { + switch (kind) { + case ATTRIBUTE: + return ATTRIBUTE_BINDER; + case CLASS: + return CLASS_BINDER; + case CONSTRUCTOR: + return CONSTRUCTOR_BINDER; + case FUNCTION: + return FUNCTION_BINDER; + case METHOD: + return METHOD_BINDER; + case MODULE: + return MODULE_BINDER; + case PARAMETER: + return PARAMETER_BINDER; + case VARIABLE: + return VARIABLE_BINDER; + default: + return DEFAULT_BINDER; + } + } + + private NameBinder() { + } + + private NameBinder(NBinding.Kind kind) { + this.kind = kind; + } + + public void bind(Scope s, NNode target, NType rvalue) throws Exception { + if (target instanceof NName) { + bindName(s, (NName)target, rvalue); + return; + } + if (target instanceof NTuple) { + bind(s, ((NTuple)target).elts, rvalue); + return; + } + if (target instanceof NList) { + bind(s, ((NList)target).elts, rvalue); + return; + } + if (target instanceof NAttribute) { + // This causes various problems if we let it happen during the + // name-binding pass. I believe the only name-binding context + // in which an NAttribute can be an lvalue is in an assignment. + // Assignments are statements, so they can only appear in blocks. + // Hence the scope for the top-level name is unambiguous; we can + // safely leave binding it until the resolve pass. + if (!s.isNameBindingPhase()) { + ((NAttribute)target).setAttr(s, rvalue); + } + return; + } + if (target instanceof NSubscript) { + // Ditto. No resolving is allowed during the name-binding phase. + if (!s.isNameBindingPhase()) { + target.resolveExpr(target, s); + } + return; + } + Indexer.idx.putProblem(target, "invalid location for assignment"); + } + + public void bind(Scope s, List xs, NType rvalue) throws Exception { + if (rvalue.isTupleType()) { + List vs = rvalue.asTupleType().getElementTypes(); + if (xs.size() != vs.size()) { + reportUnpackMismatch(xs, vs.size()); + } else { + for (int i = 0; i < xs.size(); i++) { + bind(s, xs.get(i), vs.get(i)); + } + } + return; + } + + if (rvalue.isListType()) { + bind(s, xs, rvalue.asListType().toTupleType(xs.size())); + return; + } + + if (rvalue.isDictType()) { + bind(s, xs, rvalue.asDictType().toTupleType(xs.size())); + return; + } + + if (!rvalue.isUnknownType()) { + Indexer.idx.putProblem(xs.get(0).getFile(), + xs.get(0).start(), + xs.get(xs.size()-1).end(), + "unpacking non-iterable: " + rvalue); + } + for (int i = 0; i < xs.size(); i++) { + bind(s, xs.get(i), new NUnknownType()); + } + } + + public NBinding bindName(Scope s, NName name, NType rvalue) throws Exception { + NBinding b; + + if (s.isGlobalName(name.id)) { + b = s.getGlobalTable().put(name.id, name, rvalue, kindOr(SCOPE)); + Indexer.idx.putLocation(name, b); + } else { + Scope bindingScope = s.getScopeSymtab(); + b = bindingScope.put(name.id, name, rvalue, + kindOr(bindingScope.isFunctionScope() ? VARIABLE : SCOPE)); + } + + name.setType(b.followType()); + + // XXX: this seems like a bit of a hack; should at least figure out + // and document what use cases require it. + NType nameType = name.getType(); + if (!(nameType.isModuleType() || nameType.isClassType())) { + nameType.getTable().setPath(b.getQname()); + } + + return b; + } + + public void bindIter(Scope s, NNode target, NNode iter) throws Exception { + NType iterType = NNode.resolveExpr(iter, s); + + if (iterType.isListType()) { + bind(s, target, iterType.asListType().getElementType()); + } else if (iterType.isTupleType()) { + bind(s, target, iterType.asTupleType().toListType().getElementType()); + } else { + NBinding ent = iterType.getTable().lookupAttr("__iter__"); + if (ent == null || !ent.getType().isFuncType()) { + if (!iterType.isUnknownType()) { + iter.addWarning("not an iterable type: " + iterType); + } + bind(s, target, new NUnknownType()); + } else { + bind(s, target, ent.getType().asFuncType().getReturnType()); + } + } + } + + private void reportUnpackMismatch(List xs, int vsize) { + int xsize = xs.size(); + int beg = xs.get(0).start(); + int end = xs.get(xs.size() - 1).end(); + int diff = xsize - vsize; + String msg; + if (diff > 0) { + msg = "ValueError: need more than " + vsize + " values to unpack"; + } else { + msg = "ValueError: too many values to unpack"; + } + Indexer.idx.putProblem(xs.get(0).getFile(), beg, end, msg); + } + + private NBinding.Kind kindOr(NBinding.Kind k) { + return kind != null ? kind : k; + } +} Index: tests/java/org/python/indexer/data/yinw/yinw-21.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-21.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-21.py (revision 0) @@ -0,0 +1,30 @@ +class A: + a = 1 + +class B: + a = 2 + +def g(x): + print x.a + +def h(x): + print x.a + +def gen(u): + if (u>10): return g + elif (u>5): return 1 + else: return h + +ff = gen(2) + +# o1 = A() +# o2 = B() + +# ff(o1) +# ff(o2) + +def k(t): + if (t>1): return A() + else: return B() + +ff(k(2)) Index: tests/java/org/python/indexer/data/foo.py =================================================================== --- tests/java/org/python/indexer/data/foo.py (revision 0) +++ tests/java/org/python/indexer/data/foo.py (revision 0) @@ -0,0 +1,6 @@ +def f(x): + return x+1 + +def g(y): + x = 2 + return x+1 Index: tests/java/org/python/indexer/data/pkg/misc/__init__.py =================================================================== --- tests/java/org/python/indexer/data/pkg/misc/__init__.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/misc/__init__.py (revision 0) @@ -0,0 +1 @@ + Index: src/org/python/antlr/ast/Global.java =================================================================== --- src/org/python/antlr/ast/Global.java (revision 6695) +++ src/org/python/antlr/ast/Global.java (working copy) @@ -40,8 +40,11 @@ public void setNames(PyObject names) { this.names = AstAdapters.py2identifierList(names); } + private java.util.List nameNodes; + public java.util.List getInternalNameNodes() { + return nameNodes; + } - private final static PyString[] fields = new PyString[] {new PyString("names")}; @ExposedGet(name = "_fields") @@ -85,6 +88,12 @@ this.names = names; } + public Global(Integer ttype, Token token, java.util.List names, java.util.List nameNodes) { + super(ttype, token); + this.names = names; + this.nameNodes = nameNodes; + } + public Global(Integer ttype, Token token, java.util.List names) { super(ttype, token); this.names = names; Index: src/org/python/indexer/ast/NNodeVisitor.java =================================================================== --- src/org/python/indexer/ast/NNodeVisitor.java (revision 0) +++ src/org/python/indexer/ast/NNodeVisitor.java (revision 0) @@ -0,0 +1,73 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +/** + * Preorder-traversal node visitor interface. + */ +public interface NNodeVisitor { + /** + * Convenience exception for subclasses. The caller that initiates + * the visit should catch this exception if the subclass is expected + * to throw it. + */ + public static final class StopIterationException extends RuntimeException { + public StopIterationException() {} + } + + public boolean visit(NAlias m); + public boolean visit(NAssert m); + public boolean visit(NAssign m); + public boolean visit(NAttribute m); + public boolean visit(NAugAssign m); + public boolean visit(NBinOp m); + public boolean visit(NBlock m); + public boolean visit(NBoolOp m); + public boolean visit(NBreak m); + public boolean visit(NCall m); + public boolean visit(NClassDef m); + public boolean visit(NCompare m); + public boolean visit(NComprehension m); + public boolean visit(NContinue m); + public boolean visit(NDelete m); + public boolean visit(NDict m); + public boolean visit(NEllipsis m); + public boolean visit(NExceptHandler m); + public boolean visit(NExec m); + public boolean visit(NFor m); + public boolean visit(NFunctionDef m); + public boolean visit(NGeneratorExp m); + public boolean visit(NGlobal m); + public boolean visit(NIf m); + public boolean visit(NIfExp m); + public boolean visit(NImport m); + public boolean visit(NImportFrom m); + public boolean visit(NIndex m); + public boolean visit(NKeyword m); + public boolean visit(NLambda m); + public boolean visit(NList m); + public boolean visit(NListComp m); + public boolean visit(NModule m); + public boolean visit(NName m); + public boolean visit(NNum m); + public boolean visit(NPass m); + public boolean visit(NPlaceHolder m); + public boolean visit(NPrint m); + public boolean visit(NQname m); + public boolean visit(NRaise m); + public boolean visit(NRepr m); + public boolean visit(NReturn m); + public boolean visit(NExprStmt m); + public boolean visit(NSlice m); + public boolean visit(NStr m); + public boolean visit(NSubscript m); + public boolean visit(NTryExcept m); + public boolean visit(NTryFinally m); + public boolean visit(NTuple m); + public boolean visit(NUnaryOp m); + public boolean visit(NUrl m); + public boolean visit(NWhile m); + public boolean visit(NWith m); + public boolean visit(NYield m); +} Index: src/org/python/indexer/AstConverter.java =================================================================== --- src/org/python/indexer/AstConverter.java (revision 0) +++ src/org/python/indexer/AstConverter.java (revision 0) @@ -0,0 +1,675 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import org.python.antlr.PythonTree; +import org.python.antlr.Visitor; +import org.python.antlr.ast.Assert; +import org.python.antlr.ast.Assign; +import org.python.antlr.ast.Attribute; +import org.python.antlr.ast.AugAssign; +import org.python.antlr.ast.BinOp; +import org.python.antlr.ast.BoolOp; +import org.python.antlr.ast.Break; +import org.python.antlr.ast.Call; +import org.python.antlr.ast.ClassDef; +import org.python.antlr.ast.Compare; +import org.python.antlr.ast.Continue; +import org.python.antlr.ast.Delete; +import org.python.antlr.ast.Dict; +import org.python.antlr.ast.Ellipsis; +import org.python.antlr.ast.ExceptHandler; +import org.python.antlr.ast.Exec; +import org.python.antlr.ast.Expr; +import org.python.antlr.ast.For; +import org.python.antlr.ast.FunctionDef; +import org.python.antlr.ast.GeneratorExp; +import org.python.antlr.ast.Global; +import org.python.antlr.ast.If; +import org.python.antlr.ast.IfExp; +import org.python.antlr.ast.Import; +import org.python.antlr.ast.ImportFrom; +import org.python.antlr.ast.Index; +import org.python.antlr.ast.Lambda; +import org.python.antlr.ast.ListComp; +import org.python.antlr.ast.Module; +import org.python.antlr.ast.Name; +import org.python.antlr.ast.Num; +import org.python.antlr.ast.Pass; +import org.python.antlr.ast.Print; +import org.python.antlr.ast.Raise; +import org.python.antlr.ast.Repr; +import org.python.antlr.ast.Return; +import org.python.antlr.ast.Slice; +import org.python.antlr.ast.Str; +import org.python.antlr.ast.Subscript; +import org.python.antlr.ast.TryExcept; +import org.python.antlr.ast.TryFinally; +import org.python.antlr.ast.Tuple; +import org.python.antlr.ast.UnaryOp; +import org.python.antlr.ast.While; +import org.python.antlr.ast.With; +import org.python.antlr.ast.Yield; +import org.python.antlr.ast.alias; +import org.python.antlr.ast.arguments; +import org.python.antlr.ast.boolopType; +import org.python.antlr.ast.cmpopType; +import org.python.antlr.ast.comprehension; +import org.python.antlr.ast.keyword; +import org.python.antlr.ast.operatorType; +import org.python.antlr.ast.unaryopType; +import org.python.antlr.base.excepthandler; +import org.python.antlr.base.expr; +import org.python.antlr.base.stmt; +import org.python.indexer.ast.NAlias; +import org.python.indexer.ast.NAssert; +import org.python.indexer.ast.NAssign; +import org.python.indexer.ast.NAttribute; +import org.python.indexer.ast.NAugAssign; +import org.python.indexer.ast.NBinOp; +import org.python.indexer.ast.NBlock; +import org.python.indexer.ast.NBoolOp; +import org.python.indexer.ast.NBreak; +import org.python.indexer.ast.NCall; +import org.python.indexer.ast.NClassDef; +import org.python.indexer.ast.NCompare; +import org.python.indexer.ast.NComprehension; +import org.python.indexer.ast.NContinue; +import org.python.indexer.ast.NDelete; +import org.python.indexer.ast.NDict; +import org.python.indexer.ast.NEllipsis; +import org.python.indexer.ast.NExceptHandler; +import org.python.indexer.ast.NExec; +import org.python.indexer.ast.NFor; +import org.python.indexer.ast.NFunctionDef; +import org.python.indexer.ast.NGeneratorExp; +import org.python.indexer.ast.NGlobal; +import org.python.indexer.ast.NIf; +import org.python.indexer.ast.NIfExp; +import org.python.indexer.ast.NImport; +import org.python.indexer.ast.NImportFrom; +import org.python.indexer.ast.NIndex; +import org.python.indexer.ast.NKeyword; +import org.python.indexer.ast.NLambda; +import org.python.indexer.ast.NList; +import org.python.indexer.ast.NListComp; +import org.python.indexer.ast.NModule; +import org.python.indexer.ast.NName; +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NNum; +import org.python.indexer.ast.NPass; +import org.python.indexer.ast.NPrint; +import org.python.indexer.ast.NQname; +import org.python.indexer.ast.NRaise; +import org.python.indexer.ast.NRepr; +import org.python.indexer.ast.NReturn; +import org.python.indexer.ast.NExprStmt; +import org.python.indexer.ast.NSlice; +import org.python.indexer.ast.NStr; +import org.python.indexer.ast.NSubscript; +import org.python.indexer.ast.NTryExcept; +import org.python.indexer.ast.NTryFinally; +import org.python.indexer.ast.NTuple; +import org.python.indexer.ast.NUnaryOp; +import org.python.indexer.ast.NWhile; +import org.python.indexer.ast.NWith; +import org.python.indexer.ast.NYield; + +import java.util.ArrayList; +import java.util.List; + +/** + * Converts the antlr AST into the indexer's AST format. + */ +public class AstConverter extends Visitor { + + public String convOp(Object t) { + if (t instanceof operatorType) { + switch((operatorType)t) { + case Add: + return "+"; + case Sub: + return "-"; + case Mult: + return "*"; + case Div: + return "/"; + case Mod: + return "%"; + case Pow: + return "**"; + case LShift: + return "<<"; + case RShift: + return ">>"; + case BitOr: + return "|"; + case BitXor: + return "^"; + case BitAnd: + return "&"; + case FloorDiv: + return "//"; + default: + return null; + } + } + if (t instanceof boolopType) { + switch ((boolopType)t) { + case And: + return "and"; + case Or: + return "or"; + default: + return null; + } + } + if (t instanceof unaryopType) { + switch ((unaryopType)t) { + case Invert: + return "~"; + case Not: + return "not"; + case USub: + return "-"; + case UAdd: + return "+"; + default: + return null; + } + } + if (t instanceof cmpopType) { + switch ((cmpopType)t) { + case Eq: + return "=="; + case NotEq: + return "!="; + case Gt: + return ">"; + case GtE: + return ">="; + case Lt: + return "<"; + case LtE: + return "<="; + case In: + return "in"; + case NotIn: + return "not in"; + case Is: + return "is"; + case IsNot: + return "is not"; + default: + return null; + } + } + return null; + } + + // Helpers for converting lists of things + + private List convertListExceptHandler(List in) throws Exception { + List out = new ArrayList(in == null ? 0 : in.size()); + if (in != null) { + for (excepthandler e : in) { + NExceptHandler nxh = (NExceptHandler)e.accept(this); + if (nxh != null) { + out.add(nxh); + } + } + } + return out; + } + + private List convertListExpr(List in) throws Exception { + List out = new ArrayList(in == null ? 0 : in.size()); + if (in != null) { + for (expr e : in) { + NNode nx = (NNode)e.accept(this); + if (nx != null) { + out.add(nx); + } + } + } + return out; + } + + private List convertListName(List in) throws Exception { + List out = new ArrayList(in == null ? 0 : in.size()); + if (in != null) { + for (expr e : in) { + NName nn = (NName)e.accept(this); + if (nn != null) { + out.add(nn); + } + } + } + return out; + } + + private NQname convertQname(List in) throws Exception { + if (in == null) { + return null; + } + // This would be less ugly if we generated Qname nodes in the antlr ast. + NQname out = null; + int end = -1; + for (int i = in.size() - 1; i >= 0; i--) { + Name n = in.get(i); + if (end == -1) { + end = n.getCharStopIndex(); + } + NName nn = (NName)n.accept(this); + out = new NQname(out, nn, n.getCharStartIndex(), end); + } + return out; + } + + private List convertListKeyword(List in) throws Exception { + List out = new ArrayList(in == null ? 0 : in.size()); + if (in != null) { + for (keyword e : in) { + NKeyword nkw = new NKeyword(e.getInternalArg(), convExpr(e.getInternalValue())); + if (nkw != null) { + out.add(nkw); + } + } + } + return out; + } + + private NBlock convertListStmt(List in) throws Exception { + List out = new ArrayList(in == null ? 0 : in.size()); + if (in != null) { + for (stmt e : in) { + NNode nx = (NNode)e.accept(this); + if (nx != null) { + out.add(nx); + } + } + } + return new NBlock(out, 0, 0); + } + + private NNode convExpr(PythonTree e) throws Exception { + if (e == null) { + return null; + } + Object o = e.accept(this); + if (o instanceof NNode) { + return (NNode)o; + } + return null; + } + + private int start(PythonTree tree) { + return tree.getCharStartIndex(); + } + + private int stop(PythonTree tree) { + return tree.getCharStopIndex(); + } + + @Override + public Object visitAssert(Assert n) throws Exception { + return new NAssert(convExpr(n.getInternalTest()), + convExpr(n.getInternalMsg()), + start(n), stop(n)); + } + + @Override + public Object visitAssign(Assign n) throws Exception { + return new NAssign(convertListExpr(n.getInternalTargets()), + convExpr(n.getInternalValue()), + start(n), stop(n)); + } + + @Override + public Object visitAttribute(Attribute n) throws Exception { + return new NAttribute(convExpr(n.getInternalValue()), + (NName)convExpr(n.getInternalAttrName()), + start(n), stop(n)); + } + + @Override + public Object visitAugAssign(AugAssign n) throws Exception { + return new NAugAssign(convExpr(n.getInternalTarget()), + convExpr(n.getInternalValue()), + convOp(n.getInternalOp()), + start(n), stop(n)); + } + + @Override + public Object visitBinOp(BinOp n) throws Exception { + return new NBinOp(convExpr(n.getInternalLeft()), + convExpr(n.getInternalRight()), + convOp(n.getInternalOp()), + start(n), stop(n)); + } + + @Override + public Object visitBoolOp(BoolOp n) throws Exception { + NBoolOp.OpType op; + switch ((boolopType)n.getInternalOp()) { + case And: + op = NBoolOp.OpType.AND; + break; + case Or: + op = NBoolOp.OpType.OR; + break; + default: + op = NBoolOp.OpType.UNDEFINED; + break; + } + return new NBoolOp(op, convertListExpr(n.getInternalValues()), start(n), stop(n)); + } + + @Override + public Object visitBreak(Break n) throws Exception { + return new NBreak(start(n), stop(n)); + } + + @Override + public Object visitCall(Call n) throws Exception { + return new NCall(convExpr(n.getInternalFunc()), + convertListExpr(n.getInternalArgs()), + convertListKeyword(n.getInternalKeywords()), + convExpr(n.getInternalKwargs()), + convExpr(n.getInternalStarargs()), + start(n), stop(n)); + } + + @Override + public Object visitClassDef(ClassDef n) throws Exception { + return new NClassDef((NName)convExpr(n.getInternalNameNode()), + convertListExpr(n.getInternalBases()), + convertListStmt(n.getInternalBody()), + start(n), stop(n)); + } + + @Override + public Object visitCompare(Compare n) throws Exception { + return new NCompare(convExpr(n.getInternalLeft()), + null, // XXX: why null? + convertListExpr(n.getInternalComparators()), + start(n), stop(n)); + } + + @Override + public Object visitContinue(Continue n) throws Exception { + return new NContinue(start(n), stop(n)); + } + + @Override + public Object visitDelete(Delete n) throws Exception { + return new NDelete(convertListExpr(n.getInternalTargets()), start(n), stop(n)); + } + + @Override + public Object visitDict(Dict n) throws Exception { + return new NDict(convertListExpr(n.getInternalKeys()), + convertListExpr(n.getInternalValues()), + start(n), stop(n)); + } + + @Override + public Object visitEllipsis(Ellipsis n) throws Exception { + return new NEllipsis(start(n), stop(n)); + } + + @Override + public Object visitExceptHandler(ExceptHandler n) throws Exception { + return new NExceptHandler(convExpr(n.getInternalName()), + convExpr(n.getInternalType()), + convertListStmt(n.getInternalBody()), + start(n), stop(n)); + } + + @Override + public Object visitExec(Exec n) throws Exception { + return new NExec(convExpr(n.getInternalBody()), + convExpr(n.getInternalGlobals()), + convExpr(n.getInternalLocals()), + start(n), stop(n)); + } + + @Override + public Object visitExpr(Expr n) throws Exception { + return new NExprStmt(convExpr(n.getInternalValue()), start(n), stop(n)); + } + + @Override + public Object visitFor(For n) throws Exception { + return new NFor(convExpr(n.getInternalTarget()), + convExpr(n.getInternalIter()), + convertListStmt(n.getInternalBody()), + convertListStmt(n.getInternalOrelse()), + start(n), stop(n)); + } + + @Override + public Object visitFunctionDef(FunctionDef n) throws Exception { + arguments args = n.getInternalArgs(); + NFunctionDef fn = new NFunctionDef((NName)convExpr(n.getInternalNameNode()), + convertListExpr(args.getInternalArgs()), + convertListStmt(n.getInternalBody()), + convertListExpr(args.getInternalDefaults()), + (NName)convExpr(args.getInternalVarargName()), + (NName)convExpr(args.getInternalKwargName()), + start(n), stop(n)); + fn.setDecoratorList(convertListExpr(n.getInternalDecorator_list())); + return fn; + } + + @Override + public Object visitGeneratorExp(GeneratorExp n) throws Exception { + List generators = + new ArrayList(n.getInternalGenerators().size()); + for (comprehension c : n.getInternalGenerators()) { + generators.add(new NComprehension(convExpr(c.getInternalTarget()), + convExpr(c.getInternalIter()), + convertListExpr(c.getInternalIfs()), + start(c), stop(c))); + } + return new NGeneratorExp(convExpr(n.getInternalElt()), generators, start(n), stop(n)); + } + + @Override + public Object visitGlobal(Global n) throws Exception { + return new NGlobal(convertListName(n.getInternalNameNodes()), + start(n), stop(n)); + } + + @Override + public Object visitIf(If n) throws Exception { + return new NIf(convExpr(n.getInternalTest()), + convertListStmt(n.getInternalBody()), + convertListStmt(n.getInternalOrelse()), + start(n), stop(n)); + } + + @Override + public Object visitIfExp(IfExp n) throws Exception { + return new NIfExp(convExpr(n.getInternalTest()), + convExpr(n.getInternalBody()), + convExpr(n.getInternalOrelse()), + start(n), stop(n)); + } + + @Override + public Object visitImport(Import n) throws Exception { + List aliases = new ArrayList(n.getInternalNames().size()); + for (alias e : n.getInternalNames()) { + aliases.add(new NAlias(e.getInternalName(), + convertQname(e.getInternalNameNodes()), + (NName)convExpr(e.getInternalAsnameNode()), + start(e), stop(e))); + } + return new NImport(aliases, start(n), stop(n)); + } + + @Override + public Object visitImportFrom(ImportFrom n) throws Exception { + List aliases = new ArrayList(n.getInternalNames().size()); + for (alias e : n.getInternalNames()) { + aliases.add(new NAlias(e.getInternalName(), + convertQname(e.getInternalNameNodes()), + (NName)convExpr(e.getInternalAsnameNode()), + start(e), stop(e))); + } + return new NImportFrom(n.getInternalModule(), + convertQname(n.getInternalModuleNames()), + aliases, start(n), stop(n)); + } + + @Override + public Object visitIndex(Index n) throws Exception { + return new NIndex(convExpr(n.getInternalValue()), start(n), stop(n)); + } + + @Override + public Object visitLambda(Lambda n) throws Exception { + arguments args = n.getInternalArgs(); + return new NLambda(convertListExpr(args.getInternalArgs()), + convExpr(n.getInternalBody()), + convertListExpr(args.getInternalDefaults()), + (NName)convExpr(args.getInternalVarargName()), + (NName)convExpr(args.getInternalKwargName()), + start(n), stop(n)); + } + + @Override + public Object visitList(org.python.antlr.ast.List n) throws Exception { + return new NList(convertListExpr(n.getInternalElts()), start(n), stop(n)); + } + + // This is more complex than it should be, but let's wait until Jython add + // visitors to comprehensions + @Override + public Object visitListComp(ListComp n) throws Exception { + List generators = + new ArrayList(n.getInternalGenerators().size()); + for (comprehension c : n.getInternalGenerators()) { + generators.add(new NComprehension(convExpr(c.getInternalTarget()), + convExpr(c.getInternalIter()), + convertListExpr(c.getInternalIfs()), + start(c), stop(c))); + } + return new NListComp(convExpr(n.getInternalElt()), generators, start(n), stop(n)); + } + + @Override + public Object visitModule(Module n) throws Exception { + return new NModule(convertListStmt(n.getInternalBody()), start(n), stop(n)); + } + + @Override + public Object visitName(Name n) throws Exception { + return new NName(n.getInternalId(), start(n), stop(n)); + } + + @Override + public Object visitNum(Num n) throws Exception { + return new NNum(n.getInternalN(), start(n), stop(n)); + } + + @Override + public Object visitPass(Pass n) throws Exception { + return new NPass(start(n), stop(n)); + } + + @Override + public Object visitPrint(Print n) throws Exception { + return new NPrint(convExpr(n.getInternalDest()), + convertListExpr(n.getInternalValues()), + start(n), stop(n)); + } + + @Override + public Object visitRaise(Raise n) throws Exception { + return new NRaise(convExpr(n.getInternalType()), + convExpr(n.getInternalInst()), + convExpr(n.getInternalTback()), + start(n), stop(n)); + } + + @Override + public Object visitRepr(Repr n) throws Exception { + return new NRepr(convExpr(n.getInternalValue()), start(n), stop(n)); + } + + @Override + public Object visitReturn(Return n) throws Exception { + return new NReturn(convExpr(n.getInternalValue()), start(n), stop(n)); + } + + @Override + public Object visitSlice(Slice n) throws Exception { + return new NSlice(convExpr(n.getInternalLower()), + convExpr(n.getInternalStep()), + convExpr(n.getInternalUpper()), + start(n), stop(n)); + } + + @Override + public Object visitStr(Str n) throws Exception { + return new NStr(n.getInternalS(), start(n), stop(n)); + } + + @Override + public Object visitSubscript(Subscript n) throws Exception { + return new NSubscript(convExpr(n.getInternalValue()), + convExpr(n.getInternalSlice()), + start(n), stop(n)); + } + + @Override + public Object visitTryExcept(TryExcept n) throws Exception { + return new NTryExcept(convertListExceptHandler(n.getInternalHandlers()), + convertListStmt(n.getInternalBody()), + convertListStmt(n.getInternalOrelse()), + start(n), stop(n)); + } + + @Override + public Object visitTryFinally(TryFinally n) throws Exception { + return new NTryFinally(convertListStmt(n.getInternalBody()), + convertListStmt(n.getInternalFinalbody()), + start(n), stop(n)); + } + + @Override + public Object visitTuple(Tuple n) throws Exception { + return new NTuple(convertListExpr(n.getInternalElts()), start(n), stop(n)); + } + + @Override + public Object visitUnaryOp(UnaryOp n) throws Exception { + return new NUnaryOp(null, // XXX: why null for operator? + convExpr(n.getInternalOperand()), + start(n), stop(n)); + } + + @Override + public Object visitWhile(While n) throws Exception { + return new NWhile(convExpr(n.getInternalTest()), + convertListStmt(n.getInternalBody()), + convertListStmt(n.getInternalOrelse()), + start(n), stop(n)); + } + + @Override + public Object visitWith(With n) throws Exception { + return new NWith(convExpr(n.getInternalOptional_vars()), + convExpr(n.getInternalContext_expr()), + convertListStmt(n.getInternalBody()), + start(n), stop(n)); + } + + @Override + public Object visitYield(Yield n) throws Exception { + return new NYield(convExpr(n.getInternalValue()), start(n), stop(n)); + } +} Index: src/org/python/antlr/ast/alias.java =================================================================== --- src/org/python/antlr/ast/alias.java (revision 6695) +++ src/org/python/antlr/ast/alias.java (working copy) @@ -42,6 +42,10 @@ this.name = AstAdapters.py2identifier(name); } + private java.util.List nameNodes; + public java.util.List getInternalNameNodes() { + return nameNodes; + } private String asname; public String getInternalAsname() { return asname; @@ -56,6 +60,10 @@ this.asname = AstAdapters.py2identifier(asname); } + private Name asnameNode; + public Name getInternalAsnameNode() { + return asnameNode; + } private final static PyString[] fields = new PyString[] {new PyString("name"), new PyString("asname")}; @@ -92,6 +100,21 @@ this.asname = asname; } + // [import] name [as asname] + public alias(Name name, Name asname) { + this(java.util.Arrays.asList(new Name[]{name}), asname); + } + + // [import] ...foo.bar.baz [as asname] + public alias(java.util.List nameNodes, Name asname) { + this.nameNodes = nameNodes; + this.name = dottedNameListToString(nameNodes); + if (asname != null) { + this.asnameNode = asname; + this.asname = asname.getInternalId(); + } + } + public alias(Integer ttype, Token token, String name, String asname) { super(ttype, token); this.name = name; Index: src/org/python/indexer/Util.java =================================================================== --- src/org/python/indexer/Util.java (revision 0) +++ src/org/python/indexer/Util.java (revision 0) @@ -0,0 +1,211 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.util.Collection; +import java.util.Set; +import java.util.TreeSet; + +public class Util { + + public static final Charset UTF_8 = Charset.forName("UTF-8"); + + private static int gensymCount = -1; + + public static String gensym(String base) { + gensymCount++; + return base + gensymCount; + } + + public static String getSystemTempDir() { + String tmp = System.getProperty("java.io.tmpdir"); + String sep = System.getProperty("file.separator"); + if (tmp.endsWith(sep)) { + return tmp; + } + return tmp + sep; + } + + /** + * Returns the parent qname of {@code qname} -- everything up to the + * last dot (exclusive), or if there are no dots, the empty string. + */ + public static String getQnameParent(String qname) { + if (qname == null || qname.isEmpty()) { + return ""; + } + int index = qname.lastIndexOf("."); + if (index == -1) { + return ""; + } + return qname.substring(0, index); + } + + /** + * Determines the fully-qualified module name for the specified file. A + * module's qname is a function of the module's absolute path and the sys + * path; it does not depend on how the module name may have been specified + * in import statements. The module qname is the relative path of the module + * under the load path, with slashes converted to dots. + * + * @param file absolute canonical path to a file (__init__.py for dirs) + * @return null if {@code file} is not somewhere under the load path + */ + public static String moduleQname(String file) { + boolean initpy = file.endsWith("/__init__.py"); + if (initpy) { + file = file.substring(0, file.length() - "/__init__.py".length()); + } else if (file.endsWith(".py")) { + file = file.substring(0, file.length() - ".py".length()); + } + for (String root : Indexer.idx.getLoadPath()) { + if (file.startsWith(root)) { + return file.substring(root.length()).replace('/', '.'); + } + } + return null; + } + + public static String arrayToString(Collection strings) { + StringBuffer sb = new StringBuffer(); + for (String s : strings) { + sb.append(s).append("\n"); + } + return sb.toString(); + } + + public static String arrayToSortedStringSet(Collection strings) { + Set sorter = new TreeSet(); + sorter.addAll(strings); + return arrayToString(sorter); + } + + /** + * Given an absolute {@code path} to a file (not a directory), + * returns the module name for the file. If the file is an __init__.py, + * returns the last component of the file's parent directory, else + * returns the filename without path or extension. + */ + public static String moduleNameFor(String path) { + File f = new File(path); + if (f.isDirectory()) { + throw new IllegalStateException("failed assertion: " + path); + } + String fname = f.getName(); + if (fname.equals("__init__.py")) { + return f.getParentFile().getName(); + } + return fname.substring(0, fname.lastIndexOf('.')); + } + + public static File joinPath(File dir, String file) { + return joinPath(dir.getAbsolutePath(), file); + } + + public static File joinPath(String dir, String file) { + if (dir.endsWith("/")) { + return new File(dir + file); + } + return new File(dir + "/" + file); + } + + public static void writeFile(String path, String contents) throws Exception { + PrintWriter out = null; + try { + out = new PrintWriter(new BufferedWriter(new FileWriter(path))); + out.print(contents); + out.flush(); + } finally { + if (out != null) { + out.close(); + } + } + } + + public static String readFile(String filename) throws Exception { + return readFile(new File(filename)); + } + + public static String readFile(File path) throws Exception { + // Don't use line-oriented file read -- need to retain CRLF if present + // so the style-run and link offsets are correct. + return new String(getBytesFromFile(path), UTF_8); + } + + public static byte[] getBytesFromFile(File file) throws IOException { + InputStream is = null; + + try { + is = new FileInputStream(file); + long length = file.length(); + if (length > Integer.MAX_VALUE) { + throw new IOException("file too large: " + file); + } + + byte[] bytes = new byte[(int)length]; + int offset = 0; + int numRead = 0; + while (offset < bytes.length + && (numRead = is.read(bytes, offset, bytes.length-offset)) >= 0) { + offset += numRead; + } + if (offset < bytes.length) { + throw new IOException("Failed to read whole file " + file); + } + return bytes; + } finally { + if (is != null) { + is.close(); + } + } + } + + public static String getMD5(File path) throws Exception { + byte[] bytes = getBytesFromFile(path); + return getMD5(bytes); + } + + public static String getMD5(byte[] fileContents) throws Exception { + MessageDigest algorithm = MessageDigest.getInstance("MD5"); + algorithm.reset(); + algorithm.update(fileContents); + byte messageDigest[] = algorithm.digest(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < messageDigest.length; i++) { + sb.append(String.format("%02x", 0xFF & messageDigest[i])); + } + return sb.toString(); + } + + /** + * Return absolute path for {@code path}. + * Make sure path ends with "/" if it's a directory. + * Does _not_ resolve symlinks, since the caller may need to play + * symlink tricks to produce the desired paths for loaded modules. + */ + public static String canonicalize(String path) throws IOException { + File f = new File(path); + path = f.getAbsolutePath(); + if (f.isDirectory() && !path.endsWith("/")) { + return path + "/"; + } + return path; + } + + static boolean isReadableFile(String path) { + File f = new File(path); + return f.canRead() && f.isFile(); + } +} Index: src/org/python/indexer/Def.java =================================================================== --- src/org/python/indexer/Def.java (revision 0) +++ src/org/python/indexer/Def.java (revision 0) @@ -0,0 +1,161 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import org.python.indexer.ast.NName; +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NUrl; + +/** + * Encapsulates information about a binding definition site. + */ +public class Def { + + // Being frugal with fields here is good for memory usage. + private int start; + private int end; + private NBinding binding; + private String fileOrUrl; + private String name; + + public Def(NNode node) { + this(node, null); + } + + public Def(NNode node, NBinding b) { + if (node == null) { + throw new IllegalArgumentException("null 'node' param"); + } + binding = b; + if (node instanceof NUrl) { + String url = ((NUrl)node).getURL(); + if (url.startsWith("file://")) { + fileOrUrl = url.substring("file://".length()); + } else { + fileOrUrl = url; + } + return; + } + + // start/end offsets are invalid/bogus for NUrls + start = node.start(); + end = node.end(); + fileOrUrl = node.getFile(); + if (fileOrUrl == null) { + throw new IllegalArgumentException("Non-URL nodes must have a non-null file"); + } + if (node instanceof NName) { + name = ((NName)node).id; + } + } + + /** + * Returns the name of the node. Only applies if the definition coincides + * with a {@link NName} node. + * @return the name, or null + */ + public String getName() { + return name; + } + + /** + * Returns the file if this node is from a source file, else {@code null}. + */ + public String getFile() { + return isURL() ? null : fileOrUrl; + } + + /** + * Returns the URL if this node is from a URL, else {@code null}. + */ + public String getURL() { + return isURL() ? fileOrUrl : null; + } + + /** + * Returns the file if from a source file, else the URL. + */ + public String getFileOrUrl() { + return fileOrUrl; + } + + /** + * Returns {@code true} if this node is from a URL. + */ + public boolean isURL() { + return fileOrUrl.startsWith("http://"); + } + + public boolean isModule() { + return binding != null && binding.kind == NBinding.Kind.MODULE; + } + + public int start() { + return start; + } + + public int end() { + return end; + } + + public int length() { + return end - start; + } + + public boolean isName() { + return name != null; + } + + void setBinding(NBinding b) { + binding = b; + } + + public NBinding getBinding() { + return binding; + } + + @Override + public String toString() { + return ""; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Def)) { + return false; + } + Def def = (Def)obj; + if (start != def.start) { + return false; + } + if (end != def.end) { + return false; + } + if (name != null) { + if (!name.equals(def.name)) { + return false; + } + } else { + if (def.name != null) { + return false; + } + } + if (fileOrUrl != null) { + if (!fileOrUrl.equals(def.fileOrUrl)) { + return false; + } + } else { + if (def.fileOrUrl != null) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + return ("" + fileOrUrl + name + start + end).hashCode(); + } +} Index: tests/java/org/python/indexer/data/pkg/animal/mammal/__init__.py =================================================================== Index: tests/java/org/python/indexer/data/pkg/plant/__init__.py =================================================================== Index: src/org/python/indexer/types/NUnknownType.java =================================================================== --- src/org/python/indexer/types/NUnknownType.java (revision 0) +++ src/org/python/indexer/types/NUnknownType.java (revision 0) @@ -0,0 +1,41 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +public class NUnknownType extends NType { + + private NType pointer; + + public NUnknownType() { + pointer = null; + } + + public static void point(NType u, NType v) { + if (u.isUnknownType()) { + ((NUnknownType)u).pointer = v; + } + } + + public static NType follow(NType t) { + if (t instanceof NUnknownType) { + NUnknownType tv = (NUnknownType)t; + if (tv.pointer != null) { + return follow(tv.pointer); + } else { + return tv; + } + } else { + return t; + } + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + if (pointer == null) { + sb.append("null"); + } else { + pointer.print(ctr, sb); + } + } +} Index: tests/java/org/python/indexer/data/attr_infer.py =================================================================== --- tests/java/org/python/indexer/data/attr_infer.py (revision 0) +++ tests/java/org/python/indexer/data/attr_infer.py (revision 0) @@ -0,0 +1,9 @@ + +# test provisional binding for attribute of +# a union of an unknown type and a known type +def foo(a): + # this line should make a temp binding for b in foo's scope, + # and + b.__add__(1) + b = a + b = 2 Index: src/org/python/indexer/demos/HtmlDemo.java =================================================================== --- src/org/python/indexer/demos/HtmlDemo.java (revision 0) +++ src/org/python/indexer/demos/HtmlDemo.java (revision 0) @@ -0,0 +1,167 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.demos; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.StyleRun; +import org.python.indexer.Util; + +import java.io.File; +import java.util.List; + +/** + * Simple proof-of-concept demo app for the indexer. Generates a static-html + * cross-referenced view of the code in a file or directory, using the index to + * create links and outlines.

+ * + * The demo not attempt to show general cross references (declarations and uses + * of a symbol) from the index, nor does it display the inferred type + * information or generated error/warning diagnostics. It could be made to do + * these things, as well as be made more configurable and generally useful, with + * additional effort.

+ * + * Run it from jython source tree root dir with, e.g.: to index /usr/lib/python2.4/email + *

+ * ant jar & & java -classpath ./dist/jython-dev.jar:./extlibs/antlr-runtime-3.1.3.jar:./extlibs/constantine.jar org.python.indexer.demos.HtmlDemo /usr/lib/python2.4/email
+ * 
+ * + * Fully indexing the Python standard library may require a more complete build to pick up all the dependencies: + *
+ * rm -rf ./html/ && ant clean && ant jar && ant jar-complete && java -classpath ./dist/jython.jar:./extlibs/antlr-runtime-3.1.3.jar:./extlibs/constantine.jar org.python.indexer.demos.HtmlDemo /usr/lib/python2.4/
+ * 
+ */ +public class HtmlDemo { + + private static final File OUTPUT_DIR = + new File(new File("./html").getAbsolutePath()); + + private static final String CSS = + ".builtin {color: #5b4eaf;}\n" + + ".comment, .block-comment {color: #005000; font-style: italic;}\n" + + ".constant {color: #888888;}\n" + + ".decorator {color: #778899;}\n" + + ".doc-string {color: #005000;}\n" + + ".error {border-bottom: 1px solid red;}\n" + + ".field-name {color: #2e8b57;}\n" + + ".function {color: #880000;}\n" + + ".identifier {color: #8b7765;}\n" + + ".info {border-bottom: 1px dotted RoyalBlue;}\n" + + ".keyword {color: #0000cd;}\n" + + ".lineno {color: #aaaaaa;}\n" + + ".number {color: #483d8b;}\n" + + ".parameter {color: #2e8b57;}\n" + + ".string {color: #4169e1;}\n" + + ".type-name {color: #4682b4;}\n" + + ".warning {border-bottom: 1px dotted orange;}\n"; + + private Indexer indexer; + private File rootDir; + private Linker linker; + + private static void abort(String msg) { + System.err.println(msg); + System.exit(1); + } + + private void info(Object msg) { + System.out.println(msg); + } + + private void makeOutputDir() throws Exception { + if (!OUTPUT_DIR.exists()) { + OUTPUT_DIR.mkdirs(); + info("created directory: " + OUTPUT_DIR.getAbsolutePath()); + } + } + + private void generateHtml() throws Exception { + info("generating html..."); + makeOutputDir(); + Util.writeFile(cssPath(), CSS); + linker = new Linker(rootDir, OUTPUT_DIR); + linker.findLinks(indexer); + + int rootLength = (int)rootDir.getAbsolutePath().length(); + for (String path : indexer.getLoadedFiles()) { + if (!path.startsWith(rootDir.getAbsolutePath())) { + continue; + } + File destFile = Util.joinPath(OUTPUT_DIR, path.substring(rootLength)); + destFile.getParentFile().mkdirs(); + String destPath = destFile.getAbsolutePath() + ".html"; + String html = markup(path); + Util.writeFile(destPath, html); + } + + info("wrote " + indexer.getLoadedFiles().size() + " files to " + OUTPUT_DIR); + } + + private String markup(String path) throws Exception { + String source = Util.readFile(path); + + List styles = new Styler(indexer, linker).addStyles(path, source); + styles.addAll(linker.getStyles(path)); + + source = new StyleApplier(path, source, styles).apply(); + + String outline = new HtmlOutline(indexer).generate(path); + + return "" + + "" + + "" + + "" + + "
" + + outline + + "" + + "
" + addLineNumbers(source) + "
" + + "
"; + } + + private String cssPath() { + return Util.joinPath(OUTPUT_DIR.getAbsolutePath(), "demo.css").toString(); + } + + private String addLineNumbers(String source) { + StringBuilder result = new StringBuilder((int)(source.length() * 1.2)); + int count = 1; + for (String line : source.split("\n")) { + result.append(""); + result.append(count++); + result.append(" "); + result.append(line); + result.append("\n"); + } + return result.toString(); + } + + private void start(File fileOrDir) throws Exception { + rootDir = fileOrDir.isFile() ? fileOrDir.getParentFile() : fileOrDir; + + indexer = new Indexer(); + indexer.addPath("/usr/lib/python2.4"); + info("building index..."); + indexer.loadFileRecursive(fileOrDir.getAbsolutePath()); + indexer.ready(); + + info(indexer.getStatusReport()); + generateHtml(); + } + + public static void main(String[] args) throws Exception { + // For now, just parse a file. + if (args.length < 1) { + abort("Usage: java org.python.indexer.HtmlDemo "); + } + + File fileOrDir = new File(args[0]); + if (!fileOrDir.exists()) { + abort("File not found: " + fileOrDir); + } + + new HtmlDemo().start(fileOrDir); + } + +} Index: src/org/python/indexer/ast/NSlice.java =================================================================== --- src/org/python/indexer/ast/NSlice.java (revision 0) +++ src/org/python/indexer/ast/NSlice.java (revision 0) @@ -0,0 +1,51 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NType; + +public class NSlice extends NNode { + + static final long serialVersionUID = 8685364390631331543L; + + public NNode lower; + public NNode step; + public NNode upper; + + public NSlice(NNode lower, NNode step, NNode upper) { + this(lower, step, upper, 0, 1); + } + + public NSlice(NNode lower, NNode step, NNode upper, int start, int end) { + super(start, end); + this.lower = lower; + this.step = step; + this.upper = upper; + addChildren(lower, step, upper); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(lower, s); + resolveExpr(step, s); + resolveExpr(upper, s); + return setType(new NListType()); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(lower, v); + visitNode(step, v); + visitNode(upper, v); + } + } +} Index: src/org/python/indexer/ast/NUrl.java =================================================================== --- src/org/python/indexer/ast/NUrl.java (revision 0) +++ src/org/python/indexer/ast/NUrl.java (revision 0) @@ -0,0 +1,51 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +/** + * Non-AST node used to represent virtual source locations for builtins + * as external urls. + */ +public class NUrl extends NNode { + + static final long serialVersionUID = -3488021036061979551L; + + private String url; + + private static int count = 0; + + public NUrl(String url) { + this.url = url == null ? "" : url; + setStart(count); + setEnd(count++); + } + + public NUrl(String url, int start, int end) { + super(start, end); + this.url = url == null ? "" : url; + } + + public String getURL() { + return url; + } + + @Override + public NType resolve(Scope s) throws Exception { + return setType(Indexer.idx.builtins.BaseStr); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: src/org/python/indexer/ast/NStr.java =================================================================== --- src/org/python/indexer/ast/NStr.java (revision 0) +++ src/org/python/indexer/ast/NStr.java (revision 0) @@ -0,0 +1,43 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NStr extends NNode { + + static final long serialVersionUID = -6092297133232624953L; + + public Object n; + + public NStr() { + this(""); + } + + public NStr(Object n) { + this.n = n; + } + + public NStr(Object n, int start, int end) { + super(start, end); + this.n = n; + } + + @Override + public NType resolve(Scope s) throws Exception { + return setType(Indexer.idx.builtins.BaseStr); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: src/org/python/indexer/ast/NKeyword.java =================================================================== --- src/org/python/indexer/ast/NKeyword.java (revision 0) +++ src/org/python/indexer/ast/NKeyword.java (revision 0) @@ -0,0 +1,46 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +/** + * Represents a keyword argument (name=value) in a function call. + */ +public class NKeyword extends NNode { + + static final long serialVersionUID = 9031782645918578266L; + + public String arg; + public NNode value; + + public NKeyword(String arg, NNode value) { + this(arg, value, 0, 1); + } + + public NKeyword(String arg, NNode value, int start, int end) { + super(start, end); + this.arg = arg; + this.value = value; + addChildren(value); + } + + @Override + public NType resolve(Scope s) throws Exception { + return setType(resolveExpr(value, s)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(value, v); + } + } +} Index: tests/java/org/python/indexer/data/refs.py =================================================================== --- tests/java/org/python/indexer/data/refs.py (revision 0) +++ tests/java/org/python/indexer/data/refs.py (revision 0) @@ -0,0 +1,25 @@ +# single definition with some references in various contexts + +foo = 3.14 # 1 + +print foo, [foo] # 2, 3 + +print {foo: foo} # 4, 5 + +print "foo %s" % 1 + foo # 6, 7 + +print str(foo) # 8 + +print ( + # the default parser had this name's offset starting at prev paren: + foo) # 9 + +print (foo,) # 10 + +print ([foo]), (-foo) # 11, 12 + +print lambda(foo): foo # 13, 14 + +print (lambda(x): foo)("bar") # 15 + +print (((foo))) # 16 Index: src/org/python/indexer/ast/NLambda.java =================================================================== --- src/org/python/indexer/ast/NLambda.java (revision 0) +++ src/org/python/indexer/ast/NLambda.java (revision 0) @@ -0,0 +1,129 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NFuncType; +import org.python.indexer.types.NTupleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +public class NLambda extends NFunctionDef { + + static final long serialVersionUID = 7737836525970653522L; + + private NName fname; + + public NLambda(List args, NNode body, List defaults, + NName varargs, NName kwargs) { + this(args, body, defaults, varargs, kwargs, 0, 1); + } + + public NLambda(List args, NNode body, List defaults, + NName varargs, NName kwargs, int start, int end) { + super(null, args, null, defaults, varargs, kwargs, start, end); + this.body = body instanceof NBlock ? new NBody((NBlock)body) : body; + addChildren(this.body); + } + + @Override + public boolean isLambda() { + return true; + } + + /** + * Returns the name of the function for indexing/qname purposes. + */ + @Override + protected String getBindingName(Scope s) { + if (fname != null) { + return fname.id; + } + String fn = s.newLambdaName(); + fname = new NName(fn, start(), start() + "lambda".length()); + fname.setParent(this); + return fn; + } + + @Override + protected void bindFunctionName(Scope owner) throws Exception { + NameBinder.make(NBinding.Kind.FUNCTION).bindName(owner, fname, getType()); + } + + @Override + protected void bindMethodAttrs(Scope owner) throws Exception { + // no-op + } + + @Override + public NType resolve(Scope s) throws Exception { + if (!getType().isFuncType()) { + org.python.indexer.Indexer.idx.reportFailedAssertion( + "Bad type on " + this + ": type=" + getType() + + " in file " + getFile() + " at " + start()); + } + NTupleType fromType = new NTupleType(); + NameBinder param = NameBinder.make(NBinding.Kind.PARAMETER); + + resolveList(defaults, s); + + Scope funcTable = getTable(); + int argnum = 0; + for (NNode a : args) { + NType argtype = NFunctionDef.getArgType(args, defaults, argnum++); + param.bind(funcTable, a, argtype); + fromType.add(argtype); + } + + if (varargs != null) { + NType u = new NUnknownType(); + param.bind(funcTable, varargs, u); + fromType.add(u); + } + + if (kwargs != null) { + NType u = new NUnknownType(); + param.bind(funcTable, kwargs, u); + fromType.add(u); + } + + // A lambda body is not an NBody, so it doesn't undergo the two + // pre-resolve passes for finding global statements and name-binding + // constructs. However, the lambda expression may itself contain + // name-binding constructs (generally, other lambdas), so we need to + // perform the name-binding pass on it before resolving. + try { + funcTable.setNameBindingPhase(true); + body.visit(new BindingFinder(funcTable)); + } finally { + funcTable.setNameBindingPhase(false); + } + + NType toType = resolveExpr(body, funcTable); + if (getType().isFuncType()) { // else warning logged at method entry above + getType().asFuncType().setReturnType(toType); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(args, v); + visitNodeList(defaults, v); + visitNode(varargs, v); + visitNode(kwargs, v); + visitNode(body, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/animal/reptile/croc.py =================================================================== --- tests/java/org/python/indexer/data/pkg/animal/reptile/croc.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/animal/reptile/croc.py (revision 0) @@ -0,0 +1,20 @@ +# Unit-test stuff. + +class Crocodilian: + def __init__(self): + self.size = "unknown" + +class Crocodile(Crocodilian): + def __init__(self): + Crocodilian.__init__(self) + self.size = "Huge" + +class Alligator(Crocodilian): + def __init__(self): + Crocodilian.__init__(self) + self.size = "Large" + +class Gavial(Crocodilian): + def __init__(self): + Crocodilian.__init__(self) + self.size = "Medium" Index: tests/java/org/python/indexer/data/pkg/misc/m3.py =================================================================== --- tests/java/org/python/indexer/data/pkg/misc/m3.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/misc/m3.py (revision 0) @@ -0,0 +1,6 @@ +# test in python shell with execfile after loading m2 +# >>> execfile("/abs/path/to/this/m3.py") + +import m2 + +print "m2.distutils: %s" % m2.distutils Index: tests/java/org/python/indexer/data/pkg/animal/mammal/cat.py =================================================================== --- tests/java/org/python/indexer/data/pkg/animal/mammal/cat.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/animal/mammal/cat.py (revision 0) @@ -0,0 +1,2 @@ +# reserved for circular import test +import pkg.plant.garden.catnip Index: tests/java/org/python/indexer/data/yinw/yinw-14.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-14.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-14.py (revision 0) @@ -0,0 +1,11 @@ +x = 1 +x = 2 +x = "hello" + +print x + +def f(): + x = 1 + x = 2 + x = "hello" + print x Index: tests/java/org/python/indexer/data/pkg/misc/m2.py =================================================================== --- tests/java/org/python/indexer/data/pkg/misc/m2.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/misc/m2.py (revision 0) @@ -0,0 +1,7 @@ +# test in python shell with execfile after loading m1 +# >>> execfile("/abs/path/to/this/m2.py") + +import m1 +import distutils + +print "distutils.command: %s" % distutils.command Index: src/org/python/indexer/types/NInstanceType.java =================================================================== --- src/org/python/indexer/types/NInstanceType.java (revision 0) +++ src/org/python/indexer/types/NInstanceType.java (revision 0) @@ -0,0 +1,36 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Scope; + +/** + * Represents an instance of a class. Currently there is nothing in + * the indexer that needs to distinguish instances from classes -- both + * classes and their instances are merged into a single type. + */ +public class NInstanceType extends NType { + + private NType classType; + + public NInstanceType() { + classType = new NUnknownType(); + } + + public NInstanceType(NType c) { + this.setTable(c.getTable().copy(Scope.Type.INSTANCE)); + this.getTable().addSuper(c.getTable()); + this.getTable().setPath(c.getTable().getPath()); + classType = c; + } + + public NType getClassType() { + return classType; + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + classType.print(ctr, sb); + } +} Index: tests/java/org/python/indexer/data/testsrc.txt =================================================================== --- tests/java/org/python/indexer/data/testsrc.txt (revision 0) +++ tests/java/org/python/indexer/data/testsrc.txt (revision 0) @@ -0,0 +1,4 @@ +one +two + +three Index: src/org/python/indexer/ast/NNum.java =================================================================== --- src/org/python/indexer/ast/NNum.java (revision 0) +++ src/org/python/indexer/ast/NNum.java (revision 0) @@ -0,0 +1,39 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NNum extends NNode { + + static final long serialVersionUID = -425866329526788376L; + + public Object n; + + public NNum(int n) { + this.n = n; + } + + public NNum(Object n, int start, int end) { + super(start, end); + this.n = n; + } + + @Override + public NType resolve(Scope s) throws Exception { + return setType(Indexer.idx.builtins.BaseNum); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: src/org/python/indexer/ast/NAssert.java =================================================================== --- src/org/python/indexer/ast/NAssert.java (revision 0) +++ src/org/python/indexer/ast/NAssert.java (revision 0) @@ -0,0 +1,46 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NAssert extends NNode { + + static final long serialVersionUID = 7574732756076428388L; + + public NNode test; + public NNode msg; + + public NAssert(NNode test, NNode msg) { + this(test, msg, 0, 1); + } + + public NAssert(NNode test, NNode msg, int start, int end) { + super(start, end); + this.test = test; + this.msg = msg; + addChildren(test, msg); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(test, s); + resolveExpr(msg, s); + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(test, v); + visitNode(msg, v); + } + } +} Index: tests/java/org/python/indexer/data/test.py =================================================================== --- tests/java/org/python/indexer/data/test.py (revision 0) +++ tests/java/org/python/indexer/data/test.py (revision 0) @@ -0,0 +1,4 @@ +class Foo: + pass + +Foo() Index: tests/java/org/python/indexer/data/yinw/yinw-23.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-23.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-23.py (revision 0) @@ -0,0 +1,14 @@ +def split(s, sep=None, maxsplit=-1): + """split(s [,sep [,maxsplit]]) -> list of strings + + Return a list of the words in the string s, using sep as the + delimiter string. If maxsplit is given, splits at no more than + maxsplit places (resulting in at most maxsplit+1 words). If sep + is not specified or is None, any whitespace string is a separator. + + (split and splitfields are synonymous) + + """ + return s.split(sep, maxsplit) + +split('.') Index: src/org/python/indexer/ast/NDict.java =================================================================== --- src/org/python/indexer/ast/NDict.java (revision 0) +++ src/org/python/indexer/ast/NDict.java (revision 0) @@ -0,0 +1,53 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NDictType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +public class NDict extends NNode { + + static final long serialVersionUID = 318144953740238374L; + + public List keys; + public List values; + + public NDict(List keys, List values) { + this(keys, values, 0, 1); + } + + public NDict(List keys, List values, int start, int end) { + super(start, end); + this.keys = keys; + this.values = values; + addChildren(keys); + addChildren(values); + } + + @Override + public NType resolve(Scope s) throws Exception { + NType keyType = resolveListAsUnion(keys, s); + NType valType = resolveListAsUnion(values, s); + return setType(new NDictType(keyType, valType)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + // XXX: should visit in alternating order + visitNodeList(keys, v); + visitNodeList(values, v); + } + } +} Index: tests/java/org/python/indexer/data/classtype_builtins.py =================================================================== --- tests/java/org/python/indexer/data/classtype_builtins.py (revision 0) +++ tests/java/org/python/indexer/data/classtype_builtins.py (revision 0) @@ -0,0 +1,12 @@ +# Tests for basic class-type data model. +# See http://docs.python.org/reference/datamodel.html + +class MyClass: + """My doc string""" + def __init__(self): + print self.__doc__ + + +class MyClassNoDoc: + def __init__(self): + print self.__doc__ Index: tests/java/org/python/indexer/data/empty_file.py =================================================================== Index: tests/java/org/python/indexer/data/yinw/yinw-0.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-0.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-0.py (revision 0) @@ -0,0 +1,3774 @@ +"""Wrapper functions for Tcl/Tk. + +Tkinter provides classes which allow the display, positioning and +control of widgets. Toplevel widgets are Tk and Toplevel. Other +widgets are Frame, Label, Entry, Text, Canvas, Button, Radiobutton, +Checkbutton, Scale, Listbox, Scrollbar, OptionMenu, Spinbox +LabelFrame and PanedWindow. + +Properties of the widgets are specified with keyword arguments. +Keyword arguments have the same name as the corresponding resource +under Tk. + +Widgets are positioned with one of the geometry managers Place, Pack +or Grid. These managers can be called with methods place, pack, grid +available in every Widget. + +Actions are bound to events by resources (e.g. keyword argument +command) or with the method bind. + +Example (Hello, World): +import Tkinter +from Tkconstants import * +tk = Tkinter.Tk() +frame = Tkinter.Frame(tk, relief=RIDGE, borderwidth=2) +frame.pack(fill=BOTH,expand=1) +label = Tkinter.Label(frame, text="Hello, World") +label.pack(fill=X, expand=1) +button = Tkinter.Button(frame,text="Exit",command=tk.destroy) +button.pack(side=BOTTOM) +tk.mainloop() +""" + +__version__ = "$Revision: 70220 $" + +import sys +if sys.platform == "win32": + # Attempt to configure Tcl/Tk without requiring PATH + import FixTk +import _tkinter # If this fails your Python may not be configured for Tk +tkinter = _tkinter # b/w compat for export +TclError = _tkinter.TclError +from types import * +from Tkconstants import * + +wantobjects = 1 + +TkVersion = float(_tkinter.TK_VERSION) +TclVersion = float(_tkinter.TCL_VERSION) + +READABLE = _tkinter.READABLE +WRITABLE = _tkinter.WRITABLE +EXCEPTION = _tkinter.EXCEPTION + +# These are not always defined, e.g. not on Win32 with Tk 8.0 :-( +try: _tkinter.createfilehandler +except AttributeError: _tkinter.createfilehandler = None +try: _tkinter.deletefilehandler +except AttributeError: _tkinter.deletefilehandler = None + + +def _flatten(tuple): + """Internal function.""" + res = () + for item in tuple: + if type(item) in (TupleType, ListType): + res = res + _flatten(item) + elif item is not None: + res = res + (item,) + return res + +try: _flatten = _tkinter._flatten +except AttributeError: pass + +def _cnfmerge(cnfs): + """Internal function.""" + if type(cnfs) is DictionaryType: + return cnfs + elif type(cnfs) in (NoneType, StringType): + return cnfs + else: + cnf = {} + for c in _flatten(cnfs): + try: + cnf.update(c) + except (AttributeError, TypeError), msg: + print "_cnfmerge: fallback due to:", msg + for k, v in c.items(): + cnf[k] = v + return cnf + +try: _cnfmerge = _tkinter._cnfmerge +except AttributeError: pass + +class Event: + """Container for the properties of an event. + + Instances of this type are generated if one of the following events occurs: + + KeyPress, KeyRelease - for keyboard events + ButtonPress, ButtonRelease, Motion, Enter, Leave, MouseWheel - for mouse events + Visibility, Unmap, Map, Expose, FocusIn, FocusOut, Circulate, + Colormap, Gravity, Reparent, Property, Destroy, Activate, + Deactivate - for window events. + + If a callback function for one of these events is registered + using bind, bind_all, bind_class, or tag_bind, the callback is + called with an Event as first argument. It will have the + following attributes (in braces are the event types for which + the attribute is valid): + + serial - serial number of event + num - mouse button pressed (ButtonPress, ButtonRelease) + focus - whether the window has the focus (Enter, Leave) + height - height of the exposed window (Configure, Expose) + width - width of the exposed window (Configure, Expose) + keycode - keycode of the pressed key (KeyPress, KeyRelease) + state - state of the event as a number (ButtonPress, ButtonRelease, + Enter, KeyPress, KeyRelease, + Leave, Motion) + state - state as a string (Visibility) + time - when the event occurred + x - x-position of the mouse + y - y-position of the mouse + x_root - x-position of the mouse on the screen + (ButtonPress, ButtonRelease, KeyPress, KeyRelease, Motion) + y_root - y-position of the mouse on the screen + (ButtonPress, ButtonRelease, KeyPress, KeyRelease, Motion) + char - pressed character (KeyPress, KeyRelease) + send_event - see X/Windows documentation + keysym - keysym of the event as a string (KeyPress, KeyRelease) + keysym_num - keysym of the event as a number (KeyPress, KeyRelease) + type - type of the event as a number + widget - widget in which the event occurred + delta - delta of wheel movement (MouseWheel) + """ + pass + +_support_default_root = 1 +_default_root = None + +def NoDefaultRoot(): + """Inhibit setting of default root window. + + Call this function to inhibit that the first instance of + Tk is used for windows without an explicit parent window. + """ + global _support_default_root + _support_default_root = 0 + global _default_root + _default_root = None + del _default_root + +def _tkerror(err): + """Internal function.""" + pass + +def _exit(code='0'): + """Internal function. Calling it will throw the exception SystemExit.""" + raise SystemExit, code + +_varnum = 0 +class Variable: + """Class to define value holders for e.g. buttons. + + Subclasses StringVar, IntVar, DoubleVar, BooleanVar are specializations + that constrain the type of the value returned from get().""" + _default = "" + def __init__(self, master=None, value=None, name=None): + """Construct a variable + + MASTER can be given as master widget. + VALUE is an optional value (defaults to "") + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + global _varnum + if not master: + master = _default_root + self._master = master + self._tk = master.tk + if name: + self._name = name + else: + self._name = 'PY_VAR' + repr(_varnum) + _varnum += 1 + if value is not None: + self.set(value) + elif not self._tk.call("info", "exists", self._name): + self.set(self._default) + def __del__(self): + """Unset the variable in Tcl.""" + self._tk.globalunsetvar(self._name) + def __str__(self): + """Return the name of the variable in Tcl.""" + return self._name + def set(self, value): + """Set the variable to VALUE.""" + return self._tk.globalsetvar(self._name, value) + def get(self): + """Return value of variable.""" + return self._tk.globalgetvar(self._name) + def trace_variable(self, mode, callback): + """Define a trace callback for the variable. + + MODE is one of "r", "w", "u" for read, write, undefine. + CALLBACK must be a function which is called when + the variable is read, written or undefined. + + Return the name of the callback. + """ + cbname = self._master._register(callback) + self._tk.call("trace", "variable", self._name, mode, cbname) + return cbname + trace = trace_variable + def trace_vdelete(self, mode, cbname): + """Delete the trace callback for a variable. + + MODE is one of "r", "w", "u" for read, write, undefine. + CBNAME is the name of the callback returned from trace_variable or trace. + """ + self._tk.call("trace", "vdelete", self._name, mode, cbname) + self._master.deletecommand(cbname) + def trace_vinfo(self): + """Return all trace callback information.""" + return map(self._tk.split, self._tk.splitlist( + self._tk.call("trace", "vinfo", self._name))) + def __eq__(self, other): + """Comparison for equality (==). + + Note: if the Variable's master matters to behavior + also compare self._master == other._master + """ + return self.__class__.__name__ == other.__class__.__name__ \ + and self._name == other._name + +class StringVar(Variable): + """Value holder for strings variables.""" + _default = "" + def __init__(self, master=None, value=None, name=None): + """Construct a string variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to "") + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return value of variable as string.""" + value = self._tk.globalgetvar(self._name) + if isinstance(value, basestring): + return value + return str(value) + +class IntVar(Variable): + """Value holder for integer variables.""" + _default = 0 + def __init__(self, master=None, value=None, name=None): + """Construct an integer variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to 0) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def set(self, value): + """Set the variable to value, converting booleans to integers.""" + if isinstance(value, bool): + value = int(value) + return Variable.set(self, value) + + def get(self): + """Return the value of the variable as an integer.""" + return getint(self._tk.globalgetvar(self._name)) + +class DoubleVar(Variable): + """Value holder for float variables.""" + _default = 0.0 + def __init__(self, master=None, value=None, name=None): + """Construct a float variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to 0.0) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return the value of the variable as a float.""" + return getdouble(self._tk.globalgetvar(self._name)) + +class BooleanVar(Variable): + """Value holder for boolean variables.""" + _default = False + def __init__(self, master=None, value=None, name=None): + """Construct a boolean variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to False) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return the value of the variable as a bool.""" + return self._tk.getboolean(self._tk.globalgetvar(self._name)) + +def mainloop(n=0): + """Run the main loop of Tcl.""" + _default_root.tk.mainloop(n) + +getint = int + +getdouble = float + +def getboolean(s): + """Convert true and false to integer values 1 and 0.""" + return _default_root.tk.getboolean(s) + +# Methods defined on both toplevel and interior widgets +class Misc: + """Internal class. + + Base class which defines methods common for interior widgets.""" + + # XXX font command? + _tclCommands = None + def destroy(self): + """Internal function. + + Delete all Tcl commands created for + this widget in the Tcl interpreter.""" + if self._tclCommands is not None: + for name in self._tclCommands: + #print '- Tkinter: deleted command', name + self.tk.deletecommand(name) + self._tclCommands = None + def deletecommand(self, name): + """Internal function. + + Delete the Tcl command provided in NAME.""" + #print '- Tkinter: deleted command', name + self.tk.deletecommand(name) + try: + self._tclCommands.remove(name) + except ValueError: + pass + def tk_strictMotif(self, boolean=None): + """Set Tcl internal variable, whether the look and feel + should adhere to Motif. + + A parameter of 1 means adhere to Motif (e.g. no color + change if mouse passes over slider). + Returns the set value.""" + return self.tk.getboolean(self.tk.call( + 'set', 'tk_strictMotif', boolean)) + def tk_bisque(self): + """Change the color scheme to light brown as used in Tk 3.6 and before.""" + self.tk.call('tk_bisque') + def tk_setPalette(self, *args, **kw): + """Set a new color scheme for all widget elements. + + A single color as argument will cause that all colors of Tk + widget elements are derived from this. + Alternatively several keyword parameters and its associated + colors can be given. The following keywords are valid: + activeBackground, foreground, selectColor, + activeForeground, highlightBackground, selectBackground, + background, highlightColor, selectForeground, + disabledForeground, insertBackground, troughColor.""" + self.tk.call(('tk_setPalette',) + + _flatten(args) + _flatten(kw.items())) + def tk_menuBar(self, *args): + """Do not use. Needed in Tk 3.6 and earlier.""" + pass # obsolete since Tk 4.0 + def wait_variable(self, name='PY_VAR'): + """Wait until the variable is modified. + + A parameter of type IntVar, StringVar, DoubleVar or + BooleanVar must be given.""" + self.tk.call('tkwait', 'variable', name) + waitvar = wait_variable # XXX b/w compat + def wait_window(self, window=None): + """Wait until a WIDGET is destroyed. + + If no parameter is given self is used.""" + if window is None: + window = self + self.tk.call('tkwait', 'window', window._w) + def wait_visibility(self, window=None): + """Wait until the visibility of a WIDGET changes + (e.g. it appears). + + If no parameter is given self is used.""" + if window is None: + window = self + self.tk.call('tkwait', 'visibility', window._w) + def setvar(self, name='PY_VAR', value='1'): + """Set Tcl variable NAME to VALUE.""" + self.tk.setvar(name, value) + def getvar(self, name='PY_VAR'): + """Return value of Tcl variable NAME.""" + return self.tk.getvar(name) + getint = int + getdouble = float + def getboolean(self, s): + """Return a boolean value for Tcl boolean values true and false given as parameter.""" + return self.tk.getboolean(s) + def focus_set(self): + """Direct input focus to this widget. + + If the application currently does not have the focus + this widget will get the focus if the application gets + the focus through the window manager.""" + self.tk.call('focus', self._w) + focus = focus_set # XXX b/w compat? + def focus_force(self): + """Direct input focus to this widget even if the + application does not have the focus. Use with + caution!""" + self.tk.call('focus', '-force', self._w) + def focus_get(self): + """Return the widget which has currently the focus in the + application. + + Use focus_displayof to allow working with several + displays. Return None if application does not have + the focus.""" + name = self.tk.call('focus') + if name == 'none' or not name: return None + return self._nametowidget(name) + def focus_displayof(self): + """Return the widget which has currently the focus on the + display where this widget is located. + + Return None if the application does not have the focus.""" + name = self.tk.call('focus', '-displayof', self._w) + if name == 'none' or not name: return None + return self._nametowidget(name) + def focus_lastfor(self): + """Return the widget which would have the focus if top level + for this widget gets the focus from the window manager.""" + name = self.tk.call('focus', '-lastfor', self._w) + if name == 'none' or not name: return None + return self._nametowidget(name) + def tk_focusFollowsMouse(self): + """The widget under mouse will get automatically focus. Can not + be disabled easily.""" + self.tk.call('tk_focusFollowsMouse') + def tk_focusNext(self): + """Return the next widget in the focus order which follows + widget which has currently the focus. + + The focus order first goes to the next child, then to + the children of the child recursively and then to the + next sibling which is higher in the stacking order. A + widget is omitted if it has the takefocus resource set + to 0.""" + name = self.tk.call('tk_focusNext', self._w) + if not name: return None + return self._nametowidget(name) + def tk_focusPrev(self): + """Return previous widget in the focus order. See tk_focusNext for details.""" + name = self.tk.call('tk_focusPrev', self._w) + if not name: return None + return self._nametowidget(name) + def after(self, ms, func=None, *args): + """Call function once after given time. + + MS specifies the time in milliseconds. FUNC gives the + function which shall be called. Additional parameters + are given as parameters to the function call. Return + identifier to cancel scheduling with after_cancel.""" + if not func: + # I'd rather use time.sleep(ms*0.001) + self.tk.call('after', ms) + else: + def callit(): + try: + func(*args) + finally: + try: + self.deletecommand(name) + except TclError: + pass + name = self._register(callit) + return self.tk.call('after', ms, name) + def after_idle(self, func, *args): + """Call FUNC once if the Tcl main loop has no event to + process. + + Return an identifier to cancel the scheduling with + after_cancel.""" + return self.after('idle', func, *args) + def after_cancel(self, id): + """Cancel scheduling of function identified with ID. + + Identifier returned by after or after_idle must be + given as first parameter.""" + try: + data = self.tk.call('after', 'info', id) + # In Tk 8.3, splitlist returns: (script, type) + # In Tk 8.4, splitlist may return (script, type) or (script,) + script = self.tk.splitlist(data)[0] + self.deletecommand(script) + except TclError: + pass + self.tk.call('after', 'cancel', id) + def bell(self, displayof=0): + """Ring a display's bell.""" + self.tk.call(('bell',) + self._displayof(displayof)) + + # Clipboard handling: + def clipboard_get(self, **kw): + """Retrieve data from the clipboard on window's display. + + The window keyword defaults to the root window of the Tkinter + application. + + The type keyword specifies the form in which the data is + to be returned and should be an atom name such as STRING + or FILE_NAME. Type defaults to STRING. + + This command is equivalent to: + + selection_get(CLIPBOARD) + """ + return self.tk.call(('clipboard', 'get') + self._options(kw)) + + def clipboard_clear(self, **kw): + """Clear the data in the Tk clipboard. + + A widget specified for the optional displayof keyword + argument specifies the target display.""" + if not kw.has_key('displayof'): kw['displayof'] = self._w + self.tk.call(('clipboard', 'clear') + self._options(kw)) + def clipboard_append(self, string, **kw): + """Append STRING to the Tk clipboard. + + A widget specified at the optional displayof keyword + argument specifies the target display. The clipboard + can be retrieved with selection_get.""" + if not kw.has_key('displayof'): kw['displayof'] = self._w + self.tk.call(('clipboard', 'append') + self._options(kw) + + ('--', string)) + # XXX grab current w/o window argument + def grab_current(self): + """Return widget which has currently the grab in this application + or None.""" + name = self.tk.call('grab', 'current', self._w) + if not name: return None + return self._nametowidget(name) + def grab_release(self): + """Release grab for this widget if currently set.""" + self.tk.call('grab', 'release', self._w) + def grab_set(self): + """Set grab for this widget. + + A grab directs all events to this and descendant + widgets in the application.""" + self.tk.call('grab', 'set', self._w) + def grab_set_global(self): + """Set global grab for this widget. + + A global grab directs all events to this and + descendant widgets on the display. Use with caution - + other applications do not get events anymore.""" + self.tk.call('grab', 'set', '-global', self._w) + def grab_status(self): + """Return None, "local" or "global" if this widget has + no, a local or a global grab.""" + status = self.tk.call('grab', 'status', self._w) + if status == 'none': status = None + return status + def option_add(self, pattern, value, priority = None): + """Set a VALUE (second parameter) for an option + PATTERN (first parameter). + + An optional third parameter gives the numeric priority + (defaults to 80).""" + self.tk.call('option', 'add', pattern, value, priority) + def option_clear(self): + """Clear the option database. + + It will be reloaded if option_add is called.""" + self.tk.call('option', 'clear') + def option_get(self, name, className): + """Return the value for an option NAME for this widget + with CLASSNAME. + + Values with higher priority override lower values.""" + return self.tk.call('option', 'get', self._w, name, className) + def option_readfile(self, fileName, priority = None): + """Read file FILENAME into the option database. + + An optional second parameter gives the numeric + priority.""" + self.tk.call('option', 'readfile', fileName, priority) + def selection_clear(self, **kw): + """Clear the current X selection.""" + if not kw.has_key('displayof'): kw['displayof'] = self._w + self.tk.call(('selection', 'clear') + self._options(kw)) + def selection_get(self, **kw): + """Return the contents of the current X selection. + + A keyword parameter selection specifies the name of + the selection and defaults to PRIMARY. A keyword + parameter displayof specifies a widget on the display + to use.""" + if not kw.has_key('displayof'): kw['displayof'] = self._w + return self.tk.call(('selection', 'get') + self._options(kw)) + def selection_handle(self, command, **kw): + """Specify a function COMMAND to call if the X + selection owned by this widget is queried by another + application. + + This function must return the contents of the + selection. The function will be called with the + arguments OFFSET and LENGTH which allows the chunking + of very long selections. The following keyword + parameters can be provided: + selection - name of the selection (default PRIMARY), + type - type of the selection (e.g. STRING, FILE_NAME).""" + name = self._register(command) + self.tk.call(('selection', 'handle') + self._options(kw) + + (self._w, name)) + def selection_own(self, **kw): + """Become owner of X selection. + + A keyword parameter selection specifies the name of + the selection (default PRIMARY).""" + self.tk.call(('selection', 'own') + + self._options(kw) + (self._w,)) + def selection_own_get(self, **kw): + """Return owner of X selection. + + The following keyword parameter can + be provided: + selection - name of the selection (default PRIMARY), + type - type of the selection (e.g. STRING, FILE_NAME).""" + if not kw.has_key('displayof'): kw['displayof'] = self._w + name = self.tk.call(('selection', 'own') + self._options(kw)) + if not name: return None + return self._nametowidget(name) + def send(self, interp, cmd, *args): + """Send Tcl command CMD to different interpreter INTERP to be executed.""" + return self.tk.call(('send', interp, cmd) + args) + def lower(self, belowThis=None): + """Lower this widget in the stacking order.""" + self.tk.call('lower', self._w, belowThis) + def tkraise(self, aboveThis=None): + """Raise this widget in the stacking order.""" + self.tk.call('raise', self._w, aboveThis) + lift = tkraise + def colormodel(self, value=None): + """Useless. Not implemented in Tk.""" + return self.tk.call('tk', 'colormodel', self._w, value) + def winfo_atom(self, name, displayof=0): + """Return integer which represents atom NAME.""" + args = ('winfo', 'atom') + self._displayof(displayof) + (name,) + return getint(self.tk.call(args)) + def winfo_atomname(self, id, displayof=0): + """Return name of atom with identifier ID.""" + args = ('winfo', 'atomname') \ + + self._displayof(displayof) + (id,) + return self.tk.call(args) + def winfo_cells(self): + """Return number of cells in the colormap for this widget.""" + return getint( + self.tk.call('winfo', 'cells', self._w)) + def winfo_children(self): + """Return a list of all widgets which are children of this widget.""" + result = [] + for child in self.tk.splitlist( + self.tk.call('winfo', 'children', self._w)): + try: + # Tcl sometimes returns extra windows, e.g. for + # menus; those need to be skipped + result.append(self._nametowidget(child)) + except KeyError: + pass + return result + + def winfo_class(self): + """Return window class name of this widget.""" + return self.tk.call('winfo', 'class', self._w) + def winfo_colormapfull(self): + """Return true if at the last color request the colormap was full.""" + return self.tk.getboolean( + self.tk.call('winfo', 'colormapfull', self._w)) + def winfo_containing(self, rootX, rootY, displayof=0): + """Return the widget which is at the root coordinates ROOTX, ROOTY.""" + args = ('winfo', 'containing') \ + + self._displayof(displayof) + (rootX, rootY) + name = self.tk.call(args) + if not name: return None + return self._nametowidget(name) + def winfo_depth(self): + """Return the number of bits per pixel.""" + return getint(self.tk.call('winfo', 'depth', self._w)) + def winfo_exists(self): + """Return true if this widget exists.""" + return getint( + self.tk.call('winfo', 'exists', self._w)) + def winfo_fpixels(self, number): + """Return the number of pixels for the given distance NUMBER + (e.g. "3c") as float.""" + return getdouble(self.tk.call( + 'winfo', 'fpixels', self._w, number)) + def winfo_geometry(self): + """Return geometry string for this widget in the form "widthxheight+X+Y".""" + return self.tk.call('winfo', 'geometry', self._w) + def winfo_height(self): + """Return height of this widget.""" + return getint( + self.tk.call('winfo', 'height', self._w)) + def winfo_id(self): + """Return identifier ID for this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'id', self._w)) + def winfo_interps(self, displayof=0): + """Return the name of all Tcl interpreters for this display.""" + args = ('winfo', 'interps') + self._displayof(displayof) + return self.tk.splitlist(self.tk.call(args)) + def winfo_ismapped(self): + """Return true if this widget is mapped.""" + return getint( + self.tk.call('winfo', 'ismapped', self._w)) + def winfo_manager(self): + """Return the window mananger name for this widget.""" + return self.tk.call('winfo', 'manager', self._w) + def winfo_name(self): + """Return the name of this widget.""" + return self.tk.call('winfo', 'name', self._w) + def winfo_parent(self): + """Return the name of the parent of this widget.""" + return self.tk.call('winfo', 'parent', self._w) + def winfo_pathname(self, id, displayof=0): + """Return the pathname of the widget given by ID.""" + args = ('winfo', 'pathname') \ + + self._displayof(displayof) + (id,) + return self.tk.call(args) + def winfo_pixels(self, number): + """Rounded integer value of winfo_fpixels.""" + return getint( + self.tk.call('winfo', 'pixels', self._w, number)) + def winfo_pointerx(self): + """Return the x coordinate of the pointer on the root window.""" + return getint( + self.tk.call('winfo', 'pointerx', self._w)) + def winfo_pointerxy(self): + """Return a tuple of x and y coordinates of the pointer on the root window.""" + return self._getints( + self.tk.call('winfo', 'pointerxy', self._w)) + def winfo_pointery(self): + """Return the y coordinate of the pointer on the root window.""" + return getint( + self.tk.call('winfo', 'pointery', self._w)) + def winfo_reqheight(self): + """Return requested height of this widget.""" + return getint( + self.tk.call('winfo', 'reqheight', self._w)) + def winfo_reqwidth(self): + """Return requested width of this widget.""" + return getint( + self.tk.call('winfo', 'reqwidth', self._w)) + def winfo_rgb(self, color): + """Return tuple of decimal values for red, green, blue for + COLOR in this widget.""" + return self._getints( + self.tk.call('winfo', 'rgb', self._w, color)) + def winfo_rootx(self): + """Return x coordinate of upper left corner of this widget on the + root window.""" + return getint( + self.tk.call('winfo', 'rootx', self._w)) + def winfo_rooty(self): + """Return y coordinate of upper left corner of this widget on the + root window.""" + return getint( + self.tk.call('winfo', 'rooty', self._w)) + def winfo_screen(self): + """Return the screen name of this widget.""" + return self.tk.call('winfo', 'screen', self._w) + def winfo_screencells(self): + """Return the number of the cells in the colormap of the screen + of this widget.""" + return getint( + self.tk.call('winfo', 'screencells', self._w)) + def winfo_screendepth(self): + """Return the number of bits per pixel of the root window of the + screen of this widget.""" + return getint( + self.tk.call('winfo', 'screendepth', self._w)) + def winfo_screenheight(self): + """Return the number of pixels of the height of the screen of this widget + in pixel.""" + return getint( + self.tk.call('winfo', 'screenheight', self._w)) + def winfo_screenmmheight(self): + """Return the number of pixels of the height of the screen of + this widget in mm.""" + return getint( + self.tk.call('winfo', 'screenmmheight', self._w)) + def winfo_screenmmwidth(self): + """Return the number of pixels of the width of the screen of + this widget in mm.""" + return getint( + self.tk.call('winfo', 'screenmmwidth', self._w)) + def winfo_screenvisual(self): + """Return one of the strings directcolor, grayscale, pseudocolor, + staticcolor, staticgray, or truecolor for the default + colormodel of this screen.""" + return self.tk.call('winfo', 'screenvisual', self._w) + def winfo_screenwidth(self): + """Return the number of pixels of the width of the screen of + this widget in pixel.""" + return getint( + self.tk.call('winfo', 'screenwidth', self._w)) + def winfo_server(self): + """Return information of the X-Server of the screen of this widget in + the form "XmajorRminor vendor vendorVersion".""" + return self.tk.call('winfo', 'server', self._w) + def winfo_toplevel(self): + """Return the toplevel widget of this widget.""" + return self._nametowidget(self.tk.call( + 'winfo', 'toplevel', self._w)) + def winfo_viewable(self): + """Return true if the widget and all its higher ancestors are mapped.""" + return getint( + self.tk.call('winfo', 'viewable', self._w)) + def winfo_visual(self): + """Return one of the strings directcolor, grayscale, pseudocolor, + staticcolor, staticgray, or truecolor for the + colormodel of this widget.""" + return self.tk.call('winfo', 'visual', self._w) + def winfo_visualid(self): + """Return the X identifier for the visual for this widget.""" + return self.tk.call('winfo', 'visualid', self._w) + def winfo_visualsavailable(self, includeids=0): + """Return a list of all visuals available for the screen + of this widget. + + Each item in the list consists of a visual name (see winfo_visual), a + depth and if INCLUDEIDS=1 is given also the X identifier.""" + data = self.tk.split( + self.tk.call('winfo', 'visualsavailable', self._w, + includeids and 'includeids' or None)) + if type(data) is StringType: + data = [self.tk.split(data)] + return map(self.__winfo_parseitem, data) + def __winfo_parseitem(self, t): + """Internal function.""" + return t[:1] + tuple(map(self.__winfo_getint, t[1:])) + def __winfo_getint(self, x): + """Internal function.""" + return int(x, 0) + def winfo_vrootheight(self): + """Return the height of the virtual root window associated with this + widget in pixels. If there is no virtual root window return the + height of the screen.""" + return getint( + self.tk.call('winfo', 'vrootheight', self._w)) + def winfo_vrootwidth(self): + """Return the width of the virtual root window associated with this + widget in pixel. If there is no virtual root window return the + width of the screen.""" + return getint( + self.tk.call('winfo', 'vrootwidth', self._w)) + def winfo_vrootx(self): + """Return the x offset of the virtual root relative to the root + window of the screen of this widget.""" + return getint( + self.tk.call('winfo', 'vrootx', self._w)) + def winfo_vrooty(self): + """Return the y offset of the virtual root relative to the root + window of the screen of this widget.""" + return getint( + self.tk.call('winfo', 'vrooty', self._w)) + def winfo_width(self): + """Return the width of this widget.""" + return getint( + self.tk.call('winfo', 'width', self._w)) + def winfo_x(self): + """Return the x coordinate of the upper left corner of this widget + in the parent.""" + return getint( + self.tk.call('winfo', 'x', self._w)) + def winfo_y(self): + """Return the y coordinate of the upper left corner of this widget + in the parent.""" + return getint( + self.tk.call('winfo', 'y', self._w)) + def update(self): + """Enter event loop until all pending events have been processed by Tcl.""" + self.tk.call('update') + def update_idletasks(self): + """Enter event loop until all idle callbacks have been called. This + will update the display of windows but not process events caused by + the user.""" + self.tk.call('update', 'idletasks') + def bindtags(self, tagList=None): + """Set or get the list of bindtags for this widget. + + With no argument return the list of all bindtags associated with + this widget. With a list of strings as argument the bindtags are + set to this list. The bindtags determine in which order events are + processed (see bind).""" + if tagList is None: + return self.tk.splitlist( + self.tk.call('bindtags', self._w)) + else: + self.tk.call('bindtags', self._w, tagList) + def _bind(self, what, sequence, func, add, needcleanup=1): + """Internal function.""" + if type(func) is StringType: + self.tk.call(what + (sequence, func)) + elif func: + funcid = self._register(func, self._substitute, + needcleanup) + cmd = ('%sif {"[%s %s]" == "break"} break\n' + % + (add and '+' or '', + funcid, self._subst_format_str)) + self.tk.call(what + (sequence, cmd)) + return funcid + elif sequence: + return self.tk.call(what + (sequence,)) + else: + return self.tk.splitlist(self.tk.call(what)) + def bind(self, sequence=None, func=None, add=None): + """Bind to this widget at event SEQUENCE a call to function FUNC. + + SEQUENCE is a string of concatenated event + patterns. An event pattern is of the form + where MODIFIER is one + of Control, Mod2, M2, Shift, Mod3, M3, Lock, Mod4, M4, + Button1, B1, Mod5, M5 Button2, B2, Meta, M, Button3, + B3, Alt, Button4, B4, Double, Button5, B5 Triple, + Mod1, M1. TYPE is one of Activate, Enter, Map, + ButtonPress, Button, Expose, Motion, ButtonRelease + FocusIn, MouseWheel, Circulate, FocusOut, Property, + Colormap, Gravity Reparent, Configure, KeyPress, Key, + Unmap, Deactivate, KeyRelease Visibility, Destroy, + Leave and DETAIL is the button number for ButtonPress, + ButtonRelease and DETAIL is the Keysym for KeyPress and + KeyRelease. Examples are + for pressing Control and mouse button 1 or + for pressing A and the Alt key (KeyPress can be omitted). + An event pattern can also be a virtual event of the form + <> where AString can be arbitrary. This + event can be generated by event_generate. + If events are concatenated they must appear shortly + after each other. + + FUNC will be called if the event sequence occurs with an + instance of Event as argument. If the return value of FUNC is + "break" no further bound function is invoked. + + An additional boolean parameter ADD specifies whether FUNC will + be called additionally to the other bound function or whether + it will replace the previous function. + + Bind will return an identifier to allow deletion of the bound function with + unbind without memory leak. + + If FUNC or SEQUENCE is omitted the bound function or list + of bound events are returned.""" + + return self._bind(('bind', self._w), sequence, func, add) + def unbind(self, sequence, funcid=None): + """Unbind for this widget for event SEQUENCE the + function identified with FUNCID.""" + self.tk.call('bind', self._w, sequence, '') + if funcid: + self.deletecommand(funcid) + def bind_all(self, sequence=None, func=None, add=None): + """Bind to all widgets at an event SEQUENCE a call to function FUNC. + An additional boolean parameter ADD specifies whether FUNC will + be called additionally to the other bound function or whether + it will replace the previous function. See bind for the return value.""" + return self._bind(('bind', 'all'), sequence, func, add, 0) + def unbind_all(self, sequence): + """Unbind for all widgets for event SEQUENCE all functions.""" + self.tk.call('bind', 'all' , sequence, '') + def bind_class(self, className, sequence=None, func=None, add=None): + + """Bind to widgets with bindtag CLASSNAME at event + SEQUENCE a call of function FUNC. An additional + boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or + whether it will replace the previous function. See bind for + the return value.""" + + return self._bind(('bind', className), sequence, func, add, 0) + def unbind_class(self, className, sequence): + """Unbind for a all widgets with bindtag CLASSNAME for event SEQUENCE + all functions.""" + self.tk.call('bind', className , sequence, '') + def mainloop(self, n=0): + """Call the mainloop of Tk.""" + self.tk.mainloop(n) + def quit(self): + """Quit the Tcl interpreter. All widgets will be destroyed.""" + self.tk.quit() + def _getints(self, string): + """Internal function.""" + if string: + return tuple(map(getint, self.tk.splitlist(string))) + def _getdoubles(self, string): + """Internal function.""" + if string: + return tuple(map(getdouble, self.tk.splitlist(string))) + def _getboolean(self, string): + """Internal function.""" + if string: + return self.tk.getboolean(string) + def _displayof(self, displayof): + """Internal function.""" + if displayof: + return ('-displayof', displayof) + if displayof is None: + return ('-displayof', self._w) + return () + def _options(self, cnf, kw = None): + """Internal function.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + else: + cnf = _cnfmerge(cnf) + res = () + for k, v in cnf.items(): + if v is not None: + if k[-1] == '_': k = k[:-1] + if callable(v): + v = self._register(v) + elif isinstance(v, (tuple, list)): + nv = [] + for item in v: + if not isinstance(item, (basestring, int)): + break + elif isinstance(item, int): + nv.append('%d' % item) + else: + # format it to proper Tcl code if it contains space + nv.append(('{%s}' if ' ' in item else '%s') % item) + else: + v = ' '.join(nv) + res = res + ('-'+k, v) + return res + def nametowidget(self, name): + """Return the Tkinter instance of a widget identified by + its Tcl name NAME.""" + name = str(name).split('.') + w = self + + if not name[0]: + w = w._root() + name = name[1:] + + for n in name: + if not n: + break + w = w.children[n] + + return w + _nametowidget = nametowidget + def _register(self, func, subst=None, needcleanup=1): + """Return a newly created Tcl function. If this + function is called, the Python function FUNC will + be executed. An optional function SUBST can + be given which will be executed before FUNC.""" + f = CallWrapper(func, subst, self).__call__ + name = repr(id(f)) + try: + func = func.im_func + except AttributeError: + pass + try: + name = name + func.__name__ + except AttributeError: + pass + self.tk.createcommand(name, f) + if needcleanup: + if self._tclCommands is None: + self._tclCommands = [] + self._tclCommands.append(name) + return name + register = _register + def _root(self): + """Internal function.""" + w = self + while w.master: w = w.master + return w + _subst_format = ('%#', '%b', '%f', '%h', '%k', + '%s', '%t', '%w', '%x', '%y', + '%A', '%E', '%K', '%N', '%W', '%T', '%X', '%Y', '%D') + _subst_format_str = " ".join(_subst_format) + def _substitute(self, *args): + """Internal function.""" + if len(args) != len(self._subst_format): return args + getboolean = self.tk.getboolean + + getint = int + def getint_event(s): + """Tk changed behavior in 8.4.2, returning "??" rather more often.""" + try: + return int(s) + except ValueError: + return s + + nsign, b, f, h, k, s, t, w, x, y, A, E, K, N, W, T, X, Y, D = args + # Missing: (a, c, d, m, o, v, B, R) + e = Event() + # serial field: valid vor all events + # number of button: ButtonPress and ButtonRelease events only + # height field: Configure, ConfigureRequest, Create, + # ResizeRequest, and Expose events only + # keycode field: KeyPress and KeyRelease events only + # time field: "valid for events that contain a time field" + # width field: Configure, ConfigureRequest, Create, ResizeRequest, + # and Expose events only + # x field: "valid for events that contain a x field" + # y field: "valid for events that contain a y field" + # keysym as decimal: KeyPress and KeyRelease events only + # x_root, y_root fields: ButtonPress, ButtonRelease, KeyPress, + # KeyRelease,and Motion events + e.serial = getint(nsign) + e.num = getint_event(b) + try: e.focus = getboolean(f) + except TclError: pass + e.height = getint_event(h) + e.keycode = getint_event(k) + e.state = getint_event(s) + e.time = getint_event(t) + e.width = getint_event(w) + e.x = getint_event(x) + e.y = getint_event(y) + e.char = A + try: e.send_event = getboolean(E) + except TclError: pass + e.keysym = K + e.keysym_num = getint_event(N) + e.type = T + try: + e.widget = self._nametowidget(W) + except KeyError: + e.widget = W + e.x_root = getint_event(X) + e.y_root = getint_event(Y) + try: + e.delta = getint(D) + except ValueError: + e.delta = 0 + return (e,) + def _report_exception(self): + """Internal function.""" + import sys + exc, val, tb = sys.exc_type, sys.exc_value, sys.exc_traceback + root = self._root() + root.report_callback_exception(exc, val, tb) + def _configure(self, cmd, cnf, kw): + """Internal function.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + elif cnf: + cnf = _cnfmerge(cnf) + if cnf is None: + cnf = {} + for x in self.tk.split( + self.tk.call(_flatten((self._w, cmd)))): + cnf[x[0][1:]] = (x[0][1:],) + x[1:] + return cnf + if type(cnf) is StringType: + x = self.tk.split( + self.tk.call(_flatten((self._w, cmd, '-'+cnf)))) + return (x[0][1:],) + x[1:] + self.tk.call(_flatten((self._w, cmd)) + self._options(cnf)) + # These used to be defined in Widget: + def configure(self, cnf=None, **kw): + """Configure resources of a widget. + + The values for resources are specified as keyword + arguments. To get an overview about + the allowed keyword arguments call the method keys. + """ + return self._configure('configure', cnf, kw) + config = configure + def cget(self, key): + """Return the resource value for a KEY given as string.""" + return self.tk.call(self._w, 'cget', '-' + key) + __getitem__ = cget + def __setitem__(self, key, value): + self.configure({key: value}) + def __contains__(self, key): + raise TypeError("Tkinter objects don't support 'in' tests.") + def keys(self): + """Return a list of all resource names of this widget.""" + return map(lambda x: x[0][1:], + self.tk.split(self.tk.call(self._w, 'configure'))) + def __str__(self): + """Return the window path name of this widget.""" + return self._w + # Pack methods that apply to the master + _noarg_ = ['_noarg_'] + def pack_propagate(self, flag=_noarg_): + """Set or get the status for propagation of geometry information. + + A boolean argument specifies whether the geometry information + of the slaves will determine the size of this widget. If no argument + is given the current setting will be returned. + """ + if flag is Misc._noarg_: + return self._getboolean(self.tk.call( + 'pack', 'propagate', self._w)) + else: + self.tk.call('pack', 'propagate', self._w, flag) + propagate = pack_propagate + def pack_slaves(self): + """Return a list of all slaves of this widget + in its packing order.""" + return map(self._nametowidget, + self.tk.splitlist( + self.tk.call('pack', 'slaves', self._w))) + slaves = pack_slaves + # Place method that applies to the master + def place_slaves(self): + """Return a list of all slaves of this widget + in its packing order.""" + return map(self._nametowidget, + self.tk.splitlist( + self.tk.call( + 'place', 'slaves', self._w))) + # Grid methods that apply to the master + def grid_bbox(self, column=None, row=None, col2=None, row2=None): + """Return a tuple of integer coordinates for the bounding + box of this widget controlled by the geometry manager grid. + + If COLUMN, ROW is given the bounding box applies from + the cell with row and column 0 to the specified + cell. If COL2 and ROW2 are given the bounding box + starts at that cell. + + The returned integers specify the offset of the upper left + corner in the master widget and the width and height. + """ + args = ('grid', 'bbox', self._w) + if column is not None and row is not None: + args = args + (column, row) + if col2 is not None and row2 is not None: + args = args + (col2, row2) + return self._getints(self.tk.call(*args)) or None + + bbox = grid_bbox + def _grid_configure(self, command, index, cnf, kw): + """Internal function.""" + if type(cnf) is StringType and not kw: + if cnf[-1:] == '_': + cnf = cnf[:-1] + if cnf[:1] != '-': + cnf = '-'+cnf + options = (cnf,) + else: + options = self._options(cnf, kw) + if not options: + res = self.tk.call('grid', + command, self._w, index) + words = self.tk.splitlist(res) + dict = {} + for i in range(0, len(words), 2): + key = words[i][1:] + value = words[i+1] + if not value: + value = None + elif '.' in value: + value = getdouble(value) + else: + value = getint(value) + dict[key] = value + return dict + res = self.tk.call( + ('grid', command, self._w, index) + + options) + if len(options) == 1: + if not res: return None + # In Tk 7.5, -width can be a float + if '.' in res: return getdouble(res) + return getint(res) + def grid_columnconfigure(self, index, cnf={}, **kw): + """Configure column INDEX of a grid. + + Valid resources are minsize (minimum size of the column), + weight (how much does additional space propagate to this column) + and pad (how much space to let additionally).""" + return self._grid_configure('columnconfigure', index, cnf, kw) + columnconfigure = grid_columnconfigure + def grid_location(self, x, y): + """Return a tuple of column and row which identify the cell + at which the pixel at position X and Y inside the master + widget is located.""" + return self._getints( + self.tk.call( + 'grid', 'location', self._w, x, y)) or None + def grid_propagate(self, flag=_noarg_): + """Set or get the status for propagation of geometry information. + + A boolean argument specifies whether the geometry information + of the slaves will determine the size of this widget. If no argument + is given, the current setting will be returned. + """ + if flag is Misc._noarg_: + return self._getboolean(self.tk.call( + 'grid', 'propagate', self._w)) + else: + self.tk.call('grid', 'propagate', self._w, flag) + def grid_rowconfigure(self, index, cnf={}, **kw): + """Configure row INDEX of a grid. + + Valid resources are minsize (minimum size of the row), + weight (how much does additional space propagate to this row) + and pad (how much space to let additionally).""" + return self._grid_configure('rowconfigure', index, cnf, kw) + rowconfigure = grid_rowconfigure + def grid_size(self): + """Return a tuple of the number of column and rows in the grid.""" + return self._getints( + self.tk.call('grid', 'size', self._w)) or None + size = grid_size + def grid_slaves(self, row=None, column=None): + """Return a list of all slaves of this widget + in its packing order.""" + args = () + if row is not None: + args = args + ('-row', row) + if column is not None: + args = args + ('-column', column) + return map(self._nametowidget, + self.tk.splitlist(self.tk.call( + ('grid', 'slaves', self._w) + args))) + + # Support for the "event" command, new in Tk 4.2. + # By Case Roole. + + def event_add(self, virtual, *sequences): + """Bind a virtual event VIRTUAL (of the form <>) + to an event SEQUENCE such that the virtual event is triggered + whenever SEQUENCE occurs.""" + args = ('event', 'add', virtual) + sequences + self.tk.call(args) + + def event_delete(self, virtual, *sequences): + """Unbind a virtual event VIRTUAL from SEQUENCE.""" + args = ('event', 'delete', virtual) + sequences + self.tk.call(args) + + def event_generate(self, sequence, **kw): + """Generate an event SEQUENCE. Additional + keyword arguments specify parameter of the event + (e.g. x, y, rootx, rooty).""" + args = ('event', 'generate', self._w, sequence) + for k, v in kw.items(): + args = args + ('-%s' % k, str(v)) + self.tk.call(args) + + def event_info(self, virtual=None): + """Return a list of all virtual events or the information + about the SEQUENCE bound to the virtual event VIRTUAL.""" + return self.tk.splitlist( + self.tk.call('event', 'info', virtual)) + + # Image related commands + + def image_names(self): + """Return a list of all existing image names.""" + return self.tk.call('image', 'names') + + def image_types(self): + """Return a list of all available image types (e.g. phote bitmap).""" + return self.tk.call('image', 'types') + + +class CallWrapper: + """Internal class. Stores function to call when some user + defined Tcl function is called e.g. after an event occurred.""" + def __init__(self, func, subst, widget): + """Store FUNC, SUBST and WIDGET as members.""" + self.func = func + self.subst = subst + self.widget = widget + def __call__(self, *args): + """Apply first function SUBST to arguments, than FUNC.""" + try: + if self.subst: + args = self.subst(*args) + return self.func(*args) + except SystemExit, msg: + raise SystemExit, msg + except: + self.widget._report_exception() + + +class Wm: + """Provides functions for the communication with the window manager.""" + + def wm_aspect(self, + minNumer=None, minDenom=None, + maxNumer=None, maxDenom=None): + """Instruct the window manager to set the aspect ratio (width/height) + of this widget to be between MINNUMER/MINDENOM and MAXNUMER/MAXDENOM. Return a tuple + of the actual values if no argument is given.""" + return self._getints( + self.tk.call('wm', 'aspect', self._w, + minNumer, minDenom, + maxNumer, maxDenom)) + aspect = wm_aspect + + def wm_attributes(self, *args): + """This subcommand returns or sets platform specific attributes + + The first form returns a list of the platform specific flags and + their values. The second form returns the value for the specific + option. The third form sets one or more of the values. The values + are as follows: + + On Windows, -disabled gets or sets whether the window is in a + disabled state. -toolwindow gets or sets the style of the window + to toolwindow (as defined in the MSDN). -topmost gets or sets + whether this is a topmost window (displays above all other + windows). + + On Macintosh, XXXXX + + On Unix, there are currently no special attribute values. + """ + args = ('wm', 'attributes', self._w) + args + return self.tk.call(args) + attributes=wm_attributes + + def wm_client(self, name=None): + """Store NAME in WM_CLIENT_MACHINE property of this widget. Return + current value.""" + return self.tk.call('wm', 'client', self._w, name) + client = wm_client + def wm_colormapwindows(self, *wlist): + """Store list of window names (WLIST) into WM_COLORMAPWINDOWS property + of this widget. This list contains windows whose colormaps differ from their + parents. Return current list of widgets if WLIST is empty.""" + if len(wlist) > 1: + wlist = (wlist,) # Tk needs a list of windows here + args = ('wm', 'colormapwindows', self._w) + wlist + return map(self._nametowidget, self.tk.call(args)) + colormapwindows = wm_colormapwindows + def wm_command(self, value=None): + """Store VALUE in WM_COMMAND property. It is the command + which shall be used to invoke the application. Return current + command if VALUE is None.""" + return self.tk.call('wm', 'command', self._w, value) + command = wm_command + def wm_deiconify(self): + """Deiconify this widget. If it was never mapped it will not be mapped. + On Windows it will raise this widget and give it the focus.""" + return self.tk.call('wm', 'deiconify', self._w) + deiconify = wm_deiconify + def wm_focusmodel(self, model=None): + """Set focus model to MODEL. "active" means that this widget will claim + the focus itself, "passive" means that the window manager shall give + the focus. Return current focus model if MODEL is None.""" + return self.tk.call('wm', 'focusmodel', self._w, model) + focusmodel = wm_focusmodel + def wm_frame(self): + """Return identifier for decorative frame of this widget if present.""" + return self.tk.call('wm', 'frame', self._w) + frame = wm_frame + def wm_geometry(self, newGeometry=None): + """Set geometry to NEWGEOMETRY of the form =widthxheight+x+y. Return + current value if None is given.""" + return self.tk.call('wm', 'geometry', self._w, newGeometry) + geometry = wm_geometry + def wm_grid(self, + baseWidth=None, baseHeight=None, + widthInc=None, heightInc=None): + """Instruct the window manager that this widget shall only be + resized on grid boundaries. WIDTHINC and HEIGHTINC are the width and + height of a grid unit in pixels. BASEWIDTH and BASEHEIGHT are the + number of grid units requested in Tk_GeometryRequest.""" + return self._getints(self.tk.call( + 'wm', 'grid', self._w, + baseWidth, baseHeight, widthInc, heightInc)) + grid = wm_grid + def wm_group(self, pathName=None): + """Set the group leader widgets for related widgets to PATHNAME. Return + the group leader of this widget if None is given.""" + return self.tk.call('wm', 'group', self._w, pathName) + group = wm_group + def wm_iconbitmap(self, bitmap=None, default=None): + """Set bitmap for the iconified widget to BITMAP. Return + the bitmap if None is given. + + Under Windows, the DEFAULT parameter can be used to set the icon + for the widget and any descendents that don't have an icon set + explicitly. DEFAULT can be the relative path to a .ico file + (example: root.iconbitmap(default='myicon.ico') ). See Tk + documentation for more information.""" + if default: + return self.tk.call('wm', 'iconbitmap', self._w, '-default', default) + else: + return self.tk.call('wm', 'iconbitmap', self._w, bitmap) + iconbitmap = wm_iconbitmap + def wm_iconify(self): + """Display widget as icon.""" + return self.tk.call('wm', 'iconify', self._w) + iconify = wm_iconify + def wm_iconmask(self, bitmap=None): + """Set mask for the icon bitmap of this widget. Return the + mask if None is given.""" + return self.tk.call('wm', 'iconmask', self._w, bitmap) + iconmask = wm_iconmask + def wm_iconname(self, newName=None): + """Set the name of the icon for this widget. Return the name if + None is given.""" + return self.tk.call('wm', 'iconname', self._w, newName) + iconname = wm_iconname + def wm_iconposition(self, x=None, y=None): + """Set the position of the icon of this widget to X and Y. Return + a tuple of the current values of X and X if None is given.""" + return self._getints(self.tk.call( + 'wm', 'iconposition', self._w, x, y)) + iconposition = wm_iconposition + def wm_iconwindow(self, pathName=None): + """Set widget PATHNAME to be displayed instead of icon. Return the current + value if None is given.""" + return self.tk.call('wm', 'iconwindow', self._w, pathName) + iconwindow = wm_iconwindow + def wm_maxsize(self, width=None, height=None): + """Set max WIDTH and HEIGHT for this widget. If the window is gridded + the values are given in grid units. Return the current values if None + is given.""" + return self._getints(self.tk.call( + 'wm', 'maxsize', self._w, width, height)) + maxsize = wm_maxsize + def wm_minsize(self, width=None, height=None): + """Set min WIDTH and HEIGHT for this widget. If the window is gridded + the values are given in grid units. Return the current values if None + is given.""" + return self._getints(self.tk.call( + 'wm', 'minsize', self._w, width, height)) + minsize = wm_minsize + def wm_overrideredirect(self, boolean=None): + """Instruct the window manager to ignore this widget + if BOOLEAN is given with 1. Return the current value if None + is given.""" + return self._getboolean(self.tk.call( + 'wm', 'overrideredirect', self._w, boolean)) + overrideredirect = wm_overrideredirect + def wm_positionfrom(self, who=None): + """Instruct the window manager that the position of this widget shall + be defined by the user if WHO is "user", and by its own policy if WHO is + "program".""" + return self.tk.call('wm', 'positionfrom', self._w, who) + positionfrom = wm_positionfrom + def wm_protocol(self, name=None, func=None): + """Bind function FUNC to command NAME for this widget. + Return the function bound to NAME if None is given. NAME could be + e.g. "WM_SAVE_YOURSELF" or "WM_DELETE_WINDOW".""" + if hasattr(func, '__call__'): + command = self._register(func) + else: + command = func + return self.tk.call( + 'wm', 'protocol', self._w, name, command) + protocol = wm_protocol + def wm_resizable(self, width=None, height=None): + """Instruct the window manager whether this width can be resized + in WIDTH or HEIGHT. Both values are boolean values.""" + return self.tk.call('wm', 'resizable', self._w, width, height) + resizable = wm_resizable + def wm_sizefrom(self, who=None): + """Instruct the window manager that the size of this widget shall + be defined by the user if WHO is "user", and by its own policy if WHO is + "program".""" + return self.tk.call('wm', 'sizefrom', self._w, who) + sizefrom = wm_sizefrom + def wm_state(self, newstate=None): + """Query or set the state of this widget as one of normal, icon, + iconic (see wm_iconwindow), withdrawn, or zoomed (Windows only).""" + return self.tk.call('wm', 'state', self._w, newstate) + state = wm_state + def wm_title(self, string=None): + """Set the title of this widget.""" + return self.tk.call('wm', 'title', self._w, string) + title = wm_title + def wm_transient(self, master=None): + """Instruct the window manager that this widget is transient + with regard to widget MASTER.""" + return self.tk.call('wm', 'transient', self._w, master) + transient = wm_transient + def wm_withdraw(self): + """Withdraw this widget from the screen such that it is unmapped + and forgotten by the window manager. Re-draw it with wm_deiconify.""" + return self.tk.call('wm', 'withdraw', self._w) + withdraw = wm_withdraw + + +class Tk(Misc, Wm): + """Toplevel widget of Tk which represents mostly the main window + of an appliation. It has an associated Tcl interpreter.""" + _w = '.' + def __init__(self, screenName=None, baseName=None, className='Tk', + useTk=1, sync=0, use=None): + """Return a new Toplevel widget on screen SCREENNAME. A new Tcl interpreter will + be created. BASENAME will be used for the identification of the profile file (see + readprofile). + It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME + is the name of the widget class.""" + self.master = None + self.children = {} + self._tkloaded = 0 + # to avoid recursions in the getattr code in case of failure, we + # ensure that self.tk is always _something_. + self.tk = None + if baseName is None: + import sys, os + baseName = os.path.basename(sys.argv[0]) + baseName, ext = os.path.splitext(baseName) + if ext not in ('.py', '.pyc', '.pyo'): + baseName = baseName + ext + interactive = 0 + self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use) + if useTk: + self._loadtk() + self.readprofile(baseName, className) + def loadtk(self): + if not self._tkloaded: + self.tk.loadtk() + self._loadtk() + def _loadtk(self): + self._tkloaded = 1 + global _default_root + # Version sanity checks + tk_version = self.tk.getvar('tk_version') + if tk_version != _tkinter.TK_VERSION: + raise RuntimeError, \ + "tk.h version (%s) doesn't match libtk.a version (%s)" \ + % (_tkinter.TK_VERSION, tk_version) + # Under unknown circumstances, tcl_version gets coerced to float + tcl_version = str(self.tk.getvar('tcl_version')) + if tcl_version != _tkinter.TCL_VERSION: + raise RuntimeError, \ + "tcl.h version (%s) doesn't match libtcl.a version (%s)" \ + % (_tkinter.TCL_VERSION, tcl_version) + if TkVersion < 4.0: + raise RuntimeError, \ + "Tk 4.0 or higher is required; found Tk %s" \ + % str(TkVersion) + # Create and register the tkerror and exit commands + # We need to inline parts of _register here, _ register + # would register differently-named commands. + if self._tclCommands is None: + self._tclCommands = [] + self.tk.createcommand('tkerror', _tkerror) + self.tk.createcommand('exit', _exit) + self._tclCommands.append('tkerror') + self._tclCommands.append('exit') + if _support_default_root and not _default_root: + _default_root = self + self.protocol("WM_DELETE_WINDOW", self.destroy) + def destroy(self): + """Destroy this and all descendants widgets. This will + end the application of this Tcl interpreter.""" + for c in self.children.values(): c.destroy() + self.tk.call('destroy', self._w) + Misc.destroy(self) + global _default_root + if _support_default_root and _default_root is self: + _default_root = None + def readprofile(self, baseName, className): + """Internal function. It reads BASENAME.tcl and CLASSNAME.tcl into + the Tcl Interpreter and calls execfile on BASENAME.py and CLASSNAME.py if + such a file exists in the home directory.""" + import os + if os.environ.has_key('HOME'): home = os.environ['HOME'] + else: home = os.curdir + class_tcl = os.path.join(home, '.%s.tcl' % className) + class_py = os.path.join(home, '.%s.py' % className) + base_tcl = os.path.join(home, '.%s.tcl' % baseName) + base_py = os.path.join(home, '.%s.py' % baseName) + dir = {'self': self} + exec 'from Tkinter import *' in dir + if os.path.isfile(class_tcl): + self.tk.call('source', class_tcl) + if os.path.isfile(class_py): + execfile(class_py, dir) + if os.path.isfile(base_tcl): + self.tk.call('source', base_tcl) + if os.path.isfile(base_py): + execfile(base_py, dir) + def report_callback_exception(self, exc, val, tb): + """Internal function. It reports exception on sys.stderr.""" + import traceback, sys + sys.stderr.write("Exception in Tkinter callback\n") + sys.last_type = exc + sys.last_value = val + sys.last_traceback = tb + traceback.print_exception(exc, val, tb) + def __getattr__(self, attr): + "Delegate attribute access to the interpreter object" + return getattr(self.tk, attr) + +# Ideally, the classes Pack, Place and Grid disappear, the +# pack/place/grid methods are defined on the Widget class, and +# everybody uses w.pack_whatever(...) instead of Pack.whatever(w, +# ...), with pack(), place() and grid() being short for +# pack_configure(), place_configure() and grid_columnconfigure(), and +# forget() being short for pack_forget(). As a practical matter, I'm +# afraid that there is too much code out there that may be using the +# Pack, Place or Grid class, so I leave them intact -- but only as +# backwards compatibility features. Also note that those methods that +# take a master as argument (e.g. pack_propagate) have been moved to +# the Misc class (which now incorporates all methods common between +# toplevel and interior widgets). Again, for compatibility, these are +# copied into the Pack, Place or Grid class. + + +def Tcl(screenName=None, baseName=None, className='Tk', useTk=0): + return Tk(screenName, baseName, className, useTk) + +class Pack: + """Geometry manager Pack. + + Base class to use the methods pack_* in every widget.""" + def pack_configure(self, cnf={}, **kw): + """Pack a widget in the parent widget. Use as options: + after=widget - pack it after you have packed widget + anchor=NSEW (or subset) - position widget according to + given direction + before=widget - pack it before you will pack widget + expand=bool - expand widget if parent size grows + fill=NONE or X or Y or BOTH - fill widget if widget grows + in=master - use master to contain this widget + in_=master - see 'in' option description + ipadx=amount - add internal padding in x direction + ipady=amount - add internal padding in y direction + padx=amount - add padding in x direction + pady=amount - add padding in y direction + side=TOP or BOTTOM or LEFT or RIGHT - where to add this widget. + """ + self.tk.call( + ('pack', 'configure', self._w) + + self._options(cnf, kw)) + pack = configure = config = pack_configure + def pack_forget(self): + """Unmap this widget and do not use it for the packing order.""" + self.tk.call('pack', 'forget', self._w) + forget = pack_forget + def pack_info(self): + """Return information about the packing options + for this widget.""" + words = self.tk.splitlist( + self.tk.call('pack', 'info', self._w)) + dict = {} + for i in range(0, len(words), 2): + key = words[i][1:] + value = words[i+1] + if value[:1] == '.': + value = self._nametowidget(value) + dict[key] = value + return dict + info = pack_info + propagate = pack_propagate = Misc.pack_propagate + slaves = pack_slaves = Misc.pack_slaves + +class Place: + """Geometry manager Place. + + Base class to use the methods place_* in every widget.""" + def place_configure(self, cnf={}, **kw): + """Place a widget in the parent widget. Use as options: + in=master - master relative to which the widget is placed + in_=master - see 'in' option description + x=amount - locate anchor of this widget at position x of master + y=amount - locate anchor of this widget at position y of master + relx=amount - locate anchor of this widget between 0.0 and 1.0 + relative to width of master (1.0 is right edge) + rely=amount - locate anchor of this widget between 0.0 and 1.0 + relative to height of master (1.0 is bottom edge) + anchor=NSEW (or subset) - position anchor according to given direction + width=amount - width of this widget in pixel + height=amount - height of this widget in pixel + relwidth=amount - width of this widget between 0.0 and 1.0 + relative to width of master (1.0 is the same width + as the master) + relheight=amount - height of this widget between 0.0 and 1.0 + relative to height of master (1.0 is the same + height as the master) + bordermode="inside" or "outside" - whether to take border width of + master widget into account + """ + self.tk.call( + ('place', 'configure', self._w) + + self._options(cnf, kw)) + place = configure = config = place_configure + def place_forget(self): + """Unmap this widget.""" + self.tk.call('place', 'forget', self._w) + forget = place_forget + def place_info(self): + """Return information about the placing options + for this widget.""" + words = self.tk.splitlist( + self.tk.call('place', 'info', self._w)) + dict = {} + for i in range(0, len(words), 2): + key = words[i][1:] + value = words[i+1] + if value[:1] == '.': + value = self._nametowidget(value) + dict[key] = value + return dict + info = place_info + slaves = place_slaves = Misc.place_slaves + +class Grid: + """Geometry manager Grid. + + Base class to use the methods grid_* in every widget.""" + # Thanks to Masazumi Yoshikawa (yosikawa@isi.edu) + def grid_configure(self, cnf={}, **kw): + """Position a widget in the parent widget in a grid. Use as options: + column=number - use cell identified with given column (starting with 0) + columnspan=number - this widget will span several columns + in=master - use master to contain this widget + in_=master - see 'in' option description + ipadx=amount - add internal padding in x direction + ipady=amount - add internal padding in y direction + padx=amount - add padding in x direction + pady=amount - add padding in y direction + row=number - use cell identified with given row (starting with 0) + rowspan=number - this widget will span several rows + sticky=NSEW - if cell is larger on which sides will this + widget stick to the cell boundary + """ + self.tk.call( + ('grid', 'configure', self._w) + + self._options(cnf, kw)) + grid = configure = config = grid_configure + bbox = grid_bbox = Misc.grid_bbox + columnconfigure = grid_columnconfigure = Misc.grid_columnconfigure + def grid_forget(self): + """Unmap this widget.""" + self.tk.call('grid', 'forget', self._w) + forget = grid_forget + def grid_remove(self): + """Unmap this widget but remember the grid options.""" + self.tk.call('grid', 'remove', self._w) + def grid_info(self): + """Return information about the options + for positioning this widget in a grid.""" + words = self.tk.splitlist( + self.tk.call('grid', 'info', self._w)) + dict = {} + for i in range(0, len(words), 2): + key = words[i][1:] + value = words[i+1] + if value[:1] == '.': + value = self._nametowidget(value) + dict[key] = value + return dict + info = grid_info + location = grid_location = Misc.grid_location + propagate = grid_propagate = Misc.grid_propagate + rowconfigure = grid_rowconfigure = Misc.grid_rowconfigure + size = grid_size = Misc.grid_size + slaves = grid_slaves = Misc.grid_slaves + +class BaseWidget(Misc): + """Internal class.""" + def _setup(self, master, cnf): + """Internal function. Sets up information about children.""" + if _support_default_root: + global _default_root + if not master: + if not _default_root: + _default_root = Tk() + master = _default_root + self.master = master + self.tk = master.tk + name = None + if cnf.has_key('name'): + name = cnf['name'] + del cnf['name'] + if not name: + name = repr(id(self)) + self._name = name + if master._w=='.': + self._w = '.' + name + else: + self._w = master._w + '.' + name + self.children = {} + if self.master.children.has_key(self._name): + self.master.children[self._name].destroy() + self.master.children[self._name] = self + def __init__(self, master, widgetName, cnf={}, kw={}, extra=()): + """Construct a widget with the parent widget MASTER, a name WIDGETNAME + and appropriate options.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + self.widgetName = widgetName + BaseWidget._setup(self, master, cnf) + if self._tclCommands is None: + self._tclCommands = [] + classes = [] + for k in cnf.keys(): + if type(k) is ClassType: + classes.append((k, cnf[k])) + del cnf[k] + self.tk.call( + (widgetName, self._w) + extra + self._options(cnf)) + for k, v in classes: + k.configure(self, v) + def destroy(self): + """Destroy this and all descendants widgets.""" + for c in self.children.values(): c.destroy() + self.tk.call('destroy', self._w) + if self.master.children.has_key(self._name): + del self.master.children[self._name] + Misc.destroy(self) + def _do(self, name, args=()): + # XXX Obsolete -- better use self.tk.call directly! + return self.tk.call((self._w, name) + args) + +class Widget(BaseWidget, Pack, Place, Grid): + """Internal class. + + Base class for a widget which can be positioned with the geometry managers + Pack, Place or Grid.""" + pass + +class Toplevel(BaseWidget, Wm): + """Toplevel widget, e.g. for dialogs.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a toplevel widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, class, + colormap, container, cursor, height, highlightbackground, + highlightcolor, highlightthickness, menu, relief, screen, takefocus, + use, visual, width.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + extra = () + for wmkey in ['screen', 'class_', 'class', 'visual', + 'colormap']: + if cnf.has_key(wmkey): + val = cnf[wmkey] + # TBD: a hack needed because some keys + # are not valid as keyword arguments + if wmkey[-1] == '_': opt = '-'+wmkey[:-1] + else: opt = '-'+wmkey + extra = extra + (opt, val) + del cnf[wmkey] + BaseWidget.__init__(self, master, 'toplevel', cnf, {}, extra) + root = self._root() + self.iconname(root.iconname()) + self.title(root.title()) + self.protocol("WM_DELETE_WINDOW", self.destroy) + +class Button(Widget): + """Button widget.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a button widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, activeforeground, anchor, + background, bitmap, borderwidth, cursor, + disabledforeground, font, foreground + highlightbackground, highlightcolor, + highlightthickness, image, justify, + padx, pady, relief, repeatdelay, + repeatinterval, takefocus, text, + textvariable, underline, wraplength + + WIDGET-SPECIFIC OPTIONS + + command, compound, default, height, + overrelief, state, width + """ + Widget.__init__(self, master, 'button', cnf, kw) + + def tkButtonEnter(self, *dummy): + self.tk.call('tkButtonEnter', self._w) + + def tkButtonLeave(self, *dummy): + self.tk.call('tkButtonLeave', self._w) + + def tkButtonDown(self, *dummy): + self.tk.call('tkButtonDown', self._w) + + def tkButtonUp(self, *dummy): + self.tk.call('tkButtonUp', self._w) + + def tkButtonInvoke(self, *dummy): + self.tk.call('tkButtonInvoke', self._w) + + def flash(self): + """Flash the button. + + This is accomplished by redisplaying + the button several times, alternating between active and + normal colors. At the end of the flash the button is left + in the same normal/active state as when the command was + invoked. This command is ignored if the button's state is + disabled. + """ + self.tk.call(self._w, 'flash') + + def invoke(self): + """Invoke the command associated with the button. + + The return value is the return value from the command, + or an empty string if there is no command associated with + the button. This command is ignored if the button's state + is disabled. + """ + return self.tk.call(self._w, 'invoke') + +# Indices: +# XXX I don't like these -- take them away +def AtEnd(): + return 'end' +def AtInsert(*args): + s = 'insert' + for a in args: + if a: s = s + (' ' + a) + return s +def AtSelFirst(): + return 'sel.first' +def AtSelLast(): + return 'sel.last' +def At(x, y=None): + if y is None: + return '@%r' % (x,) + else: + return '@%r,%r' % (x, y) + +class Canvas(Widget): + """Canvas widget to display graphical elements like lines or text.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a canvas widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, closeenough, + confine, cursor, height, highlightbackground, highlightcolor, + highlightthickness, insertbackground, insertborderwidth, + insertofftime, insertontime, insertwidth, offset, relief, + scrollregion, selectbackground, selectborderwidth, selectforeground, + state, takefocus, width, xscrollcommand, xscrollincrement, + yscrollcommand, yscrollincrement.""" + Widget.__init__(self, master, 'canvas', cnf, kw) + def addtag(self, *args): + """Internal function.""" + self.tk.call((self._w, 'addtag') + args) + def addtag_above(self, newtag, tagOrId): + """Add tag NEWTAG to all items above TAGORID.""" + self.addtag(newtag, 'above', tagOrId) + def addtag_all(self, newtag): + """Add tag NEWTAG to all items.""" + self.addtag(newtag, 'all') + def addtag_below(self, newtag, tagOrId): + """Add tag NEWTAG to all items below TAGORID.""" + self.addtag(newtag, 'below', tagOrId) + def addtag_closest(self, newtag, x, y, halo=None, start=None): + """Add tag NEWTAG to item which is closest to pixel at X, Y. + If several match take the top-most. + All items closer than HALO are considered overlapping (all are + closests). If START is specified the next below this tag is taken.""" + self.addtag(newtag, 'closest', x, y, halo, start) + def addtag_enclosed(self, newtag, x1, y1, x2, y2): + """Add tag NEWTAG to all items in the rectangle defined + by X1,Y1,X2,Y2.""" + self.addtag(newtag, 'enclosed', x1, y1, x2, y2) + def addtag_overlapping(self, newtag, x1, y1, x2, y2): + """Add tag NEWTAG to all items which overlap the rectangle + defined by X1,Y1,X2,Y2.""" + self.addtag(newtag, 'overlapping', x1, y1, x2, y2) + def addtag_withtag(self, newtag, tagOrId): + """Add tag NEWTAG to all items with TAGORID.""" + self.addtag(newtag, 'withtag', tagOrId) + def bbox(self, *args): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle + which encloses all items with tags specified as arguments.""" + return self._getints( + self.tk.call((self._w, 'bbox') + args)) or None + def tag_unbind(self, tagOrId, sequence, funcid=None): + """Unbind for all items with TAGORID for event SEQUENCE the + function identified with FUNCID.""" + self.tk.call(self._w, 'bind', tagOrId, sequence, '') + if funcid: + self.deletecommand(funcid) + def tag_bind(self, tagOrId, sequence=None, func=None, add=None): + """Bind to all items with TAGORID at event SEQUENCE a call to function FUNC. + + An additional boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or whether it will + replace the previous function. See bind for the return value.""" + return self._bind((self._w, 'bind', tagOrId), + sequence, func, add) + def canvasx(self, screenx, gridspacing=None): + """Return the canvas x coordinate of pixel position SCREENX rounded + to nearest multiple of GRIDSPACING units.""" + return getdouble(self.tk.call( + self._w, 'canvasx', screenx, gridspacing)) + def canvasy(self, screeny, gridspacing=None): + """Return the canvas y coordinate of pixel position SCREENY rounded + to nearest multiple of GRIDSPACING units.""" + return getdouble(self.tk.call( + self._w, 'canvasy', screeny, gridspacing)) + def coords(self, *args): + """Return a list of coordinates for the item given in ARGS.""" + # XXX Should use _flatten on args + return map(getdouble, + self.tk.splitlist( + self.tk.call((self._w, 'coords') + args))) + def _create(self, itemType, args, kw): # Args: (val, val, ..., cnf={}) + """Internal function.""" + args = _flatten(args) + cnf = args[-1] + if type(cnf) in (DictionaryType, TupleType): + args = args[:-1] + else: + cnf = {} + return getint(self.tk.call( + self._w, 'create', itemType, + *(args + self._options(cnf, kw)))) + def create_arc(self, *args, **kw): + """Create arc shaped region with coordinates x1,y1,x2,y2.""" + return self._create('arc', args, kw) + def create_bitmap(self, *args, **kw): + """Create bitmap with coordinates x1,y1.""" + return self._create('bitmap', args, kw) + def create_image(self, *args, **kw): + """Create image item with coordinates x1,y1.""" + return self._create('image', args, kw) + def create_line(self, *args, **kw): + """Create line with coordinates x1,y1,...,xn,yn.""" + return self._create('line', args, kw) + def create_oval(self, *args, **kw): + """Create oval with coordinates x1,y1,x2,y2.""" + return self._create('oval', args, kw) + def create_polygon(self, *args, **kw): + """Create polygon with coordinates x1,y1,...,xn,yn.""" + return self._create('polygon', args, kw) + def create_rectangle(self, *args, **kw): + """Create rectangle with coordinates x1,y1,x2,y2.""" + return self._create('rectangle', args, kw) + def create_text(self, *args, **kw): + """Create text with coordinates x1,y1.""" + return self._create('text', args, kw) + def create_window(self, *args, **kw): + """Create window with coordinates x1,y1,x2,y2.""" + return self._create('window', args, kw) + def dchars(self, *args): + """Delete characters of text items identified by tag or id in ARGS (possibly + several times) from FIRST to LAST character (including).""" + self.tk.call((self._w, 'dchars') + args) + def delete(self, *args): + """Delete items identified by all tag or ids contained in ARGS.""" + self.tk.call((self._w, 'delete') + args) + def dtag(self, *args): + """Delete tag or id given as last arguments in ARGS from items + identified by first argument in ARGS.""" + self.tk.call((self._w, 'dtag') + args) + def find(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'find') + args)) or () + def find_above(self, tagOrId): + """Return items above TAGORID.""" + return self.find('above', tagOrId) + def find_all(self): + """Return all items.""" + return self.find('all') + def find_below(self, tagOrId): + """Return all items below TAGORID.""" + return self.find('below', tagOrId) + def find_closest(self, x, y, halo=None, start=None): + """Return item which is closest to pixel at X, Y. + If several match take the top-most. + All items closer than HALO are considered overlapping (all are + closests). If START is specified the next below this tag is taken.""" + return self.find('closest', x, y, halo, start) + def find_enclosed(self, x1, y1, x2, y2): + """Return all items in rectangle defined + by X1,Y1,X2,Y2.""" + return self.find('enclosed', x1, y1, x2, y2) + def find_overlapping(self, x1, y1, x2, y2): + """Return all items which overlap the rectangle + defined by X1,Y1,X2,Y2.""" + return self.find('overlapping', x1, y1, x2, y2) + def find_withtag(self, tagOrId): + """Return all items with TAGORID.""" + return self.find('withtag', tagOrId) + def focus(self, *args): + """Set focus to the first item specified in ARGS.""" + return self.tk.call((self._w, 'focus') + args) + def gettags(self, *args): + """Return tags associated with the first item specified in ARGS.""" + return self.tk.splitlist( + self.tk.call((self._w, 'gettags') + args)) + def icursor(self, *args): + """Set cursor at position POS in the item identified by TAGORID. + In ARGS TAGORID must be first.""" + self.tk.call((self._w, 'icursor') + args) + def index(self, *args): + """Return position of cursor as integer in item specified in ARGS.""" + return getint(self.tk.call((self._w, 'index') + args)) + def insert(self, *args): + """Insert TEXT in item TAGORID at position POS. ARGS must + be TAGORID POS TEXT.""" + self.tk.call((self._w, 'insert') + args) + def itemcget(self, tagOrId, option): + """Return the resource value for an OPTION for item TAGORID.""" + return self.tk.call( + (self._w, 'itemcget') + (tagOrId, '-'+option)) + def itemconfigure(self, tagOrId, cnf=None, **kw): + """Configure resources of an item TAGORID. + + The values for resources are specified as keyword + arguments. To get an overview about + the allowed keyword arguments call the method without arguments. + """ + return self._configure(('itemconfigure', tagOrId), cnf, kw) + itemconfig = itemconfigure + # lower, tkraise/lift hide Misc.lower, Misc.tkraise/lift, + # so the preferred name for them is tag_lower, tag_raise + # (similar to tag_bind, and similar to the Text widget); + # unfortunately can't delete the old ones yet (maybe in 1.6) + def tag_lower(self, *args): + """Lower an item TAGORID given in ARGS + (optional below another item).""" + self.tk.call((self._w, 'lower') + args) + lower = tag_lower + def move(self, *args): + """Move an item TAGORID given in ARGS.""" + self.tk.call((self._w, 'move') + args) + def postscript(self, cnf={}, **kw): + """Print the contents of the canvas to a postscript + file. Valid options: colormap, colormode, file, fontmap, + height, pageanchor, pageheight, pagewidth, pagex, pagey, + rotate, witdh, x, y.""" + return self.tk.call((self._w, 'postscript') + + self._options(cnf, kw)) + def tag_raise(self, *args): + """Raise an item TAGORID given in ARGS + (optional above another item).""" + self.tk.call((self._w, 'raise') + args) + lift = tkraise = tag_raise + def scale(self, *args): + """Scale item TAGORID with XORIGIN, YORIGIN, XSCALE, YSCALE.""" + self.tk.call((self._w, 'scale') + args) + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + def scan_dragto(self, x, y, gain=10): + """Adjust the view of the canvas to GAIN times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y, gain) + def select_adjust(self, tagOrId, index): + """Adjust the end of the selection near the cursor of an item TAGORID to index.""" + self.tk.call(self._w, 'select', 'adjust', tagOrId, index) + def select_clear(self): + """Clear the selection if it is in this widget.""" + self.tk.call(self._w, 'select', 'clear') + def select_from(self, tagOrId, index): + """Set the fixed end of a selection in item TAGORID to INDEX.""" + self.tk.call(self._w, 'select', 'from', tagOrId, index) + def select_item(self): + """Return the item which has the selection.""" + return self.tk.call(self._w, 'select', 'item') or None + def select_to(self, tagOrId, index): + """Set the variable end of a selection in item TAGORID to INDEX.""" + self.tk.call(self._w, 'select', 'to', tagOrId, index) + def type(self, tagOrId): + """Return the type of the item TAGORID.""" + return self.tk.call(self._w, 'type', tagOrId) or None + def xview(self, *args): + """Query and change horizontal position of the view.""" + if not args: + return self._getdoubles(self.tk.call(self._w, 'xview')) + self.tk.call((self._w, 'xview') + args) + def xview_moveto(self, fraction): + """Adjusts the view in the window so that FRACTION of the + total width of the canvas is off-screen to the left.""" + self.tk.call(self._w, 'xview', 'moveto', fraction) + def xview_scroll(self, number, what): + """Shift the x-view according to NUMBER which is measured in "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'xview', 'scroll', number, what) + def yview(self, *args): + """Query and change vertical position of the view.""" + if not args: + return self._getdoubles(self.tk.call(self._w, 'yview')) + self.tk.call((self._w, 'yview') + args) + def yview_moveto(self, fraction): + """Adjusts the view in the window so that FRACTION of the + total height of the canvas is off-screen to the top.""" + self.tk.call(self._w, 'yview', 'moveto', fraction) + def yview_scroll(self, number, what): + """Shift the y-view according to NUMBER which is measured in "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'yview', 'scroll', number, what) + +class Checkbutton(Widget): + """Checkbutton widget which is either in on- or off-state.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a checkbutton widget with the parent MASTER. + + Valid resource names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, command, cursor, + disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, image, + indicatoron, justify, offvalue, onvalue, padx, pady, relief, + selectcolor, selectimage, state, takefocus, text, textvariable, + underline, variable, width, wraplength.""" + Widget.__init__(self, master, 'checkbutton', cnf, kw) + def deselect(self): + """Put the button in off-state.""" + self.tk.call(self._w, 'deselect') + def flash(self): + """Flash the button.""" + self.tk.call(self._w, 'flash') + def invoke(self): + """Toggle the button and invoke a command if given as resource.""" + return self.tk.call(self._w, 'invoke') + def select(self): + """Put the button in on-state.""" + self.tk.call(self._w, 'select') + def toggle(self): + """Toggle the button.""" + self.tk.call(self._w, 'toggle') + +class Entry(Widget): + """Entry widget which allows to display simple text.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct an entry widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, cursor, + exportselection, fg, font, foreground, highlightbackground, + highlightcolor, highlightthickness, insertbackground, + insertborderwidth, insertofftime, insertontime, insertwidth, + invalidcommand, invcmd, justify, relief, selectbackground, + selectborderwidth, selectforeground, show, state, takefocus, + textvariable, validate, validatecommand, vcmd, width, + xscrollcommand.""" + Widget.__init__(self, master, 'entry', cnf, kw) + def delete(self, first, last=None): + """Delete text from FIRST to LAST (not included).""" + self.tk.call(self._w, 'delete', first, last) + def get(self): + """Return the text.""" + return self.tk.call(self._w, 'get') + def icursor(self, index): + """Insert cursor at INDEX.""" + self.tk.call(self._w, 'icursor', index) + def index(self, index): + """Return position of cursor.""" + return getint(self.tk.call( + self._w, 'index', index)) + def insert(self, index, string): + """Insert STRING at INDEX.""" + self.tk.call(self._w, 'insert', index, string) + def scan_mark(self, x): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x) + def scan_dragto(self, x): + """Adjust the view of the canvas to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x) + def selection_adjust(self, index): + """Adjust the end of the selection near the cursor to INDEX.""" + self.tk.call(self._w, 'selection', 'adjust', index) + select_adjust = selection_adjust + def selection_clear(self): + """Clear the selection if it is in this widget.""" + self.tk.call(self._w, 'selection', 'clear') + select_clear = selection_clear + def selection_from(self, index): + """Set the fixed end of a selection to INDEX.""" + self.tk.call(self._w, 'selection', 'from', index) + select_from = selection_from + def selection_present(self): + """Return whether the widget has the selection.""" + return self.tk.getboolean( + self.tk.call(self._w, 'selection', 'present')) + select_present = selection_present + def selection_range(self, start, end): + """Set the selection from START to END (not included).""" + self.tk.call(self._w, 'selection', 'range', start, end) + select_range = selection_range + def selection_to(self, index): + """Set the variable end of a selection to INDEX.""" + self.tk.call(self._w, 'selection', 'to', index) + select_to = selection_to + def xview(self, index): + """Query and change horizontal position of the view.""" + self.tk.call(self._w, 'xview', index) + def xview_moveto(self, fraction): + """Adjust the view in the window so that FRACTION of the + total width of the entry is off-screen to the left.""" + self.tk.call(self._w, 'xview', 'moveto', fraction) + def xview_scroll(self, number, what): + """Shift the x-view according to NUMBER which is measured in "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'xview', 'scroll', number, what) + +class Frame(Widget): + """Frame widget which may contain other widgets and can have a 3D border.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a frame widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, class, + colormap, container, cursor, height, highlightbackground, + highlightcolor, highlightthickness, relief, takefocus, visual, width.""" + cnf = _cnfmerge((cnf, kw)) + extra = () + if cnf.has_key('class_'): + extra = ('-class', cnf['class_']) + del cnf['class_'] + elif cnf.has_key('class'): + extra = ('-class', cnf['class']) + del cnf['class'] + Widget.__init__(self, master, 'frame', cnf, {}, extra) + +class Label(Widget): + """Label widget which can display text and bitmaps.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a label widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, activeforeground, anchor, + background, bitmap, borderwidth, cursor, + disabledforeground, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, image, justify, + padx, pady, relief, takefocus, text, + textvariable, underline, wraplength + + WIDGET-SPECIFIC OPTIONS + + height, state, width + + """ + Widget.__init__(self, master, 'label', cnf, kw) + +class Listbox(Widget): + """Listbox widget which can display a list of strings.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a listbox widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, cursor, + exportselection, fg, font, foreground, height, highlightbackground, + highlightcolor, highlightthickness, relief, selectbackground, + selectborderwidth, selectforeground, selectmode, setgrid, takefocus, + width, xscrollcommand, yscrollcommand, listvariable.""" + Widget.__init__(self, master, 'listbox', cnf, kw) + def activate(self, index): + """Activate item identified by INDEX.""" + self.tk.call(self._w, 'activate', index) + def bbox(self, *args): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle + which encloses the item identified by index in ARGS.""" + return self._getints( + self.tk.call((self._w, 'bbox') + args)) or None + def curselection(self): + """Return list of indices of currently selected item.""" + # XXX Ought to apply self._getints()... + return self.tk.splitlist(self.tk.call( + self._w, 'curselection')) + def delete(self, first, last=None): + """Delete items from FIRST to LAST (not included).""" + self.tk.call(self._w, 'delete', first, last) + def get(self, first, last=None): + """Get list of items from FIRST to LAST (not included).""" + if last: + return self.tk.splitlist(self.tk.call( + self._w, 'get', first, last)) + else: + return self.tk.call(self._w, 'get', first) + def index(self, index): + """Return index of item identified with INDEX.""" + i = self.tk.call(self._w, 'index', index) + if i == 'none': return None + return getint(i) + def insert(self, index, *elements): + """Insert ELEMENTS at INDEX.""" + self.tk.call((self._w, 'insert', index) + elements) + def nearest(self, y): + """Get index of item which is nearest to y coordinate Y.""" + return getint(self.tk.call( + self._w, 'nearest', y)) + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + def scan_dragto(self, x, y): + """Adjust the view of the listbox to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y) + def see(self, index): + """Scroll such that INDEX is visible.""" + self.tk.call(self._w, 'see', index) + def selection_anchor(self, index): + """Set the fixed end oft the selection to INDEX.""" + self.tk.call(self._w, 'selection', 'anchor', index) + select_anchor = selection_anchor + def selection_clear(self, first, last=None): + """Clear the selection from FIRST to LAST (not included).""" + self.tk.call(self._w, + 'selection', 'clear', first, last) + select_clear = selection_clear + def selection_includes(self, index): + """Return 1 if INDEX is part of the selection.""" + return self.tk.getboolean(self.tk.call( + self._w, 'selection', 'includes', index)) + select_includes = selection_includes + def selection_set(self, first, last=None): + """Set the selection from FIRST to LAST (not included) without + changing the currently selected elements.""" + self.tk.call(self._w, 'selection', 'set', first, last) + select_set = selection_set + def size(self): + """Return the number of elements in the listbox.""" + return getint(self.tk.call(self._w, 'size')) + def xview(self, *what): + """Query and change horizontal position of the view.""" + if not what: + return self._getdoubles(self.tk.call(self._w, 'xview')) + self.tk.call((self._w, 'xview') + what) + def xview_moveto(self, fraction): + """Adjust the view in the window so that FRACTION of the + total width of the entry is off-screen to the left.""" + self.tk.call(self._w, 'xview', 'moveto', fraction) + def xview_scroll(self, number, what): + """Shift the x-view according to NUMBER which is measured in "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'xview', 'scroll', number, what) + def yview(self, *what): + """Query and change vertical position of the view.""" + if not what: + return self._getdoubles(self.tk.call(self._w, 'yview')) + self.tk.call((self._w, 'yview') + what) + def yview_moveto(self, fraction): + """Adjust the view in the window so that FRACTION of the + total width of the entry is off-screen to the top.""" + self.tk.call(self._w, 'yview', 'moveto', fraction) + def yview_scroll(self, number, what): + """Shift the y-view according to NUMBER which is measured in "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'yview', 'scroll', number, what) + def itemcget(self, index, option): + """Return the resource value for an ITEM and an OPTION.""" + return self.tk.call( + (self._w, 'itemcget') + (index, '-'+option)) + def itemconfigure(self, index, cnf=None, **kw): + """Configure resources of an ITEM. + + The values for resources are specified as keyword arguments. + To get an overview about the allowed keyword arguments + call the method without arguments. + Valid resource names: background, bg, foreground, fg, + selectbackground, selectforeground.""" + return self._configure(('itemconfigure', index), cnf, kw) + itemconfig = itemconfigure + +class Menu(Widget): + """Menu widget which allows to display menu bars, pull-down menus and pop-up menus.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct menu widget with the parent MASTER. + + Valid resource names: activebackground, activeborderwidth, + activeforeground, background, bd, bg, borderwidth, cursor, + disabledforeground, fg, font, foreground, postcommand, relief, + selectcolor, takefocus, tearoff, tearoffcommand, title, type.""" + Widget.__init__(self, master, 'menu', cnf, kw) + def tk_bindForTraversal(self): + pass # obsolete since Tk 4.0 + def tk_mbPost(self): + self.tk.call('tk_mbPost', self._w) + def tk_mbUnpost(self): + self.tk.call('tk_mbUnpost') + def tk_traverseToMenu(self, char): + self.tk.call('tk_traverseToMenu', self._w, char) + def tk_traverseWithinMenu(self, char): + self.tk.call('tk_traverseWithinMenu', self._w, char) + def tk_getMenuButtons(self): + return self.tk.call('tk_getMenuButtons', self._w) + def tk_nextMenu(self, count): + self.tk.call('tk_nextMenu', count) + def tk_nextMenuEntry(self, count): + self.tk.call('tk_nextMenuEntry', count) + def tk_invokeMenu(self): + self.tk.call('tk_invokeMenu', self._w) + def tk_firstMenu(self): + self.tk.call('tk_firstMenu', self._w) + def tk_mbButtonDown(self): + self.tk.call('tk_mbButtonDown', self._w) + def tk_popup(self, x, y, entry=""): + """Post the menu at position X,Y with entry ENTRY.""" + self.tk.call('tk_popup', self._w, x, y, entry) + def activate(self, index): + """Activate entry at INDEX.""" + self.tk.call(self._w, 'activate', index) + def add(self, itemType, cnf={}, **kw): + """Internal function.""" + self.tk.call((self._w, 'add', itemType) + + self._options(cnf, kw)) + def add_cascade(self, cnf={}, **kw): + """Add hierarchical menu item.""" + self.add('cascade', cnf or kw) + def add_checkbutton(self, cnf={}, **kw): + """Add checkbutton menu item.""" + self.add('checkbutton', cnf or kw) + def add_command(self, cnf={}, **kw): + """Add command menu item.""" + self.add('command', cnf or kw) + def add_radiobutton(self, cnf={}, **kw): + """Addd radio menu item.""" + self.add('radiobutton', cnf or kw) + def add_separator(self, cnf={}, **kw): + """Add separator.""" + self.add('separator', cnf or kw) + def insert(self, index, itemType, cnf={}, **kw): + """Internal function.""" + self.tk.call((self._w, 'insert', index, itemType) + + self._options(cnf, kw)) + def insert_cascade(self, index, cnf={}, **kw): + """Add hierarchical menu item at INDEX.""" + self.insert(index, 'cascade', cnf or kw) + def insert_checkbutton(self, index, cnf={}, **kw): + """Add checkbutton menu item at INDEX.""" + self.insert(index, 'checkbutton', cnf or kw) + def insert_command(self, index, cnf={}, **kw): + """Add command menu item at INDEX.""" + self.insert(index, 'command', cnf or kw) + def insert_radiobutton(self, index, cnf={}, **kw): + """Addd radio menu item at INDEX.""" + self.insert(index, 'radiobutton', cnf or kw) + def insert_separator(self, index, cnf={}, **kw): + """Add separator at INDEX.""" + self.insert(index, 'separator', cnf or kw) + def delete(self, index1, index2=None): + """Delete menu items between INDEX1 and INDEX2 (included).""" + if index2 is None: + index2 = index1 + + num_index1, num_index2 = self.index(index1), self.index(index2) + if (num_index1 is None) or (num_index2 is None): + num_index1, num_index2 = 0, -1 + + for i in range(num_index1, num_index2 + 1): + if 'command' in self.entryconfig(i): + c = str(self.entrycget(i, 'command')) + if c: + self.deletecommand(c) + self.tk.call(self._w, 'delete', index1, index2) + def entrycget(self, index, option): + """Return the resource value of an menu item for OPTION at INDEX.""" + return self.tk.call(self._w, 'entrycget', index, '-' + option) + def entryconfigure(self, index, cnf=None, **kw): + """Configure a menu item at INDEX.""" + return self._configure(('entryconfigure', index), cnf, kw) + entryconfig = entryconfigure + def index(self, index): + """Return the index of a menu item identified by INDEX.""" + i = self.tk.call(self._w, 'index', index) + if i == 'none': return None + return getint(i) + def invoke(self, index): + """Invoke a menu item identified by INDEX and execute + the associated command.""" + return self.tk.call(self._w, 'invoke', index) + def post(self, x, y): + """Display a menu at position X,Y.""" + self.tk.call(self._w, 'post', x, y) + def type(self, index): + """Return the type of the menu item at INDEX.""" + return self.tk.call(self._w, 'type', index) + def unpost(self): + """Unmap a menu.""" + self.tk.call(self._w, 'unpost') + def yposition(self, index): + """Return the y-position of the topmost pixel of the menu item at INDEX.""" + return getint(self.tk.call( + self._w, 'yposition', index)) + +class Menubutton(Widget): + """Menubutton widget, obsolete since Tk8.0.""" + def __init__(self, master=None, cnf={}, **kw): + Widget.__init__(self, master, 'menubutton', cnf, kw) + +class Message(Widget): + """Message widget to display multiline text. Obsolete since Label does it too.""" + def __init__(self, master=None, cnf={}, **kw): + Widget.__init__(self, master, 'message', cnf, kw) + +class Radiobutton(Widget): + """Radiobutton widget which shows only one of several buttons in on-state.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a radiobutton widget with the parent MASTER. + + Valid resource names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, command, cursor, + disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, image, + indicatoron, justify, padx, pady, relief, selectcolor, selectimage, + state, takefocus, text, textvariable, underline, value, variable, + width, wraplength.""" + Widget.__init__(self, master, 'radiobutton', cnf, kw) + def deselect(self): + """Put the button in off-state.""" + + self.tk.call(self._w, 'deselect') + def flash(self): + """Flash the button.""" + self.tk.call(self._w, 'flash') + def invoke(self): + """Toggle the button and invoke a command if given as resource.""" + return self.tk.call(self._w, 'invoke') + def select(self): + """Put the button in on-state.""" + self.tk.call(self._w, 'select') + +class Scale(Widget): + """Scale widget which can display a numerical scale.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a scale widget with the parent MASTER. + + Valid resource names: activebackground, background, bigincrement, bd, + bg, borderwidth, command, cursor, digits, fg, font, foreground, from, + highlightbackground, highlightcolor, highlightthickness, label, + length, orient, relief, repeatdelay, repeatinterval, resolution, + showvalue, sliderlength, sliderrelief, state, takefocus, + tickinterval, to, troughcolor, variable, width.""" + Widget.__init__(self, master, 'scale', cnf, kw) + def get(self): + """Get the current value as integer or float.""" + value = self.tk.call(self._w, 'get') + try: + return getint(value) + except ValueError: + return getdouble(value) + def set(self, value): + """Set the value to VALUE.""" + self.tk.call(self._w, 'set', value) + def coords(self, value=None): + """Return a tuple (X,Y) of the point along the centerline of the + trough that corresponds to VALUE or the current value if None is + given.""" + + return self._getints(self.tk.call(self._w, 'coords', value)) + def identify(self, x, y): + """Return where the point X,Y lies. Valid return values are "slider", + "though1" and "though2".""" + return self.tk.call(self._w, 'identify', x, y) + +class Scrollbar(Widget): + """Scrollbar widget which displays a slider at a certain position.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a scrollbar widget with the parent MASTER. + + Valid resource names: activebackground, activerelief, + background, bd, bg, borderwidth, command, cursor, + elementborderwidth, highlightbackground, + highlightcolor, highlightthickness, jump, orient, + relief, repeatdelay, repeatinterval, takefocus, + troughcolor, width.""" + Widget.__init__(self, master, 'scrollbar', cnf, kw) + def activate(self, index): + """Display the element at INDEX with activebackground and activerelief. + INDEX can be "arrow1","slider" or "arrow2".""" + self.tk.call(self._w, 'activate', index) + def delta(self, deltax, deltay): + """Return the fractional change of the scrollbar setting if it + would be moved by DELTAX or DELTAY pixels.""" + return getdouble( + self.tk.call(self._w, 'delta', deltax, deltay)) + def fraction(self, x, y): + """Return the fractional value which corresponds to a slider + position of X,Y.""" + return getdouble(self.tk.call(self._w, 'fraction', x, y)) + def identify(self, x, y): + """Return the element under position X,Y as one of + "arrow1","slider","arrow2" or "".""" + return self.tk.call(self._w, 'identify', x, y) + def get(self): + """Return the current fractional values (upper and lower end) + of the slider position.""" + return self._getdoubles(self.tk.call(self._w, 'get')) + def set(self, *args): + """Set the fractional values of the slider position (upper and + lower ends as value between 0 and 1).""" + self.tk.call((self._w, 'set') + args) + + + +class Text(Widget): + """Text widget which can display text in various forms.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a text widget with the parent MASTER. + + STANDARD OPTIONS + + background, borderwidth, cursor, + exportselection, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, insertbackground, + insertborderwidth, insertofftime, + insertontime, insertwidth, padx, pady, + relief, selectbackground, + selectborderwidth, selectforeground, + setgrid, takefocus, + xscrollcommand, yscrollcommand, + + WIDGET-SPECIFIC OPTIONS + + autoseparators, height, maxundo, + spacing1, spacing2, spacing3, + state, tabs, undo, width, wrap, + + """ + Widget.__init__(self, master, 'text', cnf, kw) + def bbox(self, *args): + """Return a tuple of (x,y,width,height) which gives the bounding + box of the visible part of the character at the index in ARGS.""" + return self._getints( + self.tk.call((self._w, 'bbox') + args)) or None + def tk_textSelectTo(self, index): + self.tk.call('tk_textSelectTo', self._w, index) + def tk_textBackspace(self): + self.tk.call('tk_textBackspace', self._w) + def tk_textIndexCloser(self, a, b, c): + self.tk.call('tk_textIndexCloser', self._w, a, b, c) + def tk_textResetAnchor(self, index): + self.tk.call('tk_textResetAnchor', self._w, index) + def compare(self, index1, op, index2): + """Return whether between index INDEX1 and index INDEX2 the + relation OP is satisfied. OP is one of <, <=, ==, >=, >, or !=.""" + return self.tk.getboolean(self.tk.call( + self._w, 'compare', index1, op, index2)) + def debug(self, boolean=None): + """Turn on the internal consistency checks of the B-Tree inside the text + widget according to BOOLEAN.""" + return self.tk.getboolean(self.tk.call( + self._w, 'debug', boolean)) + def delete(self, index1, index2=None): + """Delete the characters between INDEX1 and INDEX2 (not included).""" + self.tk.call(self._w, 'delete', index1, index2) + def dlineinfo(self, index): + """Return tuple (x,y,width,height,baseline) giving the bounding box + and baseline position of the visible part of the line containing + the character at INDEX.""" + return self._getints(self.tk.call(self._w, 'dlineinfo', index)) + def dump(self, index1, index2=None, command=None, **kw): + """Return the contents of the widget between index1 and index2. + + The type of contents returned in filtered based on the keyword + parameters; if 'all', 'image', 'mark', 'tag', 'text', or 'window' are + given and true, then the corresponding items are returned. The result + is a list of triples of the form (key, value, index). If none of the + keywords are true then 'all' is used by default. + + If the 'command' argument is given, it is called once for each element + of the list of triples, with the values of each triple serving as the + arguments to the function. In this case the list is not returned.""" + args = [] + func_name = None + result = None + if not command: + # Never call the dump command without the -command flag, since the + # output could involve Tcl quoting and would be a pain to parse + # right. Instead just set the command to build a list of triples + # as if we had done the parsing. + result = [] + def append_triple(key, value, index, result=result): + result.append((key, value, index)) + command = append_triple + try: + if not isinstance(command, str): + func_name = command = self._register(command) + args += ["-command", command] + for key in kw: + if kw[key]: args.append("-" + key) + args.append(index1) + if index2: + args.append(index2) + self.tk.call(self._w, "dump", *args) + return result + finally: + if func_name: + self.deletecommand(func_name) + + ## new in tk8.4 + def edit(self, *args): + """Internal method + + This method controls the undo mechanism and + the modified flag. The exact behavior of the + command depends on the option argument that + follows the edit argument. The following forms + of the command are currently supported: + + edit_modified, edit_redo, edit_reset, edit_separator + and edit_undo + + """ + return self.tk.call(self._w, 'edit', *args) + + def edit_modified(self, arg=None): + """Get or Set the modified flag + + If arg is not specified, returns the modified + flag of the widget. The insert, delete, edit undo and + edit redo commands or the user can set or clear the + modified flag. If boolean is specified, sets the + modified flag of the widget to arg. + """ + return self.edit("modified", arg) + + def edit_redo(self): + """Redo the last undone edit + + When the undo option is true, reapplies the last + undone edits provided no other edits were done since + then. Generates an error when the redo stack is empty. + Does nothing when the undo option is false. + """ + return self.edit("redo") + + def edit_reset(self): + """Clears the undo and redo stacks + """ + return self.edit("reset") + + def edit_separator(self): + """Inserts a separator (boundary) on the undo stack. + + Does nothing when the undo option is false + """ + return self.edit("separator") + + def edit_undo(self): + """Undoes the last edit action + + If the undo option is true. An edit action is defined + as all the insert and delete commands that are recorded + on the undo stack in between two separators. Generates + an error when the undo stack is empty. Does nothing + when the undo option is false + """ + return self.edit("undo") + + def get(self, index1, index2=None): + """Return the text from INDEX1 to INDEX2 (not included).""" + return self.tk.call(self._w, 'get', index1, index2) + # (Image commands are new in 8.0) + def image_cget(self, index, option): + """Return the value of OPTION of an embedded image at INDEX.""" + if option[:1] != "-": + option = "-" + option + if option[-1:] == "_": + option = option[:-1] + return self.tk.call(self._w, "image", "cget", index, option) + def image_configure(self, index, cnf=None, **kw): + """Configure an embedded image at INDEX.""" + return self._configure(('image', 'configure', index), cnf, kw) + def image_create(self, index, cnf={}, **kw): + """Create an embedded image at INDEX.""" + return self.tk.call( + self._w, "image", "create", index, + *self._options(cnf, kw)) + def image_names(self): + """Return all names of embedded images in this widget.""" + return self.tk.call(self._w, "image", "names") + def index(self, index): + """Return the index in the form line.char for INDEX.""" + return str(self.tk.call(self._w, 'index', index)) + def insert(self, index, chars, *args): + """Insert CHARS before the characters at INDEX. An additional + tag can be given in ARGS. Additional CHARS and tags can follow in ARGS.""" + self.tk.call((self._w, 'insert', index, chars) + args) + def mark_gravity(self, markName, direction=None): + """Change the gravity of a mark MARKNAME to DIRECTION (LEFT or RIGHT). + Return the current value if None is given for DIRECTION.""" + return self.tk.call( + (self._w, 'mark', 'gravity', markName, direction)) + def mark_names(self): + """Return all mark names.""" + return self.tk.splitlist(self.tk.call( + self._w, 'mark', 'names')) + def mark_set(self, markName, index): + """Set mark MARKNAME before the character at INDEX.""" + self.tk.call(self._w, 'mark', 'set', markName, index) + def mark_unset(self, *markNames): + """Delete all marks in MARKNAMES.""" + self.tk.call((self._w, 'mark', 'unset') + markNames) + def mark_next(self, index): + """Return the name of the next mark after INDEX.""" + return self.tk.call(self._w, 'mark', 'next', index) or None + def mark_previous(self, index): + """Return the name of the previous mark before INDEX.""" + return self.tk.call(self._w, 'mark', 'previous', index) or None + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + def scan_dragto(self, x, y): + """Adjust the view of the text to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y) + def search(self, pattern, index, stopindex=None, + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, elide=None): + """Search PATTERN beginning from INDEX until STOPINDEX. + Return the index of the first character of a match or an + empty string.""" + args = [self._w, 'search'] + if forwards: args.append('-forwards') + if backwards: args.append('-backwards') + if exact: args.append('-exact') + if regexp: args.append('-regexp') + if nocase: args.append('-nocase') + if elide: args.append('-elide') + if count: args.append('-count'); args.append(count) + if pattern and pattern[0] == '-': args.append('--') + args.append(pattern) + args.append(index) + if stopindex: args.append(stopindex) + return str(self.tk.call(tuple(args))) + def see(self, index): + """Scroll such that the character at INDEX is visible.""" + self.tk.call(self._w, 'see', index) + def tag_add(self, tagName, index1, *args): + """Add tag TAGNAME to all characters between INDEX1 and index2 in ARGS. + Additional pairs of indices may follow in ARGS.""" + self.tk.call( + (self._w, 'tag', 'add', tagName, index1) + args) + def tag_unbind(self, tagName, sequence, funcid=None): + """Unbind for all characters with TAGNAME for event SEQUENCE the + function identified with FUNCID.""" + self.tk.call(self._w, 'tag', 'bind', tagName, sequence, '') + if funcid: + self.deletecommand(funcid) + def tag_bind(self, tagName, sequence, func, add=None): + """Bind to all characters with TAGNAME at event SEQUENCE a call to function FUNC. + + An additional boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or whether it will + replace the previous function. See bind for the return value.""" + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + def tag_cget(self, tagName, option): + """Return the value of OPTION for tag TAGNAME.""" + if option[:1] != '-': + option = '-' + option + if option[-1:] == '_': + option = option[:-1] + return self.tk.call(self._w, 'tag', 'cget', tagName, option) + def tag_configure(self, tagName, cnf=None, **kw): + """Configure a tag TAGNAME.""" + return self._configure(('tag', 'configure', tagName), cnf, kw) + tag_config = tag_configure + def tag_delete(self, *tagNames): + """Delete all tags in TAGNAMES.""" + self.tk.call((self._w, 'tag', 'delete') + tagNames) + def tag_lower(self, tagName, belowThis=None): + """Change the priority of tag TAGNAME such that it is lower + than the priority of BELOWTHIS.""" + self.tk.call(self._w, 'tag', 'lower', tagName, belowThis) + def tag_names(self, index=None): + """Return a list of all tag names.""" + return self.tk.splitlist( + self.tk.call(self._w, 'tag', 'names', index)) + def tag_nextrange(self, tagName, index1, index2=None): + """Return a list of start and end index for the first sequence of + characters between INDEX1 and INDEX2 which all have tag TAGNAME. + The text is searched forward from INDEX1.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'nextrange', tagName, index1, index2)) + def tag_prevrange(self, tagName, index1, index2=None): + """Return a list of start and end index for the first sequence of + characters between INDEX1 and INDEX2 which all have tag TAGNAME. + The text is searched backwards from INDEX1.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'prevrange', tagName, index1, index2)) + def tag_raise(self, tagName, aboveThis=None): + """Change the priority of tag TAGNAME such that it is higher + than the priority of ABOVETHIS.""" + self.tk.call( + self._w, 'tag', 'raise', tagName, aboveThis) + def tag_ranges(self, tagName): + """Return a list of ranges of text which have tag TAGNAME.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'ranges', tagName)) + def tag_remove(self, tagName, index1, index2=None): + """Remove tag TAGNAME from all characters between INDEX1 and INDEX2.""" + self.tk.call( + self._w, 'tag', 'remove', tagName, index1, index2) + def window_cget(self, index, option): + """Return the value of OPTION of an embedded window at INDEX.""" + if option[:1] != '-': + option = '-' + option + if option[-1:] == '_': + option = option[:-1] + return self.tk.call(self._w, 'window', 'cget', index, option) + def window_configure(self, index, cnf=None, **kw): + """Configure an embedded window at INDEX.""" + return self._configure(('window', 'configure', index), cnf, kw) + window_config = window_configure + def window_create(self, index, cnf={}, **kw): + """Create a window at INDEX.""" + self.tk.call( + (self._w, 'window', 'create', index) + + self._options(cnf, kw)) + def window_names(self): + """Return all names of embedded windows in this widget.""" + return self.tk.splitlist( + self.tk.call(self._w, 'window', 'names')) + def xview(self, *what): + """Query and change horizontal position of the view.""" + if not what: + return self._getdoubles(self.tk.call(self._w, 'xview')) + self.tk.call((self._w, 'xview') + what) + def xview_moveto(self, fraction): + """Adjusts the view in the window so that FRACTION of the + total width of the canvas is off-screen to the left.""" + self.tk.call(self._w, 'xview', 'moveto', fraction) + def xview_scroll(self, number, what): + """Shift the x-view according to NUMBER which is measured + in "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'xview', 'scroll', number, what) + def yview(self, *what): + """Query and change vertical position of the view.""" + if not what: + return self._getdoubles(self.tk.call(self._w, 'yview')) + self.tk.call((self._w, 'yview') + what) + def yview_moveto(self, fraction): + """Adjusts the view in the window so that FRACTION of the + total height of the canvas is off-screen to the top.""" + self.tk.call(self._w, 'yview', 'moveto', fraction) + def yview_scroll(self, number, what): + """Shift the y-view according to NUMBER which is measured + in "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'yview', 'scroll', number, what) + def yview_pickplace(self, *what): + """Obsolete function, use see.""" + self.tk.call((self._w, 'yview', '-pickplace') + what) + + +class _setit: + """Internal class. It wraps the command in the widget OptionMenu.""" + def __init__(self, var, value, callback=None): + self.__value = value + self.__var = var + self.__callback = callback + def __call__(self, *args): + self.__var.set(self.__value) + if self.__callback: + self.__callback(self.__value, *args) + +class OptionMenu(Menubutton): + """OptionMenu which allows the user to select a value from a menu.""" + def __init__(self, master, variable, value, *values, **kwargs): + """Construct an optionmenu widget with the parent MASTER, with + the resource textvariable set to VARIABLE, the initially selected + value VALUE, the other menu values VALUES and an additional + keyword argument command.""" + kw = {"borderwidth": 2, "textvariable": variable, + "indicatoron": 1, "relief": RAISED, "anchor": "c", + "highlightthickness": 2} + Widget.__init__(self, master, "menubutton", kw) + self.widgetName = 'tk_optionMenu' + menu = self.__menu = Menu(self, name="menu", tearoff=0) + self.menuname = menu._w + # 'command' is the only supported keyword + callback = kwargs.get('command') + if kwargs.has_key('command'): + del kwargs['command'] + if kwargs: + raise TclError, 'unknown option -'+kwargs.keys()[0] + menu.add_command(label=value, + command=_setit(variable, value, callback)) + for v in values: + menu.add_command(label=v, + command=_setit(variable, v, callback)) + self["menu"] = menu + + def __getitem__(self, name): + if name == 'menu': + return self.__menu + return Widget.__getitem__(self, name) + + def destroy(self): + """Destroy this widget and the associated menu.""" + Menubutton.destroy(self) + self.__menu = None + +class Image: + """Base class for images.""" + _last_id = 0 + def __init__(self, imgtype, name=None, cnf={}, master=None, **kw): + self.name = None + if not master: + master = _default_root + if not master: + raise RuntimeError, 'Too early to create image' + self.tk = master.tk + if not name: + Image._last_id += 1 + name = "pyimage%r" % (Image._last_id,) # tk itself would use image + # The following is needed for systems where id(x) + # can return a negative number, such as Linux/m68k: + if name[0] == '-': name = '_' + name[1:] + if kw and cnf: cnf = _cnfmerge((cnf, kw)) + elif kw: cnf = kw + options = () + for k, v in cnf.items(): + if callable(v): + v = self._register(v) + options = options + ('-'+k, v) + self.tk.call(('image', 'create', imgtype, name,) + options) + self.name = name + def __str__(self): return self.name + def __del__(self): + if self.name: + try: + self.tk.call('image', 'delete', self.name) + except TclError: + # May happen if the root was destroyed + pass + def __setitem__(self, key, value): + self.tk.call(self.name, 'configure', '-'+key, value) + def __getitem__(self, key): + return self.tk.call(self.name, 'configure', '-'+key) + def configure(self, **kw): + """Configure the image.""" + res = () + for k, v in _cnfmerge(kw).items(): + if v is not None: + if k[-1] == '_': k = k[:-1] + if callable(v): + v = self._register(v) + res = res + ('-'+k, v) + self.tk.call((self.name, 'config') + res) + config = configure + def height(self): + """Return the height of the image.""" + return getint( + self.tk.call('image', 'height', self.name)) +# def type(self): +# """Return the type of the imgage, e.g. "photo" or "bitmap".""" +# return self.tk.call('image', 'type', self.name) + def width(self): + """Return the width of the image.""" + return getint( + self.tk.call('image', 'width', self.name)) + +class PhotoImage(Image): + """Widget which can display colored images in GIF, PPM/PGM format.""" + def __init__(self, name=None, cnf={}, master=None, **kw): + """Create an image with NAME. + + Valid resource names: data, format, file, gamma, height, palette, + width.""" + Image.__init__(self, 'photo', name, cnf, master, **kw) + def blank(self): + """Display a transparent image.""" + self.tk.call(self.name, 'blank') + def cget(self, option): + """Return the value of OPTION.""" + return self.tk.call(self.name, 'cget', '-' + option) + # XXX config + def __getitem__(self, key): + return self.tk.call(self.name, 'cget', '-' + key) + # XXX copy -from, -to, ...? + def copy(self): + """Return a new PhotoImage with the same image as this widget.""" + destImage = PhotoImage() + self.tk.call(destImage, 'copy', self.name) + return destImage + def zoom(self,x,y=''): + """Return a new PhotoImage with the same image as this widget + but zoom it with X and Y.""" + destImage = PhotoImage() + if y=='': y=x + self.tk.call(destImage, 'copy', self.name, '-zoom',x,y) + return destImage + def subsample(self,x,y=''): + """Return a new PhotoImage based on the same image as this widget + but use only every Xth or Yth pixel.""" + destImage = PhotoImage() + if y=='': y=x + self.tk.call(destImage, 'copy', self.name, '-subsample',x,y) + return destImage + def get(self, x, y): + """Return the color (red, green, blue) of the pixel at X,Y.""" + return self.tk.call(self.name, 'get', x, y) + def put(self, data, to=None): + """Put row formated colors to image starting from + position TO, e.g. image.put("{red green} {blue yellow}", to=(4,6))""" + args = (self.name, 'put', data) + if to: + if to[0] == '-to': + to = to[1:] + args = args + ('-to',) + tuple(to) + self.tk.call(args) + # XXX read + def write(self, filename, format=None, from_coords=None): + """Write image to file FILENAME in FORMAT starting from + position FROM_COORDS.""" + args = (self.name, 'write', filename) + if format: + args = args + ('-format', format) + if from_coords: + args = args + ('-from',) + tuple(from_coords) + self.tk.call(args) + +class BitmapImage(Image): + """Widget which can display a bitmap.""" + def __init__(self, name=None, cnf={}, master=None, **kw): + """Create a bitmap with NAME. + + Valid resource names: background, data, file, foreground, maskdata, maskfile.""" + Image.__init__(self, 'bitmap', name, cnf, master, **kw) + +def image_names(): return _default_root.tk.call('image', 'names') +def image_types(): return _default_root.tk.call('image', 'types') + + +class Spinbox(Widget): + """spinbox widget.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a spinbox widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, background, borderwidth, + cursor, exportselection, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, insertbackground, + insertborderwidth, insertofftime, + insertontime, insertwidth, justify, relief, + repeatdelay, repeatinterval, + selectbackground, selectborderwidth + selectforeground, takefocus, textvariable + xscrollcommand. + + WIDGET-SPECIFIC OPTIONS + + buttonbackground, buttoncursor, + buttondownrelief, buttonuprelief, + command, disabledbackground, + disabledforeground, format, from, + invalidcommand, increment, + readonlybackground, state, to, + validate, validatecommand values, + width, wrap, + """ + Widget.__init__(self, master, 'spinbox', cnf, kw) + + def bbox(self, index): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a + rectangle which encloses the character given by index. + + The first two elements of the list give the x and y + coordinates of the upper-left corner of the screen + area covered by the character (in pixels relative + to the widget) and the last two elements give the + width and height of the character, in pixels. The + bounding box may refer to a region outside the + visible area of the window. + """ + return self.tk.call(self._w, 'bbox', index) + + def delete(self, first, last=None): + """Delete one or more elements of the spinbox. + + First is the index of the first character to delete, + and last is the index of the character just after + the last one to delete. If last isn't specified it + defaults to first+1, i.e. a single character is + deleted. This command returns an empty string. + """ + return self.tk.call(self._w, 'delete', first, last) + + def get(self): + """Returns the spinbox's string""" + return self.tk.call(self._w, 'get') + + def icursor(self, index): + """Alter the position of the insertion cursor. + + The insertion cursor will be displayed just before + the character given by index. Returns an empty string + """ + return self.tk.call(self._w, 'icursor', index) + + def identify(self, x, y): + """Returns the name of the widget at position x, y + + Return value is one of: none, buttondown, buttonup, entry + """ + return self.tk.call(self._w, 'identify', x, y) + + def index(self, index): + """Returns the numerical index corresponding to index + """ + return self.tk.call(self._w, 'index', index) + + def insert(self, index, s): + """Insert string s at index + + Returns an empty string. + """ + return self.tk.call(self._w, 'insert', index, s) + + def invoke(self, element): + """Causes the specified element to be invoked + + The element could be buttondown or buttonup + triggering the action associated with it. + """ + return self.tk.call(self._w, 'invoke', element) + + def scan(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'scan') + args)) or () + + def scan_mark(self, x): + """Records x and the current view in the spinbox window; + + used in conjunction with later scan dragto commands. + Typically this command is associated with a mouse button + press in the widget. It returns an empty string. + """ + return self.scan("mark", x) + + def scan_dragto(self, x): + """Compute the difference between the given x argument + and the x argument to the last scan mark command + + It then adjusts the view left or right by 10 times the + difference in x-coordinates. This command is typically + associated with mouse motion events in the widget, to + produce the effect of dragging the spinbox at high speed + through the window. The return value is an empty string. + """ + return self.scan("dragto", x) + + def selection(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'selection') + args)) or () + + def selection_adjust(self, index): + """Locate the end of the selection nearest to the character + given by index, + + Then adjust that end of the selection to be at index + (i.e including but not going beyond index). The other + end of the selection is made the anchor point for future + select to commands. If the selection isn't currently in + the spinbox, then a new selection is created to include + the characters between index and the most recent selection + anchor point, inclusive. Returns an empty string. + """ + return self.selection("adjust", index) + + def selection_clear(self): + """Clear the selection + + If the selection isn't in this widget then the + command has no effect. Returns an empty string. + """ + return self.selection("clear") + + def selection_element(self, element=None): + """Sets or gets the currently selected element. + + If a spinbutton element is specified, it will be + displayed depressed + """ + return self.selection("element", element) + +########################################################################### + +class LabelFrame(Widget): + """labelframe widget.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a labelframe widget with the parent MASTER. + + STANDARD OPTIONS + + borderwidth, cursor, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, padx, pady, relief, + takefocus, text + + WIDGET-SPECIFIC OPTIONS + + background, class, colormap, container, + height, labelanchor, labelwidget, + visual, width + """ + Widget.__init__(self, master, 'labelframe', cnf, kw) + +######################################################################## + +class PanedWindow(Widget): + """panedwindow widget.""" + def __init__(self, master=None, cnf={}, **kw): + """Construct a panedwindow widget with the parent MASTER. + + STANDARD OPTIONS + + background, borderwidth, cursor, height, + orient, relief, width + + WIDGET-SPECIFIC OPTIONS + + handlepad, handlesize, opaqueresize, + sashcursor, sashpad, sashrelief, + sashwidth, showhandle, + """ + Widget.__init__(self, master, 'panedwindow', cnf, kw) + + def add(self, child, **kw): + """Add a child widget to the panedwindow in a new pane. + + The child argument is the name of the child widget + followed by pairs of arguments that specify how to + manage the windows. Options may have any of the values + accepted by the configure subcommand. + """ + self.tk.call((self._w, 'add', child) + self._options(kw)) + + def remove(self, child): + """Remove the pane containing child from the panedwindow + + All geometry management options for child will be forgotten. + """ + self.tk.call(self._w, 'forget', child) + forget=remove + + def identify(self, x, y): + """Identify the panedwindow component at point x, y + + If the point is over a sash or a sash handle, the result + is a two element list containing the index of the sash or + handle, and a word indicating whether it is over a sash + or a handle, such as {0 sash} or {2 handle}. If the point + is over any other part of the panedwindow, the result is + an empty list. + """ + return self.tk.call(self._w, 'identify', x, y) + + def proxy(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'proxy') + args)) or () + + def proxy_coord(self): + """Return the x and y pair of the most recent proxy location + """ + return self.proxy("coord") + + def proxy_forget(self): + """Remove the proxy from the display. + """ + return self.proxy("forget") + + def proxy_place(self, x, y): + """Place the proxy at the given x and y coordinates. + """ + return self.proxy("place", x, y) + + def sash(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'sash') + args)) or () + + def sash_coord(self, index): + """Return the current x and y pair for the sash given by index. + + Index must be an integer between 0 and 1 less than the + number of panes in the panedwindow. The coordinates given are + those of the top left corner of the region containing the sash. + pathName sash dragto index x y This command computes the + difference between the given coordinates and the coordinates + given to the last sash coord command for the given sash. It then + moves that sash the computed difference. The return value is the + empty string. + """ + return self.sash("coord", index) + + def sash_mark(self, index): + """Records x and y for the sash given by index; + + Used in conjunction with later dragto commands to move the sash. + """ + return self.sash("mark", index) + + def sash_place(self, index, x, y): + """Place the sash given by index at the given coordinates + """ + return self.sash("place", index, x, y) + + def panecget(self, child, option): + """Query a management option for window. + + Option may be any value allowed by the paneconfigure subcommand + """ + return self.tk.call( + (self._w, 'panecget') + (child, '-'+option)) + + def paneconfigure(self, tagOrId, cnf=None, **kw): + """Query or modify the management options for window. + + If no option is specified, returns a list describing all + of the available options for pathName. If option is + specified with no value, then the command returns a list + describing the one named option (this list will be identical + to the corresponding sublist of the value returned if no + option is specified). If one or more option-value pairs are + specified, then the command modifies the given widget + option(s) to have the given value(s); in this case the + command returns an empty string. The following options + are supported: + + after window + Insert the window after the window specified. window + should be the name of a window already managed by pathName. + before window + Insert the window before the window specified. window + should be the name of a window already managed by pathName. + height size + Specify a height for the window. The height will be the + outer dimension of the window including its border, if + any. If size is an empty string, or if -height is not + specified, then the height requested internally by the + window will be used initially; the height may later be + adjusted by the movement of sashes in the panedwindow. + Size may be any value accepted by Tk_GetPixels. + minsize n + Specifies that the size of the window cannot be made + less than n. This constraint only affects the size of + the widget in the paned dimension -- the x dimension + for horizontal panedwindows, the y dimension for + vertical panedwindows. May be any value accepted by + Tk_GetPixels. + padx n + Specifies a non-negative value indicating how much + extra space to leave on each side of the window in + the X-direction. The value may have any of the forms + accepted by Tk_GetPixels. + pady n + Specifies a non-negative value indicating how much + extra space to leave on each side of the window in + the Y-direction. The value may have any of the forms + accepted by Tk_GetPixels. + sticky style + If a window's pane is larger than the requested + dimensions of the window, this option may be used + to position (or stretch) the window within its pane. + Style is a string that contains zero or more of the + characters n, s, e or w. The string can optionally + contains spaces or commas, but they are ignored. Each + letter refers to a side (north, south, east, or west) + that the window will "stick" to. If both n and s + (or e and w) are specified, the window will be + stretched to fill the entire height (or width) of + its cavity. + width size + Specify a width for the window. The width will be + the outer dimension of the window including its + border, if any. If size is an empty string, or + if -width is not specified, then the width requested + internally by the window will be used initially; the + width may later be adjusted by the movement of sashes + in the panedwindow. Size may be any value accepted by + Tk_GetPixels. + + """ + if cnf is None and not kw: + cnf = {} + for x in self.tk.split( + self.tk.call(self._w, + 'paneconfigure', tagOrId)): + cnf[x[0][1:]] = (x[0][1:],) + x[1:] + return cnf + if type(cnf) == StringType and not kw: + x = self.tk.split(self.tk.call( + self._w, 'paneconfigure', tagOrId, '-'+cnf)) + return (x[0][1:],) + x[1:] + self.tk.call((self._w, 'paneconfigure', tagOrId) + + self._options(cnf, kw)) + paneconfig = paneconfigure + + def panes(self): + """Returns an ordered list of the child panes.""" + return self.tk.call(self._w, 'panes') + +###################################################################### +# Extensions: + +class Studbutton(Button): + def __init__(self, master=None, cnf={}, **kw): + Widget.__init__(self, master, 'studbutton', cnf, kw) + self.bind('', self.tkButtonEnter) + self.bind('', self.tkButtonLeave) + self.bind('<1>', self.tkButtonDown) + self.bind('', self.tkButtonUp) + +class Tributton(Button): + def __init__(self, master=None, cnf={}, **kw): + Widget.__init__(self, master, 'tributton', cnf, kw) + self.bind('', self.tkButtonEnter) + self.bind('', self.tkButtonLeave) + self.bind('<1>', self.tkButtonDown) + self.bind('', self.tkButtonUp) + self['fg'] = self['bg'] + self['activebackground'] = self['bg'] + +###################################################################### +# Test: + +def _test(): + root = Tk() + text = "This is Tcl/Tk version %s" % TclVersion + if TclVersion >= 8.1: + try: + text = text + unicode("\nThis should be a cedilla: \347", + "iso-8859-1") + except NameError: + pass # no unicode support + label = Label(root, text=text) + label.pack() + test = Button(root, text="Click me!", + command=lambda root=root: root.test.configure( + text="[%s]" % root.test['text'])) + test.pack() + root.test = test + quit = Button(root, text="QUIT", command=root.destroy) + quit.pack() + # The following three commands are needed so the window pops + # up on top on Windows... + root.iconify() + root.update() + root.deiconify() + root.mainloop() + +if __name__ == '__main__': + _test() Index: src/org/python/indexer/NBinding.java =================================================================== --- src/org/python/indexer/NBinding.java (revision 0) +++ src/org/python/indexer/NBinding.java (revision 0) @@ -0,0 +1,343 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NUrl; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * An {@code NBinding} collects information about a fully qualified name (qname) + * in the code graph.

+ * + * Each qname has an associated {@link NType}. When a particular qname is + * assigned values of different types at different locations, its type is + * represented as a {@link NUnionType}.

+ * + * Each qname has a set of one or more definitions, and a set of zero or + * more references. Definitions and references correspond to code locations.

+ */ +public class NBinding implements Comparable { + + /** + * In addition to its type, each binding has a {@link Kind} enumeration that + * attempts to represent the structural role the name plays in the code. + * This is a rough categorization that takes into account type information, + * structural (AST) information, and possibly other semantics. It can help + * IDEs with presentation decisions, and can be useful to expose to users as + * a parameter for filtering queries on the graph. + */ + public enum Kind { + ATTRIBUTE, // attr accessed with "." on some other object + CLASS, // class definition + CONSTRUCTOR, // __init__ functions in classes + FUNCTION, // plain function + METHOD, // static or instance method + MODULE, // file + PARAMETER, // function param + SCOPE, // top-level variable ("scope" means we assume it can have attrs) + VARIABLE // local variable + } + + private static final int PROVISIONAL = 1 << 0; // (for internal use only) + private static final int STATIC = 1 << 1; // static fields/methods + private static final int SYNTHETIC = 1 << 2; // auto-generated bindings + private static final int READONLY = 1 << 3; // non-writable attributes + private static final int DEPRECATED = 1 << 4; // documented as deprecated + private static final int BUILTIN = 1 << 5; // not from a source file + + // The indexer is heavily memory-constrained, so these sets are initially + // small. The vast majority of bindings have only one definition. + private static final int DEF_SET_INITIAL_CAPACITY = 1; + private static final int REF_SET_INITIAL_CAPACITY = 8; + + private String name; // unqualified name + private String qname; // qualified name + private NType type; // inferred type + Kind kind; // name usage context + int modifiers; // metadata flags + + private List defs; + private Set refs; + + public NBinding(String id, NNode node, NType type, Kind kind) { + this(id, node != null ? new Def(node) : null, type, kind); + } + + public NBinding(String id, Def def, NType type, Kind kind) { + if (id == null) { + throw new IllegalArgumentException("'id' param cannot be null"); + } + qname = name = id; + defs = new ArrayList(DEF_SET_INITIAL_CAPACITY); + addDef(def); + this.type = type == null ? new NUnknownType() : type; + this.kind = kind == null ? Kind.SCOPE : kind; + } + + /** + * Sets the binding's simple (unqualified) name. + */ + public void setName(String name) { + this.name = name; + } + + /** + * Returns the unqualified name. + */ + public String getName() { + return name; + } + + /** + * Sets the binding's qualified name. This should in general be the + * same as {@code binding.getType().getTable().getPath()}. + */ + public void setQname(String qname) { + this.qname = qname; + } + + /** + * Returns the qualified name. + */ + public String getQname() { + return qname; + } + + /** + * Adds {@code node} as a definition for this binding. This is called + * automatically (when appropriate) by adding the binding to a + * {@link Scope}. + */ + public void addDef(NNode node) { + if (node != null) { + addDef(new Def(node)); + } + } + + public void addDefs(Collection nodes) { + for (NNode n : nodes) { + addDef(n); + } + } + + /** + * Adds {@code def} as a definition for this binding. This is called + * automatically (when appropriate) by adding the binding to a + * {@link Scope}. If {@code node} is an {@link NUrl}, and this is the + * binding's only definition, it will be marked as a builtin. + */ + public void addDef(Def def) { + if (def == null) { + return; + } + List defs = getDefs(); + // A def may be added during the name-binding phase, and re-added + // when the type is updated during the resolve phase. + if (defs.contains(def)) { + return; + } + + defs.add(def); + def.setBinding(this); + + if (def.isURL()) { + markBuiltin(); + } + } + + public void addRef(NNode node) { + addRef(new Ref(node)); + } + + public void addRef(Ref ref) { + getRefs().add(ref); + } + + public void removeRef(Ref node) { + getRefs().remove(node); + } + + /** + * Returns the first definition, which by convention is treated as + * the one that introduced the binding. + */ + public Def getSignatureNode() { + return getDefs().isEmpty() ? null : getDefs().get(0); + } + + public void setType(NType type) { + this.type = type; + } + + public NType getType() { + return type; + } + + /** + * Returns the type, first calling {@link NUnknownType#follow} on it. + * For external consumers of the index, this is usually preferable + * to calling {@link #getType}. + */ + public NType followType() { + return NUnknownType.follow(type); + } + + public void setKind(Kind kind) { + this.kind = kind; + } + + public Kind getKind() { + return kind; + } + + public void markStatic() { + modifiers |= STATIC; + } + + public boolean isStatic() { + return (modifiers & STATIC) != 0; + } + + public void markSynthetic() { + modifiers |= SYNTHETIC; + } + + public boolean isSynthetic() { + return (modifiers & SYNTHETIC) != 0; + } + + public void markReadOnly() { + modifiers |= READONLY; + } + + public boolean isReadOnly() { + return (modifiers & READONLY) != 0; + } + + public boolean isDeprecated() { + return (modifiers & DEPRECATED) != 0; + } + + public void markDeprecated() { + modifiers |= DEPRECATED; + } + + public boolean isBuiltin() { + return (modifiers & BUILTIN) != 0; + } + + public void markBuiltin() { + modifiers |= BUILTIN; + } + + public void setProvisional(boolean isProvisional) { + if (isProvisional) { + modifiers |= PROVISIONAL; + } else { + modifiers &= ~PROVISIONAL; + } + } + + public boolean isProvisional() { + return (modifiers & PROVISIONAL) != 0; + } + + /** + * Bindings can be sorted by their location for outlining purposes. + */ + public int compareTo(Object o) { + return getSignatureNode().start() - ((NBinding)o).getSignatureNode().start(); + } + + /** + * Return the (possibly empty) set of definitions for this binding. + * @return the defs + */ + public List getDefs() { + if (defs == null) { + defs = new ArrayList(DEF_SET_INITIAL_CAPACITY); + } + return defs; + } + + /** + * Returns the number of definitions found for this binding. + */ + public int getNumDefs() { + return defs == null ? 0 : defs.size(); + } + + public boolean hasRefs() { + return refs == null ? false : !refs.isEmpty(); + } + + public int getNumRefs() { + return refs == null ? 0 : refs.size(); + } + + /** + * Returns the set of references to this binding. + */ + public Set getRefs() { + if (refs == null) { + refs = new LinkedHashSet(REF_SET_INITIAL_CAPACITY); + } + return refs; + } + + /** + * Returns a filename associated with this binding, for debug + * messages. + * @return the filename associated with the type (if present) + * or the first definition (if present), otherwise a string + * describing what is known about the binding's source. + */ + public String getFirstFile() { + NType bt = getType(); + if (bt instanceof NModuleType) { + String file = bt.asModuleType().getFile(); + return file != null ? file : ""; + } + if (defs != null) { + for (Def def : defs) { + String file = def.getFile(); + if (file != null) { + return file; + } + } + return ""; + } + return ""; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(""); + return sb.toString(); + } +} Index: tests/java/org/python/indexer/data/yinw/yinw-10.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-10.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-10.py (revision 0) @@ -0,0 +1,5 @@ +def f(): + print x + +x = 1 + Index: src/org/python/indexer/ast/NRaise.java =================================================================== --- src/org/python/indexer/ast/NRaise.java (revision 0) +++ src/org/python/indexer/ast/NRaise.java (revision 0) @@ -0,0 +1,50 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NRaise extends NNode { + + static final long serialVersionUID = 5384576775167988640L; + + public NNode exceptionType; + public NNode inst; + public NNode traceback; + + public NRaise(NNode exceptionType, NNode inst, NNode traceback) { + this(exceptionType, inst, traceback, 0, 1); + } + + public NRaise(NNode exceptionType, NNode inst, NNode traceback, int start, int end) { + super(start, end); + this.exceptionType = exceptionType; + this.inst = inst; + this.traceback = traceback; + addChildren(exceptionType, inst, traceback); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(exceptionType, s); + resolveExpr(inst, s); + resolveExpr(traceback, s); + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(exceptionType, v); + visitNode(inst, v); + visitNode(traceback, v); + } + } +} Index: src/org/python/indexer/ast/NIfExp.java =================================================================== --- src/org/python/indexer/ast/NIfExp.java (revision 0) +++ src/org/python/indexer/ast/NIfExp.java (revision 0) @@ -0,0 +1,54 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NIfExp extends NNode { + + static final long serialVersionUID = 8516153579808365723L; + + public NNode test; + public NNode body; + public NNode orelse; + + public NIfExp(NNode test, NNode body, NNode orelse) { + this(test, body, orelse, 0, 1); + } + + public NIfExp(NNode test, NNode body, NNode orelse, int start, int end) { + super(start, end); + this.test = test; + this.body = body; + this.orelse = orelse; + addChildren(test, body, orelse); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(test, s); + if (body != null) { + setType(resolveExpr(body, s)); + } + if (orelse != null) { + addType(resolveExpr(orelse, s)); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(test, v); + visitNode(body, v); + visitNode(orelse, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/mineral/stone/obsidian.py =================================================================== --- tests/java/org/python/indexer/data/pkg/mineral/stone/obsidian.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/mineral/stone/obsidian.py (revision 0) @@ -0,0 +1,4 @@ +import .lapis +import ...mineral.stone.limestone + +category = "igneous" Index: src/org/python/indexer/ast/NFor.java =================================================================== --- src/org/python/indexer/ast/NFor.java (revision 0) +++ src/org/python/indexer/ast/NFor.java (revision 0) @@ -0,0 +1,89 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NFuncType; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NTupleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +public class NFor extends NNode { + + static final long serialVersionUID = 3228529969554646406L; + + public NNode target; + public NNode iter; + public NBlock body; + public NBlock orelse; + + public NFor(NNode target, NNode iter, NBlock body, NBlock orelse) { + this(target, iter, body, orelse, 0, 1); + } + + public NFor(NNode target, NNode iter, NBlock body, NBlock orelse, + int start, int end) { + super(start, end); + this.target = target; + this.iter = iter; + this.body = body; + this.orelse = orelse; + addChildren(target, iter, body, orelse); + } + + @Override + public boolean bindsName() { + return true; + } + + @Override + protected void bindNames(Scope s) throws Exception { + bindNames(s, target, NameBinder.make()); + } + + private void bindNames(Scope s, NNode target, NameBinder binder) throws Exception { + if (target instanceof NName) { + binder.bind(s, (NName)target, new NUnknownType()); + return; + } + if (target instanceof NSequence) { + for (NNode n : ((NSequence)target).getElements()) { + bindNames(s, n, binder); + } + } + } + + @Override + public NType resolve(Scope s) throws Exception { + NameBinder.make().bindIter(s, target, iter); + + if (body == null) { + setType(new NUnknownType()); + } else { + setType(resolveExpr(body, s)); + } + if (orelse != null) { + addType(resolveExpr(orelse, s)); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(target, v); + visitNode(iter, v); + visitNode(body, v); + visitNode(orelse, v); + } + } +} Index: src/org/python/indexer/types/NTupleType.java =================================================================== --- src/org/python/indexer/types/NTupleType.java (revision 0) +++ src/org/python/indexer/types/NTupleType.java (revision 0) @@ -0,0 +1,80 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.ast.NUrl; + +import java.util.ArrayList; +import java.util.List; + +public class NTupleType extends NType { + + private List eltTypes; + + public NTupleType() { + this.eltTypes = new ArrayList(); + getTable().addSuper(Indexer.idx.builtins.BaseTuple.getTable()); + getTable().setPath(Indexer.idx.builtins.BaseTuple.getTable().getPath()); + } + + public NTupleType(List eltTypes) { + this(); + this.eltTypes = eltTypes; + } + + public NTupleType(NType elt0) { + this(); + this.eltTypes.add(elt0); + } + + public NTupleType(NType elt0, NType elt1) { + this(); + this.eltTypes.add(elt0); + this.eltTypes.add(elt1); + } + + public NTupleType(NType... types) { + this(); + for (NType type : types) { + this.eltTypes.add(type); + } + } + + public void setElementTypes(List eltTypes) { + this.eltTypes = eltTypes; + } + + public List getElementTypes() { + return eltTypes; + } + + public void add(NType elt) { + eltTypes.add(elt); + } + + public NType get(int i) { + return eltTypes.get(i); + } + + public NListType toListType() { + NListType t = new NListType(); + for (NType e : eltTypes) { + t.add(e); + } + return t; + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + sb.append("["); + for (NType elt : eltTypes) { + elt.print(ctr, sb); + sb.append(","); + } + sb.setLength(sb.length() - 1); // pop last comma + sb.append("]"); + } +} Index: src/org/python/indexer/types/NListType.java =================================================================== --- src/org/python/indexer/types/NListType.java (revision 0) +++ src/org/python/indexer/types/NListType.java (revision 0) @@ -0,0 +1,52 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.ast.NUrl; + +public class NListType extends NType { + + private NType eltType; + + public NListType() { + this(new NUnknownType()); + } + + public NListType(NType elt0) { + eltType = elt0; + getTable().addSuper(Indexer.idx.builtins.BaseList.getTable()); + getTable().setPath(Indexer.idx.builtins.BaseList.getTable().getPath()); + } + + public void setElementType(NType eltType) { + this.eltType = eltType; + } + + /** + * Returns the type of the elements. You should wrap the result + * with {@link NUnknownType#follow} to get to the actual type. + */ + public NType getElementType() { + return eltType; + } + + public void add(NType another) { + eltType = NUnionType.union(eltType, another); + } + + public NTupleType toTupleType(int n) { + NTupleType ret = new NTupleType(); + for (int i = 0; i < n; i++) { + ret.add(eltType); + } + return ret; + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + eltType.print(ctr, sb); + } +} Index: src/org/python/indexer/demos/StyleApplier.java =================================================================== --- src/org/python/indexer/demos/StyleApplier.java (revision 0) +++ src/org/python/indexer/demos/StyleApplier.java (revision 0) @@ -0,0 +1,163 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.demos; + +import org.python.indexer.StyleRun; + +import java.util.ArrayList; +import java.util.List; +import java.util.HashSet; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +/** + * Turns a list of {@link StyleRun}s into HTML spans. + */ +class StyleApplier { + + // Empirically, adding the span tags multiplies length by 6 or more. + private static final int SOURCE_BUF_MULTIPLIER = 6; + + private SortedSet tags = new TreeSet(); + + private StringBuilder buffer; // html output buffer + + private String source; // input source code + private String path; // input source path, for logging + + // Current offset into the source being copied into the html buffer. + private int sourceOffset = 0; + + abstract class Tag implements Comparable{ + int offset; + StyleRun style; + @Override + public int compareTo(Tag other) { + if (this == other) { + return 0; + } + if (this.offset < other.offset) { + return -1; + } + if (other.offset < this.offset) { + return 1; + } + return this.hashCode() - other.hashCode(); + } + void insert() { + // Copy source code up through this tag. + if (offset > sourceOffset) { + copySource(sourceOffset, offset); + } + } + } + + class StartTag extends Tag { + public StartTag(StyleRun style) { + offset = style.start(); + this.style = style; + } + @Override + void insert() { + super.insert(); + switch (style.type) { + case ANCHOR: + buffer.append(""); + } + } + + class EndTag extends Tag { + public EndTag(StyleRun style) { + offset = style.end(); + this.style = style; + } + @Override + void insert() { + super.insert(); + switch (style.type) { + case ANCHOR: + case LINK: + buffer.append(""); + break; + default: + buffer.append(""); + break; + } + } + } + + public StyleApplier(String path, String src, List runs) { + this.path = path; + source = src; + for (StyleRun run : runs) { + tags.add(new StartTag(run)); + tags.add(new EndTag(run)); + } + } + + /** + * @return the html + */ + public String apply() { + buffer = new StringBuilder(source.length() * SOURCE_BUF_MULTIPLIER); + int lastStart = -1; + for (Tag tag : tags) { + tag.insert(); + } + // Copy in remaining source beyond last tag. + if (sourceOffset < source.length()) { + copySource(sourceOffset, source.length()); + } + return buffer.toString(); + } + + /** + * Copies code from the input source to the output html. + * @param beg the starting source offset + * @param end the end offset, or -1 to go to end of file + */ + private void copySource(int beg, int end) { + // Be robust if the analyzer gives us bad offsets. + try { + String src = escape((end == -1) + ? source.substring(beg) + : source.substring(beg, end)); + buffer.append(src); + } catch (RuntimeException x) { + System.err.println("Warning: " + x); + } + sourceOffset = end; + } + + private String escape(String s) { + return s.replace("&", "&") + .replace("'", "'") + .replace("\"", """) + .replace("<", "<") + .replace(">", ">"); + } + + private String toCSS(StyleRun style) { + return style.type.toString().toLowerCase().replace("_", "-"); + } +} Index: src/org/python/indexer/ast/NWith.java =================================================================== --- src/org/python/indexer/ast/NWith.java (revision 0) +++ src/org/python/indexer/ast/NWith.java (revision 0) @@ -0,0 +1,49 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NWith extends NNode { + + static final long serialVersionUID = 560128079414064421L; + + public NNode optional_vars; + public NNode context_expr; + public NBlock body; + + public NWith(NNode optional_vars, NNode context_expr, NBlock body) { + this(optional_vars, context_expr, body, 0, 1); + } + + public NWith(NNode optional_vars, NNode context_expr, NBlock body, int start, int end) { + super(start, end); + this.optional_vars = optional_vars; + this.context_expr = context_expr; + this.body = body; + addChildren(optional_vars, context_expr, body); + } + + @Override + public NType resolve(Scope s) throws Exception { + NType val = resolveExpr(context_expr, s); + NameBinder.make().bind(s, optional_vars, val); + return setType(resolveExpr(body, s)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(optional_vars, v); + visitNode(context_expr, v); + visitNode(body, v); + } + } +} Index: tests/java/org/python/indexer/data/pkgload.py =================================================================== --- tests/java/org/python/indexer/data/pkgload.py (revision 0) +++ tests/java/org/python/indexer/data/pkgload.py (revision 0) @@ -0,0 +1,2 @@ +# test loading a package by dirname +import pkg Index: tests/java/org/python/indexer/data/pkg/mineral/stone/limestone.py =================================================================== --- tests/java/org/python/indexer/data/pkg/mineral/stone/limestone.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/mineral/stone/limestone.py (revision 0) @@ -0,0 +1,2 @@ + +category = "sedimentary" Index: tests/java/org/python/indexer/data/yinw/yinw-20.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-20.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-20.py (revision 0) @@ -0,0 +1,9 @@ +def f(): + def g1(x): + return g2(x) + def g2(y): + return y+z + z = 3 + print g1(1) + +f() Index: src/org/python/indexer/ast/NName.java =================================================================== --- src/org/python/indexer/ast/NName.java (revision 0) +++ src/org/python/indexer/ast/NName.java (revision 0) @@ -0,0 +1,93 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +public class NName extends NNode { + + static final long serialVersionUID = -1160862551327528304L; + + public final String id; // identifier + + public NName(String id) { + this(id, 0, 1); + } + + public NName(String id, int start, int end) { + super(start, end); + if (id == null) { + throw new IllegalArgumentException("'id' param cannot be null"); + } + this.id = id; + } + + @Override + public NType resolve(Scope s) throws Exception { + NBinding b = s.lookup(id); + if (b == null) { + b = makeTempBinding(s); + } + Indexer.idx.putLocation(this, b); + return setType(b.followType()); + } + + /** + * Returns {@code true} if this name is structurally in a call position. + */ + public boolean isCall() { + // foo(...) + if (parent != null && parent.isCall() && this == ((NCall)parent).func) { + return true; + } + + // .foo(...) + NNode gramps; + return parent instanceof NAttribute + && this == ((NAttribute)parent).attr + && (gramps = parent.parent) instanceof NCall + && parent == ((NCall)gramps).func; + } + + /** + * Create a temporary binding and definition for this name. + * If we later encounter a true definition we'll remove this + * node from the defs and add it to the refs. + */ + private NBinding makeTempBinding(Scope s) { + Scope scope = s.getScopeSymtab(); + + NBinding b = scope.put(id, this, new NUnknownType(), NBinding.Kind.SCOPE); + setType(b.getType().follow()); + + // Update the qname to this point in case we add attributes later. + // If we don't add attributes, this path extension is harmless/unused. + getTable().setPath(scope.extendPath(id)); + + return b; + } + + /** + * Returns {@code true} if this name node is the {@code attr} child + * (i.e. the attribute being accessed) of an {@link NAttribute} node. + */ + public boolean isAttribute() { + return parent instanceof NAttribute + && ((NAttribute)parent).getAttr() == this; + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: src/org/python/indexer/ast/NBlock.java =================================================================== --- src/org/python/indexer/ast/NBlock.java (revision 0) +++ src/org/python/indexer/ast/NBlock.java (revision 0) @@ -0,0 +1,56 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +import java.util.ArrayList; +import java.util.List; + +public class NBlock extends NNode { + + static final long serialVersionUID = -9096405259154069107L; + + public List seq; + + public NBlock(List seq) { + this(seq, 0, 1); + } + + public NBlock(List seq, int start, int end) { + super(start, end); + if (seq == null) { + seq = new ArrayList(); + } + this.seq = seq; + addChildren(seq); + } + + @Override + public NType resolve(Scope scope) throws Exception { + for (NNode n : seq) { + // XXX: This works for inferring lambda return types, but needs + // to be fixed for functions (should be union of return stmt types). + NType returnType = resolveExpr(n, scope); + if (returnType != Indexer.idx.builtins.None) { + setType(returnType); + } + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(seq, v); + } + } +} Index: src/org/python/indexer/ast/NImport.java =================================================================== --- src/org/python/indexer/ast/NImport.java (revision 0) +++ src/org/python/indexer/ast/NImport.java (revision 0) @@ -0,0 +1,128 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Builtins; +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +public class NImport extends NNode { + + static final long serialVersionUID = -2180402676651342012L; + + public List aliases; // import foo.bar as bar, ..x.y as y + + public NImport(List aliases) { + this(aliases, 0, 1); + } + + public NImport(List aliases, int start, int end) { + super(start, end); + this.aliases = aliases; + addChildren(aliases); + } + + @Override + public boolean bindsName() { + return true; + } + + @Override + protected void bindNames(Scope s) throws Exception { + bindAliases(s, aliases); + } + + static void bindAliases(Scope s, List aliases) throws Exception { + NameBinder binder = NameBinder.make(); + for (NAlias a : aliases) { + if (a.aname != null) { + binder.bind(s, a.aname, new NUnknownType()); + } + } + } + + @Override + public NType resolve(Scope s) throws Exception { + Scope scope = s.getScopeSymtab(); + for (NAlias a : aliases) { + NType modtype = resolveExpr(a, s); + if (modtype.isModuleType()) { + importName(scope, a, modtype.asModuleType()); + } + } + return getType(); + } + + /** + * Import a module's alias (if present) or top-level name into the current + * scope. Creates references to the other names in the alias. + * + * @param mt the module that is actually bound to the imported name. + * for {@code import x.y.z as foo}, it is {@code z}, a sub-module + * of {@code x.y}. For {@code import x.y.z} it is {@code x}. + */ + private void importName(Scope s, NAlias a, NModuleType mt) throws Exception { + if (a.aname != null) { + if (mt.getFile() != null) { + NameBinder.make().bind(s, a.aname, mt); + } else { + // XXX: seems like the url should be set in loadModule, not here. + // Can't the moduleTable store url-keyed modules too? + s.update(a.aname.id, + new NUrl(Builtins.LIBRARY_URL + mt.getTable().getPath() + ".html"), + mt, NBinding.Kind.SCOPE); + } + } + + addReferences(s, a.qname, true/*put top name in scope*/); + } + + static void addReferences(Scope s, NQname qname, boolean putTopInScope) { + if (qname == null) { + return; + } + if (!qname.getType().isModuleType()) { + return; + } + NModuleType mt = qname.getType().asModuleType(); + + String modQname = mt.getTable().getPath(); + NBinding mb = Indexer.idx.lookupQname(modQname); + if (mb == null) { + mb = Indexer.idx.moduleTable.lookup(modQname); + } + + if (mb == null) { + Indexer.idx.putProblem(qname.getName(), "module not found"); + return; + } + + Indexer.idx.putLocation(qname.getName(), mb); + + // All other references in the file should also point to this binding. + if (putTopInScope && qname.isTop()) { + s.put(qname.getName().id, mb); + } + + addReferences(s, qname.getNext(), false); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(aliases, v); + } + } +} Index: tests/java/org/python/indexer/TestBase.java =================================================================== --- tests/java/org/python/indexer/TestBase.java (revision 0) +++ tests/java/org/python/indexer/TestBase.java (revision 0) @@ -0,0 +1,415 @@ +package org.python.indexer; + +import junit.framework.TestCase; + +import org.python.indexer.Def; +import org.python.indexer.Ref; +import org.python.indexer.ast.NNode; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.InputStreamReader; + +/** + * Test utilities for {@link IndexerTest}. + */ +public class TestBase extends TestCase { + + static protected final String TEST_SOURCE_DIR; + static protected final String TEST_DATA_DIR; + static protected final String TEST_LIB_DIR; + + static { + TEST_SOURCE_DIR = + System.getProperties().getProperty("python.test.source.dir") + + "/org/python/indexer/"; + TEST_DATA_DIR = TEST_SOURCE_DIR + "data/"; + TEST_LIB_DIR = System.getProperties().getProperty("python.home") + "/Lib/"; + } + + protected Indexer idx; + + public TestBase() { + } + + @Override + protected void setUp() throws Exception { + idx = new Indexer(); + idx.enableAggressiveAssertions(true); + idx.setProjectDir(TEST_DATA_DIR); + AstCache.get().clearDiskCache(); + AstCache.get().clear(); + } + + /** + * Call this at the beginning of a test to permit the test code to import + * modules from the Python standard library. + */ + protected void includeStandardLibrary() throws Exception { + idx.addPath(TEST_LIB_DIR); + } + + protected String abspath(String file) { + return getTestFilePath(file); + } + + /** + * Return absolute path for {@code file}, a relative path under the + * data/ directory. + */ + protected String getTestFilePath(String file) { + return TEST_DATA_DIR + file; + } + + protected String getSource(String file) throws Exception { + String path = getTestFilePath(file); + StringBuilder sb = new StringBuilder(); + BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(path))); + String line; + while ((line = in.readLine()) != null) { + sb.append(line); + sb.append("\n"); + } + in.close(); // not overly worried about resource cleanup in unit tests + return sb.toString(); + } + + /** + * Construct python source by joining the specified lines. + */ + protected String makeModule(String... lines) { + StringBuilder sb = new StringBuilder(); + for (String line : lines) { + sb.append(line).append("\n"); + } + return sb.toString(); + } + + /** + * Build an index out of the specified file and content lines, + * and return the resulting module source. + */ + protected String index(String filename, String... lines) throws Exception { + String src = makeModule(lines); + idx.loadString(filename, src); + idx.ready(); + return src; + } + + /** + * Return offset in {@code s} of {@code n}th occurrence of {@code find}. + * {@code n} is 1-indexed. + * @throws IllegalArgumentException if the {@code n}th occurrence does not exist + */ + protected int nthIndexOf(String s, String find, int n) { + if (n <= 0) + throw new IllegalArgumentException(); + int index = -1; + for (int i = 0; i < n; i++) { + index = s.indexOf(find, index == -1 ? 0 : index + 1); + if (index == -1) + throw new IllegalArgumentException(); + } + return index; + } + + // meta-tests + + public void testHandleExceptionLoggingNulls() throws Exception { + try { + idx.enableAggressiveAssertions(false); + idx.getLogger().setLevel(java.util.logging.Level.OFF); + idx.handleException(null, new Exception()); + idx.handleException("oops", null); + } catch (Throwable t) { + fail("should not have thrown: " + t); + } + } + + public void testDataFileFindable() throws Exception { + assertTrue("Test file not found", new java.io.File(TEST_DATA_DIR).exists()); + } + + public void testLoadDataFile() throws Exception { + String path = abspath("test-load.txt"); + BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(path))); + assertEquals(in.readLine().trim(), "hello"); + in.close(); + } + + public void testGetSource() throws Exception { + String src = getSource("testsrc.txt"); + assertEquals("one\ntwo\n\nthree\n", src); + } + + public void testStringModule() throws Exception { + idx.loadString("test-string-module.py", makeModule( + "def foo():", + " pass")); + idx.ready(); + assertFunctionBinding("test-string-module.foo"); + } + + public void testNthIndexOf() throws Exception { + String s = "ab a b ab a\nb aab"; + assertEquals(0, nthIndexOf(s, "ab", 1)); + assertEquals(7, nthIndexOf(s, "ab", 2)); + assertEquals(15, nthIndexOf(s, "ab", 3)); + try { + assertEquals(-1, nthIndexOf(s, "ab", 0)); + assertTrue(false); + } catch (IllegalArgumentException ix) { + assertTrue(true); + } + try { + assertEquals(-1, nthIndexOf(s, "ab", 4)); + assertTrue(false); + } catch (IllegalArgumentException ix) { + assertTrue(true); + } + } + + public void testIndexerDefaults() throws Exception { + includeStandardLibrary(); + assertEquals("wrong project dir", TEST_DATA_DIR, idx.projDir); + assertEquals("unexpected load path entries", 1, idx.path.size()); + assertEquals(TEST_LIB_DIR, idx.path.get(0)); + } + + // utilities + + public String buildIndex(String... files) throws Exception { + for (String f : files) { + idx.loadFile(abspath(f)); + } + idx.ready(); + return getSource(files[0]); + } + + public NBinding getBinding(String qname) throws Exception { + NBinding b = idx.lookupQname(qname); + assertNotNull("no binding found for " + qname, b); + return b; + } + + public NBinding assertBinding(String qname, NBinding.Kind kind) throws Exception { + NBinding b = getBinding(qname); + assertEquals(kind, b.getKind()); + return b; + } + + public void assertNoBinding(String qname) throws Exception { + NBinding b = idx.lookupQname(qname); + assertNull("Should not have found binding for " + qname, b); + } + + public NBinding assertAttributeBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.ATTRIBUTE); + } + + public NBinding assertBuiltinBinding(String qname) throws Exception { + NBinding b = getBinding(qname); + assertTrue(b.isBuiltin()); + return b; + } + + public NBinding assertClassBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.CLASS); + } + + public NBinding assertConstructorBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.CONSTRUCTOR); + } + + public NBinding assertFunctionBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.FUNCTION); + } + + public NBinding assertMethodBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.METHOD); + } + + public NBinding assertModuleBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.MODULE); + } + + public NBinding assertScopeBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.SCOPE); + } + + public NBinding assertVariableBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.VARIABLE); + } + + public NBinding assertParamBinding(String qname) throws Exception { + return assertBinding(qname, NBinding.Kind.PARAMETER); + } + + public void assertStaticSynthetic(NBinding b) { + assertTrue(b.isStatic()); + assertTrue(b.isSynthetic()); + } + + public Def getDefinition(String qname, int offset, int length) throws Exception { + NBinding b = getBinding(qname); + assertNotNull(b.getDefs()); + for (Def def : b.getDefs()) { + if (offset == def.start() && length == def.end() - def.start()) { + return def; + } + } + return null; + } + + public void assertDefinition(String qname, int offset, int length) throws Exception { + Def def = getDefinition(qname, offset, length); + if (def == null) { + fail("No definition for " + qname + " at " + offset + " of len " + length); + } + } + + public void assertNoDefinition(String msg, String qname, int pos, int len) throws Exception { + Def def = getDefinition(qname, pos, len); + assertNull(msg, def); + } + + public void assertDefinition(String qname, int offset) throws Exception { + String[] names = qname.split("[.&@]"); + assertDefinition(qname, offset, names[names.length-1].length()); + } + + public void assertDefinition(String qname, String name, int offset) throws Exception { + assertDefinition(qname, offset, name.length()); + } + + public Ref getRefOrNull(String qname, int offset, int length) throws Exception { + NBinding b = getBinding(qname); + assertNotNull("Null refs list for " + qname, b.getRefs()); + for (Ref ref : b.getRefs()) { + if (offset == ref.start() && length == ref.length()) { + return ref; + } + } + return null; + } + + public Ref getRefOrFail(String qname, int offset, int length) throws Exception { + Ref ref = getRefOrNull(qname, offset, length); + assertNotNull("No reference to " + qname + " at offset " + offset + " of length " + length, + ref); + return ref; + } + + public void assertReference(String qname, int offset, int length) throws Exception { + assertTrue(getRefOrFail(qname, offset, length).isRef()); + } + + public void assertReference(String qname, int offset, String refname) throws Exception { + assertReference(qname, offset, refname.length()); + } + + // assume reference to "a.b.c" is called "c" -- the normal case + public void assertReference(String qname, int offset) throws Exception { + String[] names = qname.split("[.&@]"); + assertReference(qname, offset, names[names.length-1]); + } + + public void assertNoReference(String msg, String qname, int pos, int len) throws Exception { + assertNull(msg, getRefOrNull(qname, pos, len)); + } + + public void assertCall(String qname, int offset, int length) throws Exception { + assertTrue(getRefOrFail(qname, offset, length).isCall()); + } + + public void assertCall(String qname, int offset, String refname) throws Exception { + assertCall(qname, offset, refname.length()); + } + + // "a.b.c()" => look for call reference at "c" + public void assertCall(String qname, int offset) throws Exception { + String[] names = qname.split("[.&@]"); + assertCall(qname, offset, names[names.length-1]); + } + + public void assertConstructed(String qname, int offset, int length) throws Exception { + assertTrue(getRefOrFail(qname, offset, length).isNew()); + } + + public void assertConstructed(String qname, int offset, String refname) throws Exception { + assertConstructed(qname, offset, refname.length()); + } + + // "a.b.c()" => look for call reference at "c" + public void assertConstructed(String qname, int offset) throws Exception { + String[] names = qname.split("[.&@]"); + assertConstructed(qname, offset, names[names.length-1]); + } + + public NType getTypeBinding(String typeQname) throws Exception { + NType type = idx.lookupQnameType(typeQname); + assertNotNull("No recorded type for " + typeQname, type); + return type; + } + + // Assert that binding for qname has exactly one type (not a union), + // and that type has a binding with typeQname. + public NBinding assertBindingType(String bindingQname, String typeQname) throws Exception { + NBinding b = getBinding(bindingQname); + NType expected = getTypeBinding(typeQname); + assertEquals("Wrong binding type", expected, NUnknownType.follow(b.getType())); + return b; + } + + public NBinding assertBindingType(String bindingQname, Class type) throws Exception { + NBinding b = getBinding(bindingQname); + NType btype = NUnknownType.follow(b.getType()); + assertTrue("Wrong type: expected " + type + " but was " + btype, + type.isInstance(btype)); + return b; + } + + public void assertListType(String bindingQname) throws Exception { + assertListType(bindingQname, null); + } + + /** + * Asserts that the binding named by {@code bindingQname} exists and + * its type is a List type. If {@code eltTypeQname} is non-{@code null}, + * asserts that the List type's element type is an existing binding with + * {@code eltTypeQname}. + */ + public void assertListType(String bindingQname, String eltTypeQname) throws Exception { + NBinding b = getBinding(bindingQname); + NType btype = b.followType(); + assertTrue(btype.isListType()); + if (eltTypeQname != null) { + NType eltType = getTypeBinding(eltTypeQname); + assertEquals(eltType, NUnknownType.follow(btype.asListType().getElementType())); + } + } + + public void assertStringType(String bindingQname) throws Exception { + assertBindingType(bindingQname, "__builtin__.str"); + } + + public void assertNumType(String bindingQname) throws Exception { + assertBindingType(bindingQname, "__builtin__.float"); + } + + public void assertInstanceType(String bindingQname, String classQname) throws Exception { + if (true) { + assertBindingType(bindingQname, classQname); + return; + } + + // XXX: we've disabled support for NInstanceType for now + NBinding b = getBinding(bindingQname); + NType btype = b.followType(); + assertTrue(btype.isInstanceType()); + NType ctype = getTypeBinding(classQname); + assertEquals(btype.asInstanceType().getClassType(), ctype); + } +} Index: tests/java/org/python/indexer/data/mod2.py =================================================================== --- tests/java/org/python/indexer/data/mod2.py (revision 0) +++ tests/java/org/python/indexer/data/mod2.py (revision 0) @@ -0,0 +1,5 @@ + +import distutils.command + +def mod2test(): + return dir(distutils) Index: tests/java/org/python/indexer/data/mod1.py =================================================================== --- tests/java/org/python/indexer/data/mod1.py (revision 0) +++ tests/java/org/python/indexer/data/mod1.py (revision 0) @@ -0,0 +1,5 @@ + +import distutils + +def mod1test(): + return dir(distutils) Index: src/org/python/indexer/ast/NIf.java =================================================================== --- src/org/python/indexer/ast/NIf.java (revision 0) +++ src/org/python/indexer/ast/NIf.java (revision 0) @@ -0,0 +1,55 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; + +public class NIf extends NNode { + + static final long serialVersionUID = -3744458754599083332L; + + public NNode test; + public NBlock body; + public NBlock orelse; + + public NIf(NNode test, NBlock body, NBlock orelse) { + this(test, body, orelse, 0, 1); + } + + public NIf(NNode test, NBlock body, NBlock orelse, int start, int end) { + super(start, end); + this.test = test; + this.body = body; + this.orelse = orelse; + addChildren(test, body, orelse); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(test, s); + if (body != null) { + setType(resolveExpr(body, s)); + } + if (orelse != null) { + addType(resolveExpr(orelse, s)); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(test, v); + visitNode(body, v); + visitNode(orelse, v); + } + } +} Index: tests/java/org/python/indexer/data/callnewref.py =================================================================== --- tests/java/org/python/indexer/data/callnewref.py (revision 0) +++ tests/java/org/python/indexer/data/callnewref.py (revision 0) @@ -0,0 +1,16 @@ +# Test differentiating call/new/ref + +def myfunc(): + pass + +myfunc # ref to func +myfunc() # call to func + +class MyClass(): + def mymethod(self): + pass + +MyClass # ref to class +a = MyClass() # instantiation +a.mymethod # ref to method +a.mymethod() # call to method Index: src/org/python/indexer/ast/NTuple.java =================================================================== --- src/org/python/indexer/ast/NTuple.java (revision 0) +++ src/org/python/indexer/ast/NTuple.java (revision 0) @@ -0,0 +1,44 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NTupleType; +import org.python.indexer.types.NType; + +import java.util.List; + +public class NTuple extends NSequence { + + static final long serialVersionUID = -7647425038559142921L; + + public NTuple(List elts) { + this(elts, 0, 1); + } + + public NTuple(List elts, int start, int end) { + super(elts, start, end); + } + + @Override + public NType resolve(Scope s) throws Exception { + NTupleType thisType = new NTupleType(); + for (NNode e : elts) { + thisType.add(resolveExpr(e, s)); + } + return setType(thisType); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(elts, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/plant/food/tofu.py =================================================================== Index: src/org/python/indexer/ast/NBoolOp.java =================================================================== --- src/org/python/indexer/ast/NBoolOp.java (revision 0) +++ src/org/python/indexer/ast/NBoolOp.java (revision 0) @@ -0,0 +1,61 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +/** + * Represents the "and"/"or" operators. + */ +public class NBoolOp extends NNode { + + static final long serialVersionUID = -5261954056600388069L; + + public enum OpType { AND, OR, UNDEFINED } + + OpType op; + public List values; + + public NBoolOp(OpType op, List values) { + this(op, values, 0, 1); + } + + public NBoolOp(OpType op, List values, int start, int end) { + super(start, end); + this.op = op; + this.values = values; + addChildren(values); + } + + @Override + public NType resolve(Scope s) throws Exception { + if (op == OpType.AND) { + NType last = null; + for (NNode e : values) { + last = resolveExpr(e, s); + } + return setType(last == null ? new NUnknownType() : last); + } + + // OR + return setType(resolveListAsUnion(values, s)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(values, v); + } + } +} Index: build.xml =================================================================== --- build.xml (revision 6961) +++ build.xml (working copy) @@ -182,6 +182,7 @@ + @@ -883,6 +884,7 @@ + @@ -893,6 +895,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: src/org/python/indexer/IndexingException.java =================================================================== --- src/org/python/indexer/IndexingException.java (revision 0) +++ src/org/python/indexer/IndexingException.java (revision 0) @@ -0,0 +1,26 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +/** + * Signals that indexing is being aborted. + * @see {Indexer#enableAggressiveAssertions} + */ +public class IndexingException extends RuntimeException { + + public IndexingException() { + } + + public IndexingException(String msg) { + super(msg); + } + + public IndexingException(String msg, Throwable cause) { + super(msg, cause); + } + + public IndexingException(Throwable cause) { + super(cause); + } +} Index: tests/java/org/python/indexer/data/test-load.txt =================================================================== --- tests/java/org/python/indexer/data/test-load.txt (revision 0) +++ tests/java/org/python/indexer/data/test-load.txt (revision 0) @@ -0,0 +1 @@ +hello Index: src/org/python/indexer/ast/NSequence.java =================================================================== --- src/org/python/indexer/ast/NSequence.java (revision 0) +++ src/org/python/indexer/ast/NSequence.java (revision 0) @@ -0,0 +1,28 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import java.util.ArrayList; +import java.util.List; + +public abstract class NSequence extends NNode { + + static final long serialVersionUID = 7996591535766850065L; + + public List elts; + + public NSequence(List elts) { + this(elts, 0, 1); + } + + public NSequence(List elts, int start, int end) { + super(start, end); + this.elts = (elts != null) ? elts : new ArrayList(); + addChildren(elts); + } + + public List getElements() { + return elts; + } +} Index: tests/java/org/python/indexer/data/yinw/yinw-12.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-12.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-12.py (revision 0) @@ -0,0 +1,3 @@ +import foo + +foo.f(1) Index: src/org/python/indexer/ast/NPlaceHolder.java =================================================================== --- src/org/python/indexer/ast/NPlaceHolder.java (revision 0) +++ src/org/python/indexer/ast/NPlaceHolder.java (revision 0) @@ -0,0 +1,26 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +public class NPlaceHolder extends NNode { + + static final long serialVersionUID = -8732894605739403419L; + + public NPlaceHolder() { + } + + public NPlaceHolder(int start, int end) { + super(start, end); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: tests/java/org/python/indexer/data/yinw/yinw-6.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-6.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-6.py (revision 0) @@ -0,0 +1,25 @@ +def ok(): + x = 1 + class A: + x = 2 + def f(self,y): + x = 3 + print x,y + def g(me): + print me.x + print me + print x + print im_self + print __class__ + o = A() + o.g() + + +# locations: +# = [] +# = [] +# .x:9:18> = [] +# = [] +# = [] +# = [] +# .g:12:4> = [] Index: src/org/python/antlr/AnalyzingParser.java =================================================================== --- src/org/python/antlr/AnalyzingParser.java (revision 0) +++ src/org/python/antlr/AnalyzingParser.java (revision 0) @@ -0,0 +1,67 @@ +package org.python.antlr; + +import org.antlr.runtime.ANTLRFileStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.Token; +import org.python.antlr.ast.Name; +import org.python.antlr.base.mod; + +import java.util.List; + +/** + * Parser used by the indexer. + */ +public class AnalyzingParser extends BaseParser { + + public static class AnalyzerTreeAdaptor extends PythonTreeAdaptor { + /** + * Make sure a parenthesized {@link Name} expr has its start/stop bounds + * set to the bounds of the identifier. + */ + @Override + public void setTokenBoundaries(Object t, Token startToken, Token stopToken) { + //XXX: should do this for all expr types, and have a prop list on Expr + //that records all enclosing paren locations for IDE use cases. + if (!(t instanceof Name) + || startToken == null + || stopToken == null + || startToken.getType() != PythonParser.LPAREN + || stopToken.getType() != PythonParser.RPAREN) { + super.setTokenBoundaries(t, startToken, stopToken); + } + } + } + + public AnalyzingParser(CharStream stream, String filename, String encoding) { + super(stream, filename, encoding); + errorHandler = new RecordingErrorHandler(); + } + + public List getRecognitionErrors() { + return ((RecordingErrorHandler)errorHandler).errs; + } + + @Override + protected PythonParser setupParser(boolean single) { + PythonParser parser = super.setupParser(single); + parser.setTreeAdaptor(new AnalyzerTreeAdaptor()); + return parser; + } + + public static void main(String[] args) { + CharStream in = null; + try { + in = new ANTLRFileStream(args[0]); + } catch (Exception x) { + x.printStackTrace(); + } + AnalyzingParser p = new AnalyzingParser(in, args[0], "ascii"); + mod ast = p.parseModule(); + if (ast != null) { + System.out.println("parse result: \n" + ast.toStringTree()); + } else { + System.out.println("failure: \n" + p.getRecognitionErrors()); + } + } +} Index: tests/java/org/python/indexer/data/pkg/other/color/green.py =================================================================== --- tests/java/org/python/indexer/data/pkg/other/color/green.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/other/color/green.py (revision 0) @@ -0,0 +1,3 @@ +r = 0 +g = 255 +b = 0 Index: src/org/python/indexer/ast/NTryExcept.java =================================================================== --- src/org/python/indexer/ast/NTryExcept.java (revision 0) +++ src/org/python/indexer/ast/NTryExcept.java (revision 0) @@ -0,0 +1,59 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; + +import java.util.List; + +public class NTryExcept extends NNode { + + static final long serialVersionUID = 7210847998428480831L; + + public List handlers; + public NBlock body; + public NBlock orelse; + + public NTryExcept(List handlers, NBlock body, NBlock orelse) { + this(handlers, body, orelse, 0, 1); + } + + public NTryExcept(List handlers, NBlock body, NBlock orelse, + int start, int end) { + super(start, end); + this.handlers = handlers; + this.body = body; + this.orelse = orelse; + addChildren(handlers); + addChildren(body, orelse); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveList(handlers, s); + if (body != null) { + setType(resolveExpr(body, s)); + } + if (orelse != null) { + addType(resolveExpr(orelse, s)); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(handlers, v); + visitNode(body, v); + visitNode(orelse, v); + } + } +} Index: src/org/python/indexer/ast/GenericNodeVisitor.java =================================================================== --- src/org/python/indexer/ast/GenericNodeVisitor.java (revision 0) +++ src/org/python/indexer/ast/GenericNodeVisitor.java (revision 0) @@ -0,0 +1,237 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +/** + * A visitor that passes every visited node to a single function. + * Subclasses need only implement {@link #dispatch} to receive + * every node as a generic {@link NNode}. + */ +public abstract class GenericNodeVisitor extends DefaultNodeVisitor { + + /** + * Every visited node is passed to this method. The semantics + * for halting traversal are the same as for {@link DefaultNodeVisitor}. + * @return {@code true} to traverse this node's children + */ + public boolean dispatch(NNode n) { + return traverseIntoNodes; + } + + public boolean visit(NAlias n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NAssert n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NAssign n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NAttribute n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NAugAssign n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NBinOp n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NBlock n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NBoolOp n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NBreak n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NCall n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NClassDef n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NCompare n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NComprehension n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NContinue n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NDelete n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NDict n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NEllipsis n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NExceptHandler n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NExec n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NFor n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NFunctionDef n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NGeneratorExp n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NGlobal n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NIf n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NIfExp n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NImport n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NImportFrom n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NIndex n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NKeyword n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NLambda n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NList n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NListComp n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NModule n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NName n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NNum n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NPass n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NPlaceHolder n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NPrint n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NQname n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NRaise n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NRepr n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NReturn n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NExprStmt n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NSlice n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NStr n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NSubscript n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NTryExcept n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NTryFinally n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NTuple n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NUnaryOp n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NUrl n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NWhile n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NWith n) { + return traverseIntoNodes && dispatch(n); + } + + public boolean visit(NYield n) { + return traverseIntoNodes && dispatch(n); + } +} Index: tests/java/org/python/indexer/data/pkg/plant/garden/__init__.py =================================================================== Index: grammar/Python.g =================================================================== --- grammar/Python.g (revision 6820) +++ grammar/Python.g (working copy) @@ -367,7 +367,7 @@ } | { - $etype = new Name($n1, $n1.text, expr_contextType.Load); + $etype = actions.makeNameNode($n1); } ) ; @@ -783,13 +783,16 @@ import_from : FROM (d+=DOT* dotted_name | d+=DOT+) IMPORT (STAR - -> ^(FROM[$FROM, actions.makeFromText($d, $dotted_name.name), + -> ^(FROM[$FROM, actions.makeFromText($d, $dotted_name.names), + actions.makeModuleNameNode($d, $dotted_name.names), actions.makeStarAlias($STAR), actions.makeLevel($d)]) | i1=import_as_names - -> ^(FROM[$FROM, actions.makeFromText($d, $dotted_name.name), + -> ^(FROM[$FROM, actions.makeFromText($d, $dotted_name.names), + actions.makeModuleNameNode($d, $dotted_name.names), actions.makeAliases($i1.atypes), actions.makeLevel($d)]) | LPAREN i2=import_as_names COMMA? RPAREN - -> ^(FROM[$FROM, actions.makeFromText($d, $dotted_name.name), + -> ^(FROM[$FROM, actions.makeFromText($d, $dotted_name.names), + actions.makeModuleNameNode($d, $dotted_name.names), actions.makeAliases($i2.atypes), actions.makeLevel($d)]) ) ; @@ -811,7 +814,7 @@ } : name=NAME (AS asname=NAME)? { - $atype = new alias($name, $name.text, $asname.text); + $atype = new alias(actions.makeNameNode($name), actions.makeNameNode($asname)); } ; @@ -822,10 +825,9 @@ @after { $dotted_as_name.tree = $atype; } - - : dotted_name (AS NAME)? + : dotted_name (AS asname=NAME)? { - $atype = new alias($NAME, $dotted_name.name, $NAME.text); + $atype = new alias($dotted_name.names, actions.makeNameNode($asname)); } ; @@ -840,17 +842,17 @@ //dotted_name: NAME ('.' NAME)* dotted_name - returns [String name] + returns [List names] : NAME (DOT dn+=attr)* { - $name = actions.makeDottedText($NAME, $dn); + $names = actions.makeDottedName($NAME, $dn); } ; //global_stmt: 'global' NAME (',' NAME)* global_stmt : GLOBAL n+=NAME (COMMA n+=NAME)* - -> ^(GLOBAL[$GLOBAL, actions.makeNames($n)]) + -> ^(GLOBAL[$GLOBAL, actions.makeNames($n), actions.makeNameNodes($n)]) ; //exec_stmt: 'exec' expr ['in' test [',' test]] @@ -1522,7 +1524,7 @@ | LBRACK subscriptlist[$begin] RBRACK -> ^(LBRACK[$begin, actions.castExpr($tree), actions.castSlice($subscriptlist.tree), $expr::ctype]) | DOT attr - -> ^(DOT[$begin, actions.castExpr($tree), $attr.text, $expr::ctype]) + -> ^(DOT[$begin, actions.castExpr($tree), new Name($attr.tree, $attr.text, expr_contextType.Load), $expr::ctype]) ; //subscriptlist: subscript (',' subscript)* [','] @@ -1630,7 +1632,7 @@ if ($decorators.start != null) { t = $decorators.start; } - stype = new ClassDef(t, actions.cantBeNone($NAME), + stype = new ClassDef(t, actions.cantBeNoneName($NAME), actions.makeBases(actions.castExpr($testlist.tree)), actions.castStmts($suite.stypes), actions.castExprs($decorators.etypes)); Index: tests/java/org/python/indexer/data/pkg/mineral/stone/__init__.py =================================================================== Index: tests/java/org/python/indexer/data/pkg/plant/food/peach.py =================================================================== Index: tests/java/org/python/indexer/data/yinw/yinw-18.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-18.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-18.py (revision 0) @@ -0,0 +1,5 @@ +def g(x): + return x + 1 + +def f(x, y=g(3), *kw): + return x, y, kw Index: tests/java/org/python/indexer/data/pkg/other/color/white.py =================================================================== --- tests/java/org/python/indexer/data/pkg/other/color/white.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/other/color/white.py (revision 0) @@ -0,0 +1,7 @@ +from red import r as R +from green import g +from pkg.other.color.blue import b as B + +r = R +g # this comment required for unit test +b = B Index: src/org/python/indexer/Diagnostic.java =================================================================== --- src/org/python/indexer/Diagnostic.java (revision 0) +++ src/org/python/indexer/Diagnostic.java (revision 0) @@ -0,0 +1,33 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +public class Diagnostic { + public enum Type { + INFO, WARNING, ERROR + } + + public String file; + public Type type; + public int start; + public int end; + public int line; + public int column; + public String msg; + + public Diagnostic(String file, Type type, int start, int end, String msg) { + this.type = type; + this.file = file; + this.start = start; + this.end = end; + this.msg = msg; + } + + // XXX: support line/column + + @Override + public String toString() { + return ""; + } +} Index: src/org/python/indexer/ast/NFunctionDef.java =================================================================== --- src/org/python/indexer/ast/NFunctionDef.java (revision 0) +++ src/org/python/indexer/ast/NFunctionDef.java (revision 0) @@ -0,0 +1,257 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Builtins; +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NDictType; +import org.python.indexer.types.NFuncType; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NTupleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.ArrayList; +import java.util.List; + +import static org.python.indexer.NBinding.Kind.ATTRIBUTE; +import static org.python.indexer.NBinding.Kind.CLASS; +import static org.python.indexer.NBinding.Kind.CONSTRUCTOR; +import static org.python.indexer.NBinding.Kind.FUNCTION; +import static org.python.indexer.NBinding.Kind.METHOD; +import static org.python.indexer.NBinding.Kind.PARAMETER; + +public class NFunctionDef extends NNode { + + static final long serialVersionUID = 5495886181960463846L; + + public NName name; + public List args; + public List defaults; + public NName varargs; // *args + public NName kwargs; // **kwargs + public NNode body; + private List decoratorList; + + public NFunctionDef(NName name, List args, NBlock body, List defaults, + NName varargs, NName kwargs) { + this(name, args, body, defaults, kwargs, varargs, 0, 1); + } + + public NFunctionDef(NName name, List args, NBlock body, List defaults, + NName varargs, NName kwargs, int start, int end) { + super(start, end); + this.name = name; + this.args = args; + this.body = body != null ? new NBody(body) : new NBlock(null); + this.defaults = defaults; + this.varargs = varargs; + this.kwargs = kwargs; + addChildren(name); + addChildren(args); + addChildren(defaults); + addChildren(varargs, kwargs, this.body); + } + + public void setDecoratorList(List decoratorList) { + this.decoratorList = decoratorList; + addChildren(decoratorList); + } + + public List getDecoratorList() { + if (decoratorList == null) { + decoratorList = new ArrayList(); + } + return decoratorList; + } + + @Override + public boolean isFunctionDef() { + return true; + } + + @Override + public boolean bindsName() { + return true; + } + + /** + * Returns the name of the function for indexing/qname purposes. + * Lambdas will return a generated name. + */ + protected String getBindingName(Scope s) { + return name.id; + } + + @Override + protected void bindNames(Scope s) throws Exception { + Scope owner = s.getScopeSymtab(); // enclosing class, function or module + + setType(new NFuncType()); + Scope funcTable = new Scope(s.getEnclosingLexicalScope(), Scope.Type.FUNCTION); + getType().setTable(funcTable); + funcTable.setPath(owner.extendPath(getBindingName(owner))); + + // If we already defined this function in this scope, don't try it again. + NType existing = owner.lookupType(getBindingName(owner), true /* local scope */); + if (existing != null && existing.isFuncType()) { + return; + } + + bindFunctionName(owner); + bindFunctionParams(funcTable); + bindFunctionDefaults(s); + bindMethodAttrs(owner); + } + + protected void bindFunctionName(Scope owner) throws Exception { + NBinding.Kind funkind = FUNCTION; + if (owner.getScopeType() == Scope.Type.CLASS) { + if ("__init__".equals(name.id)) { + funkind = CONSTRUCTOR; + } else { + funkind = METHOD; + } + } + NameBinder.make(funkind).bindName(owner, name, getType()); + } + + protected void bindFunctionParams(Scope funcTable) throws Exception { + NameBinder param = NameBinder.make(PARAMETER); + for (NNode a : args) { + param.bind(funcTable, a, new NUnknownType()); + } + if (varargs != null) { + param.bind(funcTable, varargs, new NListType()); + } + if (kwargs != null) { + param.bind(funcTable, kwargs, new NDictType()); + } + } + + /** + * Processes any name-binding constructs appearing as parameter defaults. + * For instance, in {@code def foo(converter=lambda name: name.upper()): ...} + * the lambda is a name-binding construct. + */ + protected void bindFunctionDefaults(Scope s) throws Exception { + for (NNode n : defaults) { + if (n.bindsName()) { + n.bindNames(s); + } + } + } + + protected void bindMethodAttrs(Scope owner) throws Exception { + NType cls = Indexer.idx.lookupQnameType(owner.getPath()); + if (cls == null || !cls.isClassType()) { + return; + } + // We don't currently differentiate between classes and instances. + addReadOnlyAttr("im_class", cls, CLASS); + addReadOnlyAttr("__class__", cls, CLASS); + addReadOnlyAttr("im_self", cls, ATTRIBUTE); + addReadOnlyAttr("__self__", cls, ATTRIBUTE); + } + + protected NBinding addSpecialAttr(String name, NType atype, NBinding.Kind kind) { + NBinding b = getTable().update(name, + Builtins.newDataModelUrl("the-standard-type-hierarchy"), + atype, kind); + b.markSynthetic(); + b.markStatic(); + return b; + } + + protected NBinding addReadOnlyAttr(String name, NType type, NBinding.Kind kind) { + NBinding b = addSpecialAttr(name, type, kind); + b.markReadOnly(); + return b; + } + + @Override + public NType resolve(Scope outer) throws Exception { + resolveList(defaults, outer); + resolveList(decoratorList, outer); + + Scope funcTable = getTable(); + NBinding selfBinding = funcTable.lookup("__self__"); + if (selfBinding != null && !selfBinding.getType().isClassType()) { + selfBinding = null; + } + + if (selfBinding != null) { + if (args.size() < 1) { + addWarning(name, "method should have at least one argument (self)"); + } else if (!(args.get(0) instanceof NName)) { + addError(name, "self parameter must be an identifier"); + } + } + + NTupleType fromType = new NTupleType(); + bindParamsToDefaults(selfBinding, fromType); + + if (varargs != null) { + NBinding b = funcTable.lookupLocal(varargs.id); + if (b != null) { + fromType.add(b.getType()); + } + } + + if (kwargs != null) { + NBinding b = funcTable.lookupLocal(kwargs.id); + if (b != null) { + fromType.add(b.getType()); + } + } + + NType toType = resolveExpr(body, funcTable); + getType().asFuncType().setReturnType(toType); + return getType(); + } + + private void bindParamsToDefaults(NBinding selfBinding, NTupleType fromType) throws Exception { + NameBinder param = NameBinder.make(PARAMETER); + Scope funcTable = getTable(); + + for (int i = 0; i < args.size(); i++) { + NNode arg = args.get(i); + NType argtype = ((i == 0 && selfBinding != null) + ? selfBinding.getType() + : getArgType(args, defaults, i)); + param.bind(funcTable, arg, argtype); + fromType.add(argtype); + } + } + + static NType getArgType(List args, List defaults, int argnum) { + if (defaults == null) { + return new NUnknownType(); + } + int firstDefault = args.size() - defaults.size(); + if (firstDefault >= 0 && argnum >= firstDefault) { + return defaults.get(argnum - firstDefault).getType(); + } + return new NUnknownType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(name, v); + visitNodeList(args, v); + visitNodeList(defaults, v); + visitNode(kwargs, v); + visitNode(varargs, v); + visitNode(body, v); + } + } +} Index: src/org/python/indexer/ast/NExec.java =================================================================== --- src/org/python/indexer/ast/NExec.java (revision 0) +++ src/org/python/indexer/ast/NExec.java (revision 0) @@ -0,0 +1,50 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NExec extends NNode { + + static final long serialVersionUID = -1840017898177850339L; + + public NNode body; + public NNode globals; + public NNode locals; + + public NExec(NNode body, NNode globals, NNode locals) { + this(body, globals, locals, 0, 1); + } + + public NExec(NNode body, NNode globals, NNode locals, int start, int end) { + super(start, end); + this.body = body; + this.globals = globals; + this.locals = locals; + addChildren(body, globals, locals); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(body, s); + resolveExpr(globals, s); + resolveExpr(locals, s); + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(body, v); + visitNode(globals, v); + visitNode(locals, v); + } + } +} Index: src/org/python/indexer/ast/NClassDef.java =================================================================== --- src/org/python/indexer/ast/NClassDef.java (revision 0) +++ src/org/python/indexer/ast/NClassDef.java (revision 0) @@ -0,0 +1,108 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Builtins; +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NClassType; +import org.python.indexer.types.NDictType; +import org.python.indexer.types.NTupleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.ArrayList; +import java.util.List; + +public class NClassDef extends NNode { + + static final long serialVersionUID = 7513873538009667540L; + + public NName name; + public List bases; + public NBody body; + + public NClassDef(NName name, List bases, NBlock body) { + this(name, bases, body, 0, 1); + } + + public NClassDef(NName name, List bases, NBlock body, int start, int end) { + super(start, end); + this.name = name; + this.bases = bases; + this.body = new NBody(body); + addChildren(name, this.body); + addChildren(bases); + } + + @Override + public boolean isClassDef() { + return true; + } + + @Override + public boolean bindsName() { + return true; + } + + @Override + protected void bindNames(Scope s) throws Exception { + Scope container = s.getScopeSymtab(); + setType(new NClassType(name.id, container)); + + // If we already defined this class in this scope, don't redefine it. + NType existing = container.lookupType(name.id); + if (existing != null && existing.isClassType()) { + return; + } + + NameBinder.make(NBinding.Kind.CLASS).bind(container, name, getType()); + } + + @Override + public NType resolve(Scope s) throws Exception { + NClassType thisType = getType().asClassType(); + List baseTypes = new ArrayList(); + for (NNode base : bases) { + NType baseType = resolveExpr(base, s); + if (baseType.isClassType()) { + thisType.addSuper(baseType); + } + baseTypes.add(baseType); + } + + Builtins builtins = Indexer.idx.builtins; + addSpecialAttribute("__bases__", new NTupleType(baseTypes)); + addSpecialAttribute("__name__", builtins.BaseStr); + addSpecialAttribute("__module__", builtins.BaseStr); + addSpecialAttribute("__doc__", builtins.BaseStr); + addSpecialAttribute("__dict__", new NDictType(builtins.BaseStr, new NUnknownType())); + + resolveExpr(body, getTable()); + return getType(); + } + + private void addSpecialAttribute(String name, NType proptype) { + NBinding b = getTable().update(name, Builtins.newTutUrl("classes.html"), + proptype, NBinding.Kind.ATTRIBUTE); + b.markSynthetic(); + b.markStatic(); + b.markReadOnly(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(name, v); + visitNodeList(bases, v); + visitNode(body, v); + } + } +} Index: src/org/python/indexer/demos/Linker.java =================================================================== --- src/org/python/indexer/demos/Linker.java (revision 0) +++ src/org/python/indexer/demos/Linker.java (revision 0) @@ -0,0 +1,192 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.demos; + +import org.python.indexer.Def; +import org.python.indexer.Ref; +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.StyleRun; +import org.python.indexer.Util; +import org.python.indexer.types.NModuleType; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Collects per-file hyperlinks, as well as styles that require the + * symbol table to resolve properly. + */ +class Linker { + + private static final Pattern CONSTANT = Pattern.compile("[A-Z_][A-Z0-9_]*"); + + // Map of file-path to semantic styles & links for that path. + private Map> fileStyles = new HashMap>(); + + private File rootDir; // root of user-specified directory to index + private File outDir; // where we're generating the output html + + /** + * Constructor. + * @param root the root of the directory tree being indexed + * @param outdir the html output directory + */ + public Linker(File root, File outdir) { + rootDir = root; + outDir = outdir; + } + + /** + * Process all bindings across all files and record per-file semantic styles. + * Should be called once per index. + */ + public void findLinks(Indexer indexer) { + for (NBinding nb : indexer.getBindings().values()) { + addSemanticStyles(nb); + processDefs(nb); + processRefs(nb); + } + } + + /** + * Returns the styles (links and extra styles) generated for a given file. + * @param path an absolute source path + * @return a possibly-empty list of styles for that path + */ + public List getStyles(String path) { + return stylesForFile(path); + } + + private List stylesForFile(String path) { + List styles = fileStyles.get(path); + if (styles == null) { + styles = new ArrayList(); + fileStyles.put(path, styles); + } + return styles; + } + + private void addFileStyle(String path, StyleRun style) { + stylesForFile(path).add(style); + } + + /** + * Add additional highlighting styles based on information not evident from + * the AST. + * @param def the binding's main definition node + * @param b the binding + */ + private void addSemanticStyles(NBinding nb) { + Def def = nb.getSignatureNode(); + if (def == null || !def.isName()) { + return; + } + + boolean isConst = CONSTANT.matcher(def.getName()).matches(); + switch (nb.getKind()) { + case SCOPE: + if (isConst) { + addSemanticStyle(def, StyleRun.Type.CONSTANT); + } + break; + case VARIABLE: + addSemanticStyle(def, isConst ? StyleRun.Type.CONSTANT : StyleRun.Type.IDENTIFIER); + break; + case PARAMETER: + addSemanticStyle(def, StyleRun.Type.PARAMETER); + break; + case CLASS: + addSemanticStyle(def, StyleRun.Type.TYPE_NAME); + break; + } + } + + private void addSemanticStyle(Def def, StyleRun.Type type) { + String path = def.getFile(); + if (path != null) { + addFileStyle(path, new StyleRun(type, def.start(), def.length())); + } + } + + /** + * Create name anchors for definition sites. + */ + private void processDefs(NBinding nb) { + Def def = nb.getSignatureNode(); + if (def == null || def.isURL()) { + return; + } + StyleRun style = new StyleRun(StyleRun.Type.ANCHOR, def.start(), def.length()); + style.message = nb.getQname(); + style.url = nb.getQname(); + addFileStyle(def.getFile(), style); + } + + /** + * Collect cross-reference links for every file. + */ + private void processRefs(NBinding nb) { + if (nb.hasRefs()) { // avoid lazy ref-list instantiation + for (Ref ref : nb.getRefs()) { + processRef(ref, nb); + } + } + } + + /** + * Adds a hyperlink for a single reference. + */ + void processRef(Ref ref, NBinding nb) { + String path = ref.getFile(); + StyleRun link = new StyleRun(StyleRun.Type.LINK, ref.start(), ref.length()); + link.message = nb.getQname(); + link.url = toURL(nb, path); + addFileStyle(path, link); + } + + /** + * Generate a URL for a reference to a binding. + * @param nb the referenced binding + * @param path the path containing the reference + */ + private String toURL(NBinding nb, String path) { + Def def = nb.getSignatureNode(); + if (nb.isBuiltin()) { + return def.getURL(); + } + + if (def.isModule()) { + return toModuleUrl(nb); + } + + String anchor = "#" + nb.getQname(); + if (nb.getFirstFile().equals(path)) { + return anchor; + } + + String destPath = def.getFile(); + String relpath = destPath.substring(rootDir.getAbsolutePath().length()); + return Util.joinPath(outDir.getAbsolutePath(), relpath) + ".html" + anchor; + } + + /** + * Generate an anchorless URL linking to another file in the index. + */ + private String toModuleUrl(NBinding nb) { + NModuleType mtype = nb.getType().follow().asModuleType(); + String path = mtype.getFile(); + String root = rootDir.getAbsolutePath(); + if (!path.startsWith(root)) { + return "file://" + path; // can't find file => punt & load it directly + } + String relpath = path.substring(rootDir.getAbsolutePath().length()); + return Util.joinPath(outDir.getAbsolutePath(), relpath) + ".html"; + } +} Index: src/org/python/indexer/demos/HtmlOutline.java =================================================================== --- src/org/python/indexer/demos/HtmlOutline.java (revision 0) +++ src/org/python/indexer/demos/HtmlOutline.java (revision 0) @@ -0,0 +1,91 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.demos; + +import org.python.indexer.Indexer; +import org.python.indexer.Outliner; + +import java.util.List; + +/** + * Generates a static-html outline for a file. + */ +class HtmlOutline { + + private Indexer indexer; + private StringBuilder buffer; + + public HtmlOutline(Indexer idx) { + this.indexer = idx; + } + + /** + * Generates HTML outline for {@code path}. + * @return the html as an {@code UL} element + */ + public String generate(String path) throws Exception { + buffer = new StringBuilder(1024); + List entries = indexer.generateOutline(path); + addOutline(entries); + String html = buffer.toString(); + buffer = null; + return html; + } + + private void addOutline(List entries) { + add("
    \n"); + for (Outliner.Entry e : entries) { + addEntry(e); + } + add("
\n"); + } + + private void addEntry(Outliner.Entry e) { + add("
  • "); + + String style = null; + switch (e.getKind()) { + case FUNCTION: + case METHOD: + case CONSTRUCTOR: + style = "function"; + break; + case CLASS: + style = "type-name"; + break; + case PARAMETER: + style = "parameter"; + break; + case VARIABLE: + case SCOPE: + style = "identifier"; + break; + } + + add(""); + + if (style != null) { + add(""); + } + add(e.getName()); + if (style != null) { + add(""); + } + + add(""); + + if (e.isBranch()) { + addOutline(e.getChildren()); + } + add("
  • "); + } + + private void add(String text) { + buffer.append(text); + } +} Index: tests/java/org/python/indexer/IndexerTest.java =================================================================== --- tests/java/org/python/indexer/IndexerTest.java (revision 0) +++ tests/java/org/python/indexer/IndexerTest.java (revision 0) @@ -0,0 +1,1188 @@ +package org.python.indexer; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.ast.NAlias; +import org.python.indexer.ast.NAssert; +import org.python.indexer.ast.NAssign; +import org.python.indexer.ast.NAttribute; +import org.python.indexer.ast.NAugAssign; +import org.python.indexer.ast.NBinOp; +import org.python.indexer.ast.NBlock; +import org.python.indexer.ast.NBody; +import org.python.indexer.ast.NBoolOp; +import org.python.indexer.ast.NBreak; +import org.python.indexer.ast.NCall; +import org.python.indexer.ast.NClassDef; +import org.python.indexer.ast.NCompare; +import org.python.indexer.ast.NComprehension; +import org.python.indexer.ast.NContinue; +import org.python.indexer.ast.NDelete; +import org.python.indexer.ast.NDict; +import org.python.indexer.ast.NEllipsis; +import org.python.indexer.ast.NExceptHandler; +import org.python.indexer.ast.NExec; +import org.python.indexer.ast.NExprStmt; +import org.python.indexer.ast.NFor; +import org.python.indexer.ast.NFunctionDef; +import org.python.indexer.ast.NGeneratorExp; +import org.python.indexer.ast.NGlobal; +import org.python.indexer.ast.NIf; +import org.python.indexer.ast.NIfExp; +import org.python.indexer.ast.NImport; +import org.python.indexer.ast.NImportFrom; +import org.python.indexer.ast.NIndex; +import org.python.indexer.ast.NKeyword; +import org.python.indexer.ast.NLambda; +import org.python.indexer.ast.NList; +import org.python.indexer.ast.NListComp; +import org.python.indexer.ast.NModule; +import org.python.indexer.ast.NName; +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NNodeVisitor; +import org.python.indexer.ast.NNum; +import org.python.indexer.ast.NPass; +import org.python.indexer.ast.NPlaceHolder; +import org.python.indexer.ast.NPrint; +import org.python.indexer.ast.NQname; +import org.python.indexer.ast.NRaise; +import org.python.indexer.ast.NRepr; +import org.python.indexer.ast.NReturn; +import org.python.indexer.ast.NSlice; +import org.python.indexer.ast.NStr; +import org.python.indexer.ast.NSubscript; +import org.python.indexer.ast.NTryExcept; +import org.python.indexer.ast.NTryFinally; +import org.python.indexer.ast.NTuple; +import org.python.indexer.ast.NUnaryOp; +import org.python.indexer.ast.NUrl; +import org.python.indexer.ast.NWhile; +import org.python.indexer.ast.NWith; +import org.python.indexer.ast.NYield; +import org.python.indexer.types.NDictType; +import org.python.indexer.types.NFuncType; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NTupleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +import java.io.File; +import java.util.List; +import java.util.Set; + +public class IndexerTest extends TestBase { + + public void testBuiltinModulePresent() throws Exception { + NType mod = idx.moduleTable.lookupType("__builtin__"); + assertNotNull("missing __builtin__ module", mod); + assertTrue("wrong type: " + mod.getClass(), mod instanceof NModuleType); + } + + public void testLazyModuleLoad() throws Exception { + assertNull("'array' module should not yet be loaded", + idx.moduleTable.lookupType("array")); + assertNoBinding("array"); + + assertNotNull(idx.loadModule("array")); // lazy loads it + + assertNotNull("'array' module should have been loaded", + idx.moduleTable.lookupType("array")); + assertModuleBinding("array"); + } + + public void testNativeModulesAvailable() throws Exception { + for (String name : new String[] { + "array", "ctypes", "errno", + "math", "operator", "os", + "signal", "sys", "thread", "time",}) { + assertNoBinding(name); + assertNotNull(name, idx.loadModule(name)); + assertModuleBinding(name); + } + } + + public void testBuiltinObject() throws Exception { + assertClassBinding("__builtin__.object"); + assertClassBinding("__builtin__.object.__class__"); + } + + public void testBuiltinTuple() throws Exception { + assertClassBinding("__builtin__.tuple"); + assertMethodBinding("__builtin__.tuple.__rmul__"); + assertMethodBinding("__builtin__.tuple.__iter__"); + } + + public void testBuiltinList() throws Exception { + assertClassBinding("__builtin__.list"); + assertMethodBinding("__builtin__.list.append"); + assertMethodBinding("__builtin__.list.count"); + } + + public void testBuiltinNum() throws Exception { + assertClassBinding("__builtin__.float"); + NBinding b = assertMethodBinding("__builtin__.float.fromhex"); + assertTrue(b.isBuiltin()); + } + + public void testBuiltinStr() throws Exception { + assertClassBinding("__builtin__.str"); + assertMethodBinding("__builtin__.str.encode"); + assertMethodBinding("__builtin__.str.startswith"); + assertMethodBinding("__builtin__.str.split"); + assertMethodBinding("__builtin__.str.partition"); + } + + public void testBuiltinDict() throws Exception { + assertClassBinding("__builtin__.dict"); + assertMethodBinding("__builtin__.dict.__getitem__"); + assertMethodBinding("__builtin__.dict.keys"); + assertMethodBinding("__builtin__.dict.clear"); + } + + public void testBuiltinFile() throws Exception { + assertClassBinding("__builtin__.file"); + assertMethodBinding("__builtin__.file.__enter__"); + assertMethodBinding("__builtin__.file.readline"); + assertMethodBinding("__builtin__.file.readlines"); + assertMethodBinding("__builtin__.file.isatty"); + } + + public void testBuiltinFuncs() throws Exception { + assertFunctionBinding("__builtin__.apply"); + assertFunctionBinding("__builtin__.abs"); + assertFunctionBinding("__builtin__.hex"); + assertFunctionBinding("__builtin__.range"); + assertFunctionBinding("__builtin__.globals"); + assertFunctionBinding("__builtin__.open"); + } + + public void testBuiltinTypes() throws Exception { + assertClassBinding("__builtin__.ArithmeticError"); + assertClassBinding("__builtin__.ZeroDivisionError"); + assertAttributeBinding("__builtin__.True"); + assertAttributeBinding("__builtin__.False"); + assertAttributeBinding("__builtin__.None"); + assertAttributeBinding("__builtin__.Ellipsis"); + } + + public void testStrConstructor() throws Exception { + String src = index( + "newstr.py", + "x = str([])"); + assertStringType("newstr.x"); + } + + public void testListSubscript() throws Exception { + String src = index( + "test.py", + "x = [1, 2, 3]", + "y = x[2]"); + assertNumType("test.y"); + } + + public void testBuiltinSys() throws Exception { + idx.loadModule("sys"); + assertModuleBinding("sys"); + assertAttributeBinding("sys.__stderr__"); + NBinding b = assertFunctionBinding("sys.exit"); + assertTrue(b.isBuiltin()); + assertFunctionBinding("sys.getprofile"); + assertFunctionBinding("sys.getdefaultencoding"); + assertAttributeBinding("sys.api_version"); + assertNumType("sys.api_version"); + assertAttributeBinding("sys.argv"); + assertBindingType("sys.argv", NListType.class); + assertAttributeBinding("sys.byteorder"); + assertStringType("sys.byteorder"); + assertAttributeBinding("sys.flags"); + assertBindingType("sys.flags", NDictType.class); + } + + public void testFetchAst() throws Exception { + NModule ast = idx.getAstForFile(abspath("hello.py")); + assertNotNull("failed to load file", ast); + assertEquals("module has wrong name", "hello", ast.name); + assertNotNull("AST has no body", ast.body); + assertNotNull("AST body has no children", ast.body.seq); + assertEquals("wrong number of children", 1, ast.body.seq.size()); + NNode e = ast.body.seq.get(0); + assertTrue("Incorrect AST: " + e.getClass(), e instanceof NExprStmt); + e = ((NExprStmt)e).value; + assertTrue("Incorrect AST: " + e.getClass(), e instanceof NStr); + assertEquals("Wrong string content", "Hello", ((NStr)e).n.toString()); + } + + public void testFileLoad() throws Exception { + idx.loadFile(abspath("testfileload.py"), /*skipParentChain=*/true); + idx.ready(); + assertEquals("loaded more than 1 file", 1, idx.numFilesLoaded()); + } + + public void testAstCacheTmpDir() throws Exception { + AstCache cache = AstCache.get(); + File f = new File(AstCache.CACHE_DIR); + assertTrue(f.exists()); + assertTrue(f.canRead()); + assertTrue(f.canWrite()); + assertTrue(f.isDirectory()); + } + + public void testAstCacheNames() throws Exception { + AstCache cache = AstCache.get(); + String sourcePath = abspath("hello.py"); + String cachePath = cache.getCachePath(new File(sourcePath)); + String cachedName = new File(cachePath).getName(); + assertTrue("Invalid cache name: " + cachedName, + cachedName.matches("^hello.py[A-Za-z0-9]{32}.ast$")); + } + + public void testAstCache() throws Exception { + AstCache cache = AstCache.get(); + String sourcePath = abspath("hello.py"); + + // ensure not cached on disk + NModule ast = cache.getSerializedModule(sourcePath); + assertNull(ast); + + cache.getAST(sourcePath); + + // ensure cached on disk + ast = cache.getSerializedModule(sourcePath); + assertNotNull(ast); + + assertEquals(sourcePath, ast.getFile()); + } + + public void testAstCacheEmptyFile() throws Exception { + AstCache cache = AstCache.get(); + NModule mod = cache.getAST(abspath("empty_file.py")); + assertNotNull(mod); + NBlock seq = mod.body; + assertNotNull(seq); + assertTrue(seq.seq.isEmpty()); + } + + // Make sure node types all have NType None when constructed, + // to ensure that no nodes are relying on a particular type when being + // resolved (since deserialization won't call the constructor). + public void testConstructedTypes() throws Exception { + assertNoneType(new NAlias(null, null, null)); + assertNoneType(new NAssert(null, null)); + assertNoneType(new NAssign(null, null)); + assertNoneType(new NAttribute(new NStr(), new NName(""))); + assertNoneType(new NAugAssign(null, null, null)); + assertNoneType(new NBinOp(null, null, null)); + assertNoneType(new NBlock(null)); + assertNoneType(new NBody((List)null)); + assertNoneType(new NBoolOp(null, null)); + assertNoneType(new NBreak()); + assertNoneType(new NCall(null, null, null, null, null)); + assertNoneType(new NClassDef(null, null, null)); + assertNoneType(new NCompare(null, null, null)); + assertNoneType(new NComprehension(null, null, null)); + assertNoneType(new NContinue()); + assertNoneType(new NDelete(null)); + assertNoneType(new NDict(null, null)); + assertNoneType(new NEllipsis()); + assertNoneType(new NExceptHandler(null, null, null)); + assertNoneType(new NExec(null, null, null)); + assertNoneType(new NExprStmt(null)); + assertNoneType(new NFor(null, null, null, null)); + assertNoneType(new NFunctionDef(null, null, null, null, null, null)); + assertNoneType(new NGeneratorExp(null, null)); + assertNoneType(new NGlobal(null)); + assertNoneType(new NIf(null, null, null)); + assertNoneType(new NIfExp(null, null, null)); + assertNoneType(new NImport(null)); + assertNoneType(new NImportFrom(null, null, null)); + assertNoneType(new NIndex(null)); + assertNoneType(new NKeyword(null, null)); + assertNoneType(new NLambda(null, null, null, null, null)); + assertNoneType(new NList(null)); + assertNoneType(new NListComp(null, null)); + assertNoneType(new NModule(null, 0, 1)); + assertNoneType(new NName("")); + assertNoneType(new NNum(-1)); + assertNoneType(new NPass()); + assertNoneType(new NPlaceHolder()); + assertNoneType(new NPrint(null, null)); + assertNoneType(new NQname(null, new NName(""))); + assertNoneType(new NRaise(null, null, null)); + assertNoneType(new NRepr(null)); + assertNoneType(new NReturn(null)); + assertNoneType(new NSlice(null, null, null)); + assertNoneType(new NStr()); + assertNoneType(new NSubscript(null, null)); + assertNoneType(new NTryExcept(null, null, null)); + assertNoneType(new NTryFinally(null, null)); + assertNoneType(new NTuple(null)); + assertNoneType(new NUnaryOp(null, null)); + assertNoneType(new NUrl("")); + assertNoneType(new NWhile(null, null, null)); + assertNoneType(new NWith(null, null, null)); + assertNoneType(new NYield(null)); + } + + private void assertNoneType(NNode n) { + assertEquals(n.getType(), Indexer.idx.builtins.None); + } + + public void testClassTypeBuiltinAttrs() throws Exception { + String file = "classtype_builtins.py"; + buildIndex(file); + NModuleType module = (NModuleType)idx.moduleTable.lookupType(abspath(file)); + Scope mtable = module.getTable(); + assertTrue(mtable.lookupType("MyClass").isClassType()); + assertTrue(mtable.lookupType("MyClassNoDoc").isClassType()); + assertTrue(mtable.lookupType("MyClass").getTable().getParent() == mtable); + assertEquals(NBinding.Kind.CLASS, mtable.lookup("MyClass").getKind()); + Scope t = mtable.lookupType("MyClass").getTable(); + assertTrue(t.lookupType("__bases__").isTupleType()); + assertTrue(t.lookupType("__dict__").isDictType()); + assertEquals(idx.builtins.BaseStr, t.lookupType("__name__")); + assertEquals(idx.builtins.BaseStr, t.lookupType("__module__")); + assertEquals(idx.builtins.BaseStr, t.lookupType("__doc__")); + t = mtable.lookupType("MyClassNoDoc").getTable(); + assertEquals(idx.builtins.BaseStr, t.lookupType("__doc__")); + } + + public void testMethodBuiltinAttrs() throws Exception { + String file = "classtype_builtins.py"; + buildIndex(file); + + Scope mtable = idx.moduleTable.lookupType(abspath(file)).getTable(); + NBinding method = mtable.lookupType("MyClass").getTable().lookup("__init__"); + assertNotNull(method); + assertEquals(NBinding.Kind.CONSTRUCTOR, method.getKind()); + assertEquals("classtype_builtins.MyClass.__init__", method.getQname()); + + NType ftype = mtable.lookupType("MyClass").getTable().lookupType("__init__"); + assertTrue(ftype.isFuncType()); + + NBinding c = mtable.lookup("MyClass"); + for (String special : new String[]{"im_class", "__class__", "im_self", "__self__"}) { + NBinding attr = ftype.getTable().lookup(special); + assertNotNull("missing binding for " + special, attr); + assertEquals(c.getType(), attr.getType()); + } + } + + public void testModulePaths() throws Exception { + idx.loadModule("pkg"); + idx.loadModule("pkg.animal"); + idx.loadModule("pkg.mineral.stone.lapis"); + idx.ready(); + + assertModuleBinding("pkg"); + assertModuleBinding("pkg.animal"); + assertModuleBinding("pkg.mineral.stone.lapis"); + } + + public void testCircularImport() throws Exception { + idx.loadModule("pkg.animal.mammal.cat"); + idx.ready(); + // XXX: finish me + } + + public void testBasicDefsAndRefs() throws Exception { + idx.loadModule("refs"); + idx.ready(); + assertScopeBinding("refs.foo"); + String src = getSource("refs.py"); + assertDefinition("refs.foo", "foo", nthIndexOf(src, "foo", 1)); + + assertNoReference("Definition site should not produce a reference", + "refs.foo", nthIndexOf(src, "foo", 1), "foo".length()); + + assertReference("refs.foo", nthIndexOf(src, "foo", 2)); + assertReference("refs.foo", nthIndexOf(src, "foo", 3)); + assertReference("refs.foo", nthIndexOf(src, "foo", 4)); + assertReference("refs.foo", nthIndexOf(src, "foo", 5)); + + assertNoReference("Should not have been a reference inside a string", + "refs.foo", nthIndexOf(src, "foo", 6), "foo".length()); + + assertReference("refs.foo", nthIndexOf(src, "foo", 7)); + assertReference("refs.foo", nthIndexOf(src, "foo", 8)); + assertReference("refs.foo", nthIndexOf(src, "foo", 9)); + assertReference("refs.foo", nthIndexOf(src, "foo", 10)); + assertReference("refs.foo", nthIndexOf(src, "foo", 11)); + assertReference("refs.foo", nthIndexOf(src, "foo", 12)); + + assertNoReference("Function param cannot refer to outer scope", + "refs.foo", nthIndexOf(src, "foo", 13), "foo".length()); + + assertNoReference("Function param 'foo' should hide outer foo", + "refs.foo", nthIndexOf(src, "foo", 14), "foo".length()); + + assertReference("refs.foo", nthIndexOf(src, "foo", 15)); + assertReference("refs.foo", nthIndexOf(src, "foo", 16)); + } + + public void testAutoClassBindings() throws Exception { + idx.loadModule("class1"); + idx.ready(); + assertModuleBinding("class1"); + assertClassBinding("class1.A"); + + NBinding b = assertAttributeBinding("class1.A.__bases__"); + assertStaticSynthetic(b); + assertTrue(b.getType().isTupleType()); + assertTrue(((NTupleType)b.getType()).getElementTypes().isEmpty()); + + b = assertAttributeBinding("class1.A.__name__"); + assertStaticSynthetic(b); + assertEquals(b.getType(), idx.builtins.BaseStr); + + b = assertAttributeBinding("class1.A.__module__"); + assertStaticSynthetic(b); + assertEquals(b.getType(), idx.builtins.BaseStr); + + b = assertAttributeBinding("class1.A.__doc__"); + assertStaticSynthetic(b); + assertEquals(b.getType(), idx.builtins.BaseStr); + + b = assertAttributeBinding("class1.A.__dict__"); + assertStaticSynthetic(b); + assertTrue(b.getType().isDictType()); + assertEquals(((NDictType)b.getType()).getKeyType(), idx.builtins.BaseStr); + assertTrue(((NDictType)b.getType()).getValueType().isUnknownType()); + } + + public void testLocalVarRef() throws Exception { + idx.loadModule("class2"); + idx.ready(); + assertFunctionBinding("class2.hi"); + assertParamBinding("class2.hi@msg"); + String src = getSource("class2.py"); + assertReference("class2.hi@msg", nthIndexOf(src, "msg", 2)); + } + + public void testClassMemberBindings() throws Exception { + idx.loadModule("class1"); + idx.ready(); + assertScopeBinding("class1.A.a"); + assertConstructorBinding("class1.A.__init__"); + assertMethodBinding("class1.A.hi"); + assertParamBinding("class1.A.__init__@self"); + assertParamBinding("class1.A.hi@self"); + assertParamBinding("class1.A.hi@msg"); + + String src = getSource("class1.py"); + assertReference("class1.A.hi@msg", nthIndexOf(src, "msg", 2)); + assertReference("class1.A", src.indexOf("A.a"), 1); + assertReference("class1.A.a", src.indexOf("A.a") + 2, 1); + assertScopeBinding("class1.x"); + assertScopeBinding("class1.y"); + assertScopeBinding("class1.z"); + assertReference("class1.A", src.indexOf("= A") + 2, 1); + assertConstructed("class1.A", src.indexOf("A()"), 1); + assertReference("class1.y", src.indexOf("y.b"), 1); + + assertInstanceType("class1.y", "class1.A"); + assertReference("class1.A.b", src.indexOf("y.b") + 2, 1); + assertScopeBinding("class1.z"); + assertNumType("class1.z"); + } + + public void testCallNewRef() throws Exception { + idx.loadModule("callnewref"); + idx.ready(); + String src = getSource("callnewref.py"); + + String fsig = "callnewref.myfunc"; + assertFunctionBinding(fsig); + assertDefinition(fsig, "myfunc", src.indexOf("myfunc")); + assertReference(fsig, nthIndexOf(src, "myfunc", 2)); + assertCall(fsig, nthIndexOf(src, "myfunc", 3)); + + String csig = "callnewref.MyClass"; + assertClassBinding(csig); + assertDefinition(csig, "MyClass", src.indexOf("MyClass")); + assertReference(csig, nthIndexOf(src, "MyClass", 2)); + assertConstructed(csig, nthIndexOf(src, "MyClass", 3)); + + String msig = "callnewref.MyClass.mymethod"; + assertMethodBinding(msig); + assertDefinition(msig, "mymethod", src.indexOf("mymethod")); + assertReference(msig, nthIndexOf(src, "mymethod", 2)); + assertCall(msig, nthIndexOf(src, "mymethod", 3)); + } + + public void testPackageLoad() throws Exception { + idx.loadModule("pkgload"); + idx.ready(); + assertModuleBinding("pkgload"); + assertModuleBinding("pkg"); + assertScopeBinding("pkg.myvalue"); + } + + public void testUnqualifiedSamePkgImport() throws Exception { + idx.loadModule("pkg.animal.reptile.snake"); + idx.ready(); + assertModuleBinding("pkg.animal.reptile.snake"); + assertModuleBinding("pkg.animal.reptile.croc"); + assertClassBinding("pkg.animal.reptile.snake.Snake"); + assertClassBinding("pkg.animal.reptile.snake.Python"); + assertClassBinding("pkg.animal.reptile.croc.Crocodilian"); + assertClassBinding("pkg.animal.reptile.croc.Gavial"); + + String snakeSrc = getSource("pkg/animal/reptile/snake.py"); + assertReference("pkg.animal.reptile.croc", snakeSrc.indexOf("croc")); + assertReference("pkg.animal.reptile.croc", nthIndexOf(snakeSrc, "croc", 2)); + assertReference("pkg.animal.reptile.croc.Gavial", snakeSrc.indexOf("Gavial")); + } + + public void testAbsoluteImport() throws Exception { + idx.loadModule("pkg.mineral.metal.lead"); + idx.ready(); + assertModuleBinding("pkg"); + assertModuleBinding("pkg.plant"); + assertModuleBinding("pkg.plant.poison"); + assertModuleBinding("pkg.plant.poison.eggplant"); + + String src = getSource("pkg/mineral/metal/lead.py"); + assertReference("pkg", nthIndexOf(src, "pkg", 1)); + assertReference("pkg", nthIndexOf(src, "pkg", 2)); + + assertReference("pkg.plant", nthIndexOf(src, "plant", 1)); + assertReference("pkg.plant", nthIndexOf(src, ".plant", 2) + 1); + + assertReference("pkg.plant.poison", nthIndexOf(src, "poison", 1)); + assertReference("pkg.plant.poison", nthIndexOf(src, ".poison", 2) + 1); + + assertReference("pkg.plant.poison.eggplant", nthIndexOf(src, "eggplant", 1)); + assertReference("pkg.plant.poison.eggplant", nthIndexOf(src, ".eggplant", 2) + 1); + } + + public void testAbsoluteImportAs() throws Exception { + idx.loadModule("pkg.mineral.metal.iron"); + idx.ready(); + assertModuleBinding("pkg"); + assertModuleBinding("pkg.mineral"); + assertModuleBinding("pkg.mineral.metal"); + assertModuleBinding("pkg.mineral.metal.iron"); + assertModuleBinding("pkg.plant"); + assertModuleBinding("pkg.plant.poison"); + assertModuleBinding("pkg.plant.poison.eggplant"); + + String adjectives = "pkg.plant.poison.eggplant.adjectives"; + assertScopeBinding(adjectives); + + String aubergine = "pkg.mineral.metal.iron.aubergine"; + assertScopeBinding(aubergine); + assertBindingType(aubergine, "pkg.plant.poison.eggplant"); + + String src = getSource("pkg/mineral/metal/iron.py"); + assertReference("pkg", src.indexOf("pkg")); + assertReference("pkg.plant", src.indexOf("plant")); + assertReference("pkg.plant.poison", src.indexOf("poison")); + assertReference("pkg.plant.poison.eggplant", src.indexOf("eggplant")); + assertReference(aubergine, nthIndexOf(src, "aubergine", 2)); + assertReference(adjectives, src.indexOf("adjectives")); + } + + public void testImportFrom() throws Exception { + idx.loadModule("pkg.other.color.white"); + idx.ready(); + String src = getSource("pkg/other/color/white.py"); + assertReference("pkg.other.color.red", src.indexOf("red")); + assertReference("pkg.other.color.green", src.indexOf("green")); + assertReference("pkg.other.color.blue", src.indexOf("blue")); + + assertReference("pkg.other.color.red.r", src.indexOf("r as"), 1); + assertReference("pkg.other.color.blue.b", src.indexOf("b as"), 1); + + assertReference("pkg.other.color.red.r", src.indexOf("= R") + 2, 1); + assertReference("pkg.other.color.green.g", src.indexOf("g #"), 1); + assertReference("pkg.other.color.blue.b", src.indexOf("= B") + 2, 1); + } + + public void testImportStar() throws Exception { + idx.loadModule("pkg.other.color.crimson"); + idx.ready(); + String src = getSource("pkg/other/color/crimson.py"); + assertReference("pkg.other.color.red.r", src.indexOf("r,"), 1); + assertReference("pkg.other.color.red.g", src.indexOf("g,"), 1); + assertReference("pkg.other.color.red.b", src.indexOf("b"), 1); + } + + public void testImportStarAll() throws Exception { + idx.loadModule("pkg.misc.moduleB"); + idx.ready(); + String src = getSource("pkg/misc/moduleB.py"); + assertReference("pkg.misc.moduleA.a", src.indexOf("a #"), 1); + assertReference("pkg.misc.moduleA.b", src.indexOf("b #"), 1); + assertReference("pkg.misc.moduleA.c", src.indexOf("c #"), 1); + + assertNoReference("Should not have imported 'd'", + "pkg.misc.moduleA.d", src.indexOf("d #"), 1); + } + + public void testImportFromInitPy() throws Exception { + idx.loadModule("pkg.animal"); + idx.ready(); + assertModuleBinding("pkg"); + assertModuleBinding("pkg.animal"); + assertModuleBinding("pkg.animal.animaltest"); + assertScopeBinding("pkg.animal.success"); + assertScopeBinding("pkg.animal.animaltest.living"); + } + + // // Tests to add: + // // - import inside a function; check that names are VARIABLE (not SCOPE) + + // public void finishme_testModuleDictMerging() throws Exception { + // // goal is to test this case: + // // mod1.py: + // // a = 1 + // // mod2.py: + // // import mod1 + // // def test(): + // // print mod1.b # at this point mod1.b is an unknown attr of mod1 + // // mod3.py: + // // import mod1 + // // mod1.b = 2 # at this later point it gets defined + // // test: + // // load mod1, mod2, mod3 + // // => assert that in mod2.py, mod1.b refers to the definition in mod3.py + // } + + // test creating temp definition and then re-resolving it + public void testTempName() throws Exception { + String src = index( + "tmpname.py", + "def purge():", + " cache.clear()", + "cache = {}"); + assertScopeBinding("tmpname.cache"); + assertBindingType("tmpname.cache", NDictType.class); + assertDefinition("tmpname.cache", "cache", src.lastIndexOf("cache")); + + assertReference("tmpname.cache", src.indexOf("cache")); + assertNoDefinition("Temp-def should have been replaced", + "tmpname.cache", src.indexOf("cache"), "cache".length()); + + assertCall("__builtin__.dict.clear", src.lastIndexOf("clear")); + } + + public void testTempAttr() throws Exception { + String src = index( + "tmpattr.py", + "x = app.usage", + "app.usage = 'hi'"); + assertScopeBinding("tmpattr.x"); + assertScopeBinding("tmpattr.app"); + assertAttributeBinding("tmpattr.app.usage"); + assertStringType("tmpattr.app.usage"); + assertStringType("tmpattr.x"); + assertDefinition("tmpattr.app.usage", src.lastIndexOf("usage")); + assertReference("tmpattr.app.usage", src.indexOf("usage")); + } + + public void testTempAttrOnParam() throws Exception { + String src = index( + "tmpattr_param.py", + "def foo(x):", + " x.hello = 'hi'", + "def bar(y=None):", + " y.hello = 'hola'"); + assertFunctionBinding("tmpattr_param.foo"); + assertParamBinding("tmpattr_param.foo@x"); + assertAttributeBinding("tmpattr_param.foo@x.hello"); + assertStringType("tmpattr_param.foo@x.hello"); + assertReference("tmpattr_param.foo@x", src.indexOf("x.hello"), 1); + + assertFunctionBinding("tmpattr_param.bar"); + assertParamBinding("tmpattr_param.bar@y"); + assertAttributeBinding("tmpattr_param.bar@y.hello"); + assertStringType("tmpattr_param.bar@y.hello"); + assertReference("tmpattr_param.bar@y", src.indexOf("y.hello"), 1); + } + + public void testParamDefaultLambdaBinding() throws Exception { + String src = index( + "test.py", + "def foo(arg=lambda name: name + '!'):", + " x = arg('hi')"); + assertFunctionBinding("test.foo"); + assertParamBinding("test.foo@arg"); + assertFunctionBinding("test.lambda%1"); + assertParamBinding("test.lambda%1@name"); + assertReference("test.lambda%1@name", src.lastIndexOf("name")); + assertCall("test.foo@arg", src.lastIndexOf("arg")); + assertStringType("test.foo&x"); + } + + public void testNestedLambdaParam() throws Exception { + String src = index( + "test.py", + "def util(create):", + " return create()", + "z = lambda:util(create=lambda: str())", + "y = z()()"); + + assertScopeBinding("test.z"); + assertFunctionBinding("test.lambda%1&lambda%1"); + + // XXX: will require inferring across calls + // assertStringType("test.y"); + } + + public void testReassignAttrOfUnknown() throws Exception { + // This test has broken surprisingly often, so don't change it. + String src = index( + "reassign.py", + "app.foo = 'hello'", + "app.foo = 2"); + assertScopeBinding("reassign.app"); + NBinding nb = assertAttributeBinding("reassign.app.foo"); + NType type = nb.getType(); + assertTrue(type.isUnionType()); + Set types = ((NUnionType)type).getTypes(); + assertEquals(2, types.size()); + assertTrue(types.contains(idx.builtins.BaseStr)); + assertTrue(types.contains(idx.builtins.BaseNum)); + } + + public void testRefToProvisionalBinding() throws Exception { + String src = index( + "provisional.py", + "for a in []:", + " a.dump()", + "for a in []:", + " a.dump()"); + assertModuleBinding("provisional"); + assertScopeBinding("provisional.a"); + assertNoBinding("provisional.a.dump"); + } + + public void testRefToProvisionalBindingNewType() throws Exception { + String src = index( + "provisional.py", + "for b in []:", + " b.dump()", + "for b in ():", + " b.dump()"); + assertModuleBinding("provisional"); + assertScopeBinding("provisional.b"); + assertNoBinding("provisional.b.dump"); + } + + // http://www.python.org/dev/peps/pep-0227 + public void testSkipClassScope() throws Exception { + String src = index( + "skipclass.py", + "def aa():", + " xx = 'foo'", + " class bb:", + " xx = 10", + " def cc(self):", + " print bb.xx", + " print xx"); + assertReference("skipclass.aa&bb.xx", nthIndexOf(src, "xx", 3)); + assertReference("skipclass.aa&xx", nthIndexOf(src, "xx", 4)); + } + + public void testLambdaArgs() throws Exception { + String src = index( + "lambda_args.py", + "y = lambda x='hi': x.upper()", + "y = lambda x='there': x.lower()"); + assertScopeBinding("lambda_args.y"); + + assertFunctionBinding("lambda_args.lambda%1"); + assertParamBinding("lambda_args.lambda%1@x"); + assertStringType("lambda_args.lambda%1@x"); + assertReference("lambda_args.lambda%1@x", nthIndexOf(src, "x", 2)); + assertCall("__builtin__.str.upper", src.indexOf("upper")); + + assertFunctionBinding("lambda_args.lambda%2"); + assertParamBinding("lambda_args.lambda%1@x"); + assertReference("lambda_args.lambda%2@x", nthIndexOf(src, "x", 4)); + assertCall("__builtin__.str.lower", src.indexOf("lower")); + } + + public void testFunArgs() throws Exception { + String src = index( + "funargs.py", + "def foo(x, y='hi'):", + " z = 9", + " return x + y.upper() + z"); + assertFunctionBinding("funargs.foo"); + + assertParamBinding("funargs.foo@x"); + assertReference("funargs.foo@x", nthIndexOf(src, "x", 2)); + + assertParamBinding("funargs.foo@y"); + assertStringType("funargs.foo@y"); + assertReference("funargs.foo@y", nthIndexOf(src, "y", 2)); + + assertCall("__builtin__.str.upper", src.indexOf("upper")); + + assertVariableBinding("funargs.foo&z"); + assertReference("funargs.foo&z", nthIndexOf(src, "z", 2)); + } + + public void testDatetime() throws Exception { + String src = index( + "date_time.py", + "from datetime import datetime as dt", + "import datetime", + "now = dt.now()", + "d = now.date()", + "tz = now.tzinfo"); + assertModuleBinding("datetime"); + assertClassBinding("datetime.datetime"); + assertMethodBinding("datetime.datetime.date"); + + assertReference("datetime", nthIndexOf(src, "datetime", 1)); + assertReference("datetime.datetime", nthIndexOf(src, "datetime", 2)); + assertReference("datetime.datetime", nthIndexOf(src, "dt", 1), 2); + assertReference("datetime.datetime", nthIndexOf(src, "dt", 2), 2); + assertReference("datetime", nthIndexOf(src, "datetime", 3)); + assertCall("datetime.datetime.now", nthIndexOf(src, "now", 2)); + assertCall("datetime.datetime.date", nthIndexOf(src, "date()", 1)); + assertReference("datetime.time.tzinfo", nthIndexOf(src, "tzinfo", 1)); + assertBindingType("date_time.tz", "datetime.tzinfo"); + } + + public void testUnpackList() throws Exception { + index("unpacklist.py", + "a = [1, 2]", + "(b, c) = [3, 4]", + "[d, e] = ['hi', 'there']"); + assertScopeBinding("unpacklist.a"); + assertScopeBinding("unpacklist.b"); + assertScopeBinding("unpacklist.c"); + assertScopeBinding("unpacklist.d"); + assertScopeBinding("unpacklist.e"); + assertListType("unpacklist.a", "__builtin__.float"); + assertNumType("unpacklist.b"); + assertNumType("unpacklist.c"); + assertStringType("unpacklist.d"); + assertStringType("unpacklist.e"); + } + + public void testStringSlice() throws Exception { + String src = index( + "slicestring.py", + "a = 'hello'[2]", + "b = 'hello'[2:4]", + "test = 'testing'", + "test[-3:].lower()"); + assertScopeBinding("slicestring.a"); + assertScopeBinding("slicestring.b"); + assertStringType("slicestring.a"); + assertStringType("slicestring.b"); + assertCall("__builtin__.str.lower", src.lastIndexOf("lower")); + } + + public void testUnionStringSliceTempAttr() throws Exception { + String src = index( + "tmpattr_slice.py", + "def foo(filename):", + " module = filename or ''", + " module[-3:].lower()"); + assertCall("__builtin__.str.lower", src.lastIndexOf("lower")); + } + + public void testSelfBinding() throws Exception { + String src = index( + "selfish.py", + "class Foo():", + " def hello(self):", + " print self"); + assertClassBinding("selfish.Foo"); + assertMethodBinding("selfish.Foo.hello"); + assertParamBinding("selfish.Foo.hello@self"); + assertDefinition("selfish.Foo.hello@self", nthIndexOf(src, "self", 1)); + assertReference("selfish.Foo.hello@self", nthIndexOf(src, "self", 2)); + assertBindingType("selfish.Foo.hello@self", "selfish.Foo"); + } + + public void testInstanceAttrs() throws Exception { + String src = index( + "attr.py", + "class Foo():", + " def __init__(self):", + " self.elts = []", + " def add(self, item):", + " self.elts.append(item)"); + assertClassBinding("attr.Foo"); + assertConstructorBinding("attr.Foo.__init__"); + assertParamBinding("attr.Foo.__init__@self"); + assertDefinition("attr.Foo.__init__@self", nthIndexOf(src, "self", 1)); + assertReference("attr.Foo.__init__@self", nthIndexOf(src, "self", 2)); + assertBindingType("attr.Foo.__init__@self", "attr.Foo"); + + assertAttributeBinding("attr.Foo.elts"); + assertListType("attr.Foo.elts"); + + assertMethodBinding("attr.Foo.add"); + assertParamBinding("attr.Foo.add@self"); + assertBindingType("attr.Foo.add@self", "attr.Foo"); + assertParamBinding("attr.Foo.add@item"); + assertReference("attr.Foo.add@self", nthIndexOf(src, "self", 4)); + assertReference("attr.Foo.elts", nthIndexOf(src, "elts", 2)); + assertCall("__builtin__.list.append", src.indexOf("append")); + assertReference("attr.Foo.add@item", src.lastIndexOf("item")); + } + + public void testInstanceAttrsWithStdLib() throws Exception { + includeStandardLibrary(); + String src = index( + "dice.py", + "import random", + "class Dice(object):", + " def __init__(self):", + " self.__random = random.Random()", + " def set_seed(self, seed):", + " self.__random.seed(seed)"); + assertModuleBinding("random"); + NBinding r = assertClassBinding("random.Random"); + assertFalse(r.isBuiltin()); + + assertReference("random", nthIndexOf(src, "random", 3)); + assertConstructed("random.Random", src.indexOf("Random")); + + assertClassBinding("dice.Dice"); + assertReference("__builtin__.object", src.indexOf("object")); + + assertConstructorBinding("dice.Dice.__init__"); + assertParamBinding("dice.Dice.__init__@self"); + assertDefinition("dice.Dice.__init__@self", nthIndexOf(src, "self", 1)); + assertReference("dice.Dice.__init__@self", nthIndexOf(src, "self", 2)); + + assertBindingType("dice.Dice.__init__@self", "dice.Dice"); + + assertAttributeBinding("dice.Dice.__random"); + assertInstanceType("dice.Dice.__random", "random.Random"); + + assertMethodBinding("dice.Dice.set_seed"); + assertParamBinding("dice.Dice.set_seed@self"); + assertBindingType("dice.Dice.set_seed@self", "dice.Dice"); + assertParamBinding("dice.Dice.set_seed@seed"); + + assertReference("dice.Dice.set_seed@self", nthIndexOf(src, "self", 4)); + assertReference("dice.Dice.__random", nthIndexOf(src, "__random", 2)); + assertCall("random.Random.seed", nthIndexOf(src, "seed", 3)); + assertReference("dice.Dice.set_seed@seed", src.lastIndexOf("seed")); + } + + public void testOsPath() throws Exception { + String src = index( + "test.py", + "from os import path", + "print path.devnull", + "base, ext = path.split('/foo/bar/baz.py')", + "print ext.endswith('py')"); + assertReference("os.path.devnull", src.indexOf("devnull")); + assertStringType("os.path.devnull"); + assertStringType("test.base"); + assertStringType("test.ext"); + assertCall("os.path.split", src.indexOf("split")); + assertCall("__builtin__.str.endswith", src.indexOf("endswith")); + } + + public void testImportOsPath() throws Exception { + String src = index( + "test.py", + "import os.path", + "print os.path.devnull"); + assertReference("os", nthIndexOf(src, "os", 1)); + assertReference("os", nthIndexOf(src, "os", 2)); + assertReference("os.path", nthIndexOf(src, "path", 1)); + assertReference("os.path", nthIndexOf(src, "path", 2)); + assertReference("os.path.devnull", src.indexOf("devnull")); + } + + public void testExceptionsModule() throws Exception { + String src = index( + "test.py", + "import exceptions", + "raise exceptions.NotImplementedError"); + assertModuleBinding("exceptions"); + assertClassBinding("exceptions.NotImplementedError"); + assertReference("exceptions.NotImplementedError", src.indexOf("Not")); + } + + public void testDupFunctionDecl() throws Exception { + String src = index( + "test.py", + "if x:", + " def a(args):", + " print args", + "elif y:", + " def a(args):", + " print args"); + assertFunctionBinding("test.a"); + assertParamBinding("test.a@args"); + } + + public void testResolveExportedNames() throws Exception { + String src = index( + "test.py", + "__all__ = ['foo', 'bar' + 'baz', 'one', 'two']", + "def foo(x):", + " return x", + "bar = 6", + "baz = 7", + "barbaz = 8", + "one = 'hi'", + "two = 'there'"); + assertReference("test.foo", src.indexOf("'foo"), 5); + assertReference("test.one", src.indexOf("'one"), 5); + assertReference("test.two", src.indexOf("'two"), 5); + + assertNoReference("Should not have referenced 'bar'", + "test.bar", src.indexOf("bar"), 3); + } + + public void testImportFromPlusAssign() throws Exception { + String src = index( + "test.py", + "from os import sep", + "os = 10", + "print os"); + assertModuleBinding("os"); + assertReference("os", src.indexOf("os")); + assertNoDefinition("Import-from should not introduce a definition", + "test.os", src.indexOf("os"), "os".length()); + assertDefinition("test.os", nthIndexOf(src, "os", 2)); + assertNumType("test.os"); + assertReference("test.os", src.lastIndexOf("os")); + } + + public void testCircularTypeFunAndTuple() throws Exception { + String src = index( + "test.py", + "def foo():", + " return (foo,)"); + assertFunctionBinding("test.foo"); + NType ftype = idx.lookupQnameType("test.foo"); + assertTrue(ftype instanceof NFuncType); + NType rtype = ftype.asFuncType().getReturnType(); + assertTrue(rtype instanceof NTupleType); + assertEquals(1, rtype.asTupleType().getElementTypes().size()); + assertEquals(ftype, rtype.asTupleType().getElementTypes().get(0)); + assertEquals("]>>", ftype.toString()); + } + + public void testCircularTypeXInOwnList() throws Exception { + String src = index( + "test.py", + "x = (2,)", + "y = [x]", + "x = y"); + NType xtype = idx.lookupQnameType("test.x"); + assertTrue(xtype instanceof NUnionType); + + // Jump through some hoops to allow for either order in the union. + Set types = xtype.asUnionType().getTypes(); + assertEquals(2, types.size()); + NType[] array = types.toArray(new NType[2]); + boolean array0List = array[0] instanceof NListType; + boolean array1List = array[1] instanceof NListType; + + assertTrue(array0List || array1List); + int other = array0List ? 1 : 0; + assertTrue("Expected tuple: " + array[other], array[other].isTupleType()); + assertEquals(1, array[other].asTupleType().getElementTypes().size()); + assertEquals(idx.builtins.BaseNum, array[other].asTupleType().getElementTypes().get(0)); + + String s = xtype.toString(); + int index = s.indexOf("]>"; + String ref = "<#" + num + ">"; + + if (array0List) { + // union(list(unknown(tuple)), ref) + assertEquals("," + ref + "]>", s); + } else { + // union(tuple, list(unknown(ref))) + assertEquals("]>", s); + } + } + + public void testFunReturn() throws Exception { + // This use case used to extend the function return type by one + // wrapped NUnknownType for each static invocation of the function. + String src = index( + "fret.py", + "def foo(x): return x", + "a = foo('a')", + "b = foo('b')", + "c = foo('c')"); + NType ftype = idx.lookupQnameType("fret.foo"); + assertEquals(">", ftype.toString()); + NType ctype = idx.lookupQnameType("fret.c"); + assertEquals(ctype.follow(), ftype.asFuncType().getReturnType()); + } + + public void testListCompForIn() throws Exception { + String src = index( + "listforin.py", + "[line for line in ['foo']]"); + assertStringType("listforin.line"); + } + + public void testNoAddToBuiltin() throws Exception { + String src = index( + "nob.py", + "x = [line.rstrip() + '\\n' for line in ['a ']]"); + assertStringType("nob.line"); + assertCall("__builtin__.str.rstrip", src.indexOf("rstrip")); + assertNoBinding("__builtin__.list.rstrip"); + assertListType("nob.x", "__builtin__.str"); + } + + public void testDecoratorSyntax() throws Exception { + String deco1 = "@deco1"; + String deco2 = "@deco2 ('yargh')"; + String src = index( + "deco.py", + deco1, + deco2, + "def foo(): pass"); + assertFunctionBinding("deco.foo"); + NModule m = idx.getAstForFile("deco.py"); + assertNotNull(m); + + NNode obj = m.body.seq.get(0); + assertTrue(obj instanceof NFunctionDef); + NFunctionDef f = (NFunctionDef)obj; + List decos = f.getDecoratorList(); + assertNotNull(decos); + assertEquals(2, decos.size()); + assertTrue(decos.get(0) instanceof NName); + + NName d1 = (NName)decos.get(0); + assertEquals(nthIndexOf(src, "deco1", 1), d1.start()); + assertEquals("deco1".length(), d1.length()); + assertEquals("deco1", d1.id); + + assertTrue(decos.get(1) instanceof NCall); + NCall d2 = (NCall)decos.get(1); + assertTrue(d2.func instanceof NName); + assertEquals("deco2", ((NName)d2.func).id); + } + + public void testBasicDecoratorSyntax() throws Exception { + String src = index( + "deco.py", + "def deco1(func): print 'hello'; return func", + "@deco1()", + "def foo(): pass"); + assertFunctionBinding("deco.deco1"); + assertFunctionBinding("deco.foo"); + assertCall("deco.deco1", nthIndexOf(src, "deco1", 2)); + } +} Index: src/org/python/indexer/ast/NRepr.java =================================================================== --- src/org/python/indexer/ast/NRepr.java (revision 0) +++ src/org/python/indexer/ast/NRepr.java (revision 0) @@ -0,0 +1,43 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NRepr extends NNode { + + static final long serialVersionUID = -7920982714296311413L; + + public NNode value; + + public NRepr(NNode n) { + this(n, 0, 1); + } + + public NRepr(NNode n, int start, int end) { + super(start, end); + this.value = n; + addChildren(n); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(value, s); + return setType(Indexer.idx.builtins.BaseStr); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(value, v); + } + } +} Index: src/org/python/indexer/ast/NSubscript.java =================================================================== --- src/org/python/indexer/ast/NSubscript.java (revision 0) +++ src/org/python/indexer/ast/NSubscript.java (revision 0) @@ -0,0 +1,127 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NFuncType; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +public class NSubscript extends NNode { + + static final long serialVersionUID = -493854491438387425L; + + public NNode value; + public NNode slice; // an NIndex or NSlice + + public NSubscript(NNode value, NNode slice) { + this(value, slice, 0, 1); + } + + public NSubscript(NNode value, NNode slice, int start, int end) { + super(start, end); + this.value = value; + this.slice = slice; + addChildren(value, slice); + } + + @Override + public NType resolve(Scope s) throws Exception { + NType vt = resolveExpr(value, s); + NType st = resolveExpr(slice, s); + + // slicing + if (vt.isUnknownType()) { + if (st.isListType()) { + return setType(vt); + } + return setType(new NUnknownType()); + } + + if (st.isListType()) { + NType getslice_type = vt.getTable().lookupTypeAttr("__getslice__"); + if (getslice_type == null) { + addError("The type can't be sliced: " + vt); + return setType(new NUnknownType()); + } + if (!getslice_type.isFuncType()) { + addError("The type's __getslice__ method is not a function: " + + getslice_type); + return setType(new NUnknownType()); + } + return setType(getslice_type.asFuncType().getReturnType().follow()); + } + + // subscription + if (slice instanceof NIndex) { + if (vt.isListType()) { + warnUnlessNumIndex(st); + return setType(vt.asListType().getElementType()); + } + if (vt.isTupleType()) { + warnUnlessNumIndex(st); + return setType(vt.asTupleType().toListType().getElementType()); + } + if (vt.isStrType()) { + warnUnlessNumIndex(st); + return setType(Indexer.idx.builtins.BaseStr); + } + // XXX: unicode, buffer, xrange + + if (vt.isDictType()) { + if (!st.follow().equals(vt.asDictType().getKeyType())) { + addWarning("Possible KeyError (wrong type for subscript)"); + } + return setType(vt.asDictType().getValueType()); // infer it regardless + } + // else fall through + } + + // subscription via delegation + if (vt.isUnionType()) { + for (NType u : vt.asUnionType().getTypes()) { + NType gt = vt.getTable().lookupTypeAttr("__getitem__"); + if (gt != null) { + return setType(get__getitem__type(gt, gt)); + } + } + } + NType gt = vt.getTable().lookupTypeAttr("__getitem__"); + return setType(get__getitem__type(gt, vt)); + } + + private void warnUnlessNumIndex(NType subscriptType) { + NType follow = subscriptType.follow(); + if (!follow.isNumType() && !follow.isUnknownType()) { + addWarning("Possible KeyError: subscript should be a number; found " + follow); + } + } + + private NType get__getitem__type(NType gt, NType vt) { + if (gt == null) { + addError("indexing type without __getitem__ method: " + vt); + return new NUnknownType(); + } + if (!gt.isFuncType()) { + addError("The type's __getitem__ method is not a function: " + gt); + return new NUnknownType(); + } + return gt.asFuncType().getReturnType().follow(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(value, v); + visitNode(slice, v); + } + } +} Index: tests/java/org/python/indexer/data/yinw/yinw-22.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-22.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-22.py (revision 0) @@ -0,0 +1,15 @@ +class A: + a = 1 + +class B: + a = 2 + +def g(x): + print x.a + +o1 = A() +o2 = B() + +g(o1) +g(o2) + Index: src/org/python/indexer/StyleRun.java =================================================================== --- src/org/python/indexer/StyleRun.java (revision 0) +++ src/org/python/indexer/StyleRun.java (revision 0) @@ -0,0 +1,107 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +/** + * Represents a simple style run for purposes of source highlighting. + */ +public class StyleRun implements Comparable { + + public enum Type { + KEYWORD, + COMMENT, + STRING, + DOC_STRING, + IDENTIFIER, + BUILTIN, + NUMBER, + CONSTANT, // ALL_CAPS identifier + FUNCTION, // function name + PARAMETER, // function parameter + LOCAL, // local variable + DECORATOR, // function decorator + CLASS, // class name + ATTRIBUTE, // object attribute + LINK, // hyperlink + ANCHOR, // name anchor + DELIMITER, + TYPE_NAME, // reference to a type (e.g. function or class name) + // diagnostics + ERROR, + WARNING, + INFO + } + + public Type type; + private int offset; // file offset + private int length; // style run length + + public String message; // optional hover text + public String url; // internal or external link + + public StyleRun(Type type, int offset, int length) { + this.type = type; + this.offset = offset; + this.length = length; + } + + public StyleRun(Type type, int offset, int length, String msg, String url) { + this.type = type; + this.offset = offset; + this.length = length; + this.message = msg; + this.url = url; + } + + public int start() { + return offset; + } + + public int end() { + return offset + length; + } + + public int length() { + return length; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof StyleRun)) { + return false; + } + StyleRun other = (StyleRun)o; + return other.type == this.type + && other.offset == this.offset + && other.length == this.length + && equalFields(other.message, this.message) + && equalFields(other.url, this.url); + } + + private boolean equalFields(Object o1, Object o2) { + if (o1 == null) { + return o2 == null; + } else { + return o1.equals(o2); + } + } + + public int compareTo(StyleRun other) { + if (this.equals(other)) { + return 0; + } + if (this.offset < other.offset) { + return -1; + } + if (other.offset < this.offset) { + return 1; + } + return this.hashCode() - other.hashCode(); + } + + @Override + public String toString() { + return "[" + type + " beg=" + offset + " len=" + length + "]"; + } +} Index: tests/java/org/python/indexer/data/pkg/other/__init__.py =================================================================== Index: src/org/python/indexer/Indexer.java =================================================================== --- src/org/python/indexer/Indexer.java (revision 0) +++ src/org/python/indexer/Indexer.java (revision 0) @@ -0,0 +1,977 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NModule; +import org.python.indexer.ast.NName; +import org.python.indexer.ast.NUrl; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.TreeMap; +import java.util.Map.Entry; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Indexes a set of Python files and builds a code graph.

    + * This class is not thread-safe. + */ +public class Indexer { + + /** + * The global indexer instance. Provides convenient access to global + * resources, as well as easy cleanup of resources after the index is built. + */ + public static Indexer idx; + + /** + * A scope containing bindings for all modules currently loaded by the indexer. + */ + public Scope moduleTable = new Scope(null, Scope.Type.GLOBAL); + + /** + * The top-level (builtin) scope. + */ + public Scope globaltable = new Scope(null, Scope.Type.GLOBAL); + + /** + * A map of all bindings created, keyed on their qnames. + */ + private Map allBindings = new HashMap(); + + /** + * A map of references to their referenced bindings. Most nodes will refer + * to a single binding, and few ever refer to more than two. One situation + * in which a multiple reference can occur is the an attribute of a union + * type. For instance: + * + *

    +     *   class A:
    +     *     def foo(self): pass
    +     *   class B:
    +     *     def foo(self): pass
    +     *   if some_condition:
    +     *     var = A()
    +     *   else
    +     *     var = B()
    +     *   var.foo()  # foo here refers to both A.foo and B.foo
    +     * 
    +     */
    +    private Map> locations = new HashMap>();
    +
    +    /**
    +     * Diagnostics.
    +     */
    +    public Map> problems = new HashMap>();
    +    public Map> parseErrs = new HashMap>();
    +
    +    public String currentFile = null;
    +    public String projDir = null;
    +
    +    public List path = new ArrayList();
    +
    +    /**
    +     * Manages a store of serialized ASTs.  ANTLR parsing is one of the slower and
    +     * more expensive phases of indexing; reusing parse trees can help with resource
    +     * utilization when indexing several projects (or re-indexing one project).
    +     */
    +    private AstCache astCache;
    +
    +    /**
    +     * When resolving imports we look in various possible locations.
    +     * This set keeps track of modules we attempted but didn't find.
    +     */
    +    public Set failedModules = new HashSet();
    +
    +    /**
    +     * This set tracks module imports that could not be resolved.
    +     */
    +    private Map> unresolvedModules = new TreeMap>();
    +
    +    /**
    +     * Manages the built-in modules -- that is, modules from the standard Python
    +     * library that are implemented in C and consequently have no Python source.
    +     */
    +    public Builtins builtins;
    +
    +    private boolean aggressiveAssertions;
    +
    +    // stats counters
    +    private int nloc = 0;
    +    private int nunbound = 0;
    +    private int nunknown = 0;
    +    private int nprob = 0;
    +    private int nparsing = 0;
    +    private int loadedFiles = 0;
    +
    +    private Logger logger = Logger.getLogger(Indexer.class.getCanonicalName());
    +
    +    public Indexer() {
    +        idx = this;
    +        builtins = new Builtins(globaltable, moduleTable);
    +        builtins.init();
    +    }
    +
    +    public void setLogger(Logger logger) {
    +        if (logger == null) {
    +            throw new IllegalArgumentException("null logger param");
    +        }
    +        logger = logger;
    +    }
    +
    +    public Logger getLogger() {
    +        return logger;
    +    }
    +
    +    public void setProjectDir(String cd) throws IOException {
    +        projDir = Util.canonicalize(cd);
    +    }
    +
    +    /**
    +     * Configures whether the indexer should abort with an exception when it
    +     * encounters an internal error or unexpected program state.  Normally the
    +     * indexer attempts to continue indexing, on the assumption that having an
    +     * index with mostly good data is better than having no index at all.
    +     * Enabling aggressive assertions is useful for debugging the indexer.
    +     */
    +    public void enableAggressiveAssertions(boolean enable) {
    +        aggressiveAssertions = enable;
    +    }
    +
    +    public boolean aggressiveAssertionsEnabled() {
    +        return aggressiveAssertions;
    +    }
    +
    +    /**
    +     * If aggressive assertions are enabled, propages the passed
    +     * {@link Throwable}, wrapped in an {@link IndexingException}.
    +     * @param msg descriptive message; ok to be {@code null}
    +     * @throws IndexingException
    +     */
    +    public void handleException(String msg, Throwable cause) {
    +        // Stack overflows are still fairly common due to cyclic
    +        // types, and they take up an awful lot of log space, so we
    +        // don't log the whole trace by default.
    +        if (cause instanceof StackOverflowError) {
    +            logger.log(Level.WARNING, msg, cause);
    +            return;
    +        }
    +
    +        if (aggressiveAssertionsEnabled()) {
    +            if (msg != null) {
    +                throw new IndexingException(msg, cause);
    +            }
    +            throw new IndexingException(cause);
    +        }
    +        if (msg == null)
    +            msg = "";
    +        if (cause == null)
    +            cause = new Exception();
    +        logger.log(Level.WARNING, msg, cause);
    +    }
    +
    +    /**
    +     * Signals a failed assertion about the state of the indexer or index.
    +     * If aggressive assertions are enabled, throws an {@code IndexingException}.
    +     * Otherwise the message is logged as a warning, and indexing continues.
    +     * @param msg a descriptive message about the problem
    +     * @see enableAggressiveAssertions
    +     * @throws IndexingException
    +     */
    +    public void reportFailedAssertion(String msg) {
    +        if (aggressiveAssertionsEnabled()) {
    +            throw new IndexingException(msg, new Exception());  // capture stack
    +        }
    +        // Need more configuration control here.
    +        // Currently getting a hillion jillion of these in Grok.
    +        if (false) {
    +            logger.log(Level.WARNING, msg);
    +        }
    +    }
    +
    +    /**
    +     * Adds the specified absolute paths to the module search path.
    +     */
    +    public void addPaths(List p) throws IOException {
    +        for (String s : p) {
    +            addPath(s);
    +        }
    +    }
    +
    +    /**
    +     * Adds the specified absolute path to the module search path.
    +     */
    +    public void addPath(String p) throws IOException {
    +        path.add(Util.canonicalize(p));
    +    }
    +
    +    /**
    +     * Sets the module search path to the specified list of absolute paths.
    +     */
    +    public void setPath(List path) throws IOException {
    +        this.path = new ArrayList(path.size());
    +        addPaths(path);
    +    }
    +
    +    /**
    +     * Returns the module search path -- the project directory followed by any
    +     * paths that were added by {@link addPath}.
    +     */
    +    public List getLoadPath() {
    +        List loadPath = new ArrayList();
    +        if (projDir != null) {
    +            loadPath.add(projDir);
    +        }
    +        loadPath.addAll(path);
    +        return loadPath;
    +    }
    +
    +    public boolean isLibFile(String file) {
    +        if (file.startsWith("/")) {
    +            return true;
    +        }
    +        if (path != null) {
    +            for (String p : path) {
    +                if (file.startsWith(p)) {
    +                    return true;
    +                }
    +            }
    +        }
    +        return false;
    +    }
    +
    +    /**
    +     * Returns the mutable set of all bindings collected, keyed on their qnames.
    +     */
    +    public Map getBindings() {
    +        return allBindings;
    +    }
    +
    +    /**
    +     * Return the binding for {@code qname}, or {@code null} if not known.
    +     */
    +    public NBinding lookupQname(String qname) {
    +        return allBindings.get(qname);
    +    }
    +
    +    /**
    +     * Return the type for {@code qname}, or {@code null} if not known.
    +     * @throws IllegalStateException if {@link #ready} has not been called.
    +     */
    +    public NType lookupQnameType(String qname) {
    +        NBinding b = lookupQname(qname);
    +        if (b != null) {
    +            return b.followType();
    +        }
    +        return null;
    +    }
    +
    +    NModuleType getCachedModule(String file) {
    +        return (NModuleType)moduleTable.lookupType(file);
    +    }
    +
    +    /**
    +     * Returns (loading/resolving if necessary) the module for a given source path.
    +     * @param file absolute file path
    +     */
    +    public NModuleType getModuleForFile(String file) throws Exception {
    +        if (failedModules.contains(file)) {
    +            return null;
    +        }
    +        NModuleType m = getCachedModule(file);
    +        if (m != null) {
    +            return m;
    +        }
    +        return loadFile(file);
    +    }
    +
    +    /**
    +     * Returns the list, possibly empty but never {@code null}, of
    +     * errors and warnings generated in the file.
    +     */
    +    public List getDiagnosticsForFile(String file) {
    +        List errs = problems.get(file);
    +        if (errs != null) {
    +            return errs;
    +        }
    +        return new ArrayList();
    +    }
    +
    +    /**
    +     * Create an outline for a file in the index.
    +     * @param scope the file scope
    +     * @param path the file for which to build the outline
    +     * @return a list of entries constituting the file outline.
    +     * Returns an empty list if the indexer hasn't indexed that path.
    +     */
    +    public List generateOutline(String file) throws Exception {
    +        return new Outliner().generate(this, file);
    +    }
    +
    +    /**
    +     * Add a reference to binding {@code b} at AST node {@code node}.
    +     * @param node a node referring to a name binding.  Typically a
    +     * {@link NName}, {@link NStr} or {@link NUrl}.
    +     */
    +    public void putLocation(NNode node, NBinding b) {
    +        if (node == null) {
    +            return;
    +        }
    +        putLocation(new Ref(node), b);
    +    }
    +
    +    public void putLocation(Ref ref, NBinding b) {
    +        if (ref == null) {
    +            return;
    +        }
    +        List bindings = locations.get(ref);
    +        if (bindings == null) {
    +            // The indexer is heavily memory-constrained, so we need small overhead.
    +            // Empirically using a capacity-1 ArrayList for the binding set
    +            // uses about 1/2 the memory of a LinkedList, and 1/4 the memory
    +            // of a default HashSet.
    +            bindings = new ArrayList(1);
    +            locations.put(ref, bindings);
    +        }
    +        if (!bindings.contains(b)) {
    +            bindings.add(b);
    +            // Having > 1 is often an indicator of an indexer bug:
    +            // if (bindings.size() > 1) {
    +            //     info("now have " + bindings.size() + " bindings for " + ref + " in " +
    +            //          ref.getFile() + ": " + bindings);
    +            // }
    +        }
    +        b.addRef(ref);
    +    }
    +
    +    /**
    +     * Add {@code node} as a reference to binding {@code b}, removing
    +     * {@code node} from any other binding ref-lists that it may have occupied.
    +     * Currently only used in retargeting attribute references from provisional
    +     * bindings once the actual binding is determined.
    +     */
    +    public void updateLocation(Ref node, NBinding b) {
    +        if (node == null) {
    +            return;
    +        }
    +        List bindings = locations.get(node);
    +        if (bindings == null) {
    +            bindings = new ArrayList(1);
    +            locations.put(node, bindings);
    +        } else {
    +            for (NBinding oldb : bindings) {
    +                oldb.removeRef(node);
    +            }
    +            bindings.clear();
    +        }
    +        if (!bindings.contains(b)) {
    +            bindings.add(b);
    +        }
    +        b.addRef(node);
    +    }
    +
    +    public void removeBinding(NBinding b) {
    +        allBindings.remove(b);
    +    }
    +
    +    public NBinding putBinding(NBinding b) {
    +        if (b == null) {
    +            throw new IllegalArgumentException("null binding arg");
    +        }
    +        String qname = b.getQname();
    +        if (qname == null || qname.isEmpty()) {
    +            throw new IllegalArgumentException("Null/empty qname: " + b);
    +        }
    +
    +        NBinding existing = allBindings.get(qname);
    +        if (existing == b) {
    +            return b;
    +        }
    +        if (existing != null) {
    +            duplicateBindingFailure(b, existing);
    +
    +            // A bad edge case was triggered by an __init__.py that defined a
    +            // "Parser" binding (type unknown), and there was a Parser.py in the
    +            // same directory.  Loading Parser.py resulted in infinite recursion.
    +            //
    +            // XXX: need to revisit this logic.  It seems that bindings made in
    +            // __init__.py probably (?) ought to have __init__ in their qnames
    +            // to avoid dup-binding conflicts.  The Indexer module table also
    +            // probably ought not be a normal scope -- it's different enough that
    +            // overloading it to handle modules is making the logic rather twisty.
    +            if (b.getKind() == NBinding.Kind.MODULE) {
    +                return b;
    +            }
    +
    +            return existing;
    +        }
    +        allBindings.put(qname, b);
    +        return b;
    +    }
    +
    +    private void duplicateBindingFailure(NBinding newb, NBinding oldb) {
    +        // XXX:  this seems to happen only (and always) for duplicated
    +        // class or function defs in the same scope.  Need to figure out
    +        // what the right thing is for this scenario.
    +        if (true) {
    +            return;
    +        }
    +
    +        StringBuilder sb = new StringBuilder();
    +        sb.append("Error creating binding ");
    +        sb.append(newb);
    +        sb.append(" in file ");
    +        sb.append(newb.getFirstFile());
    +        sb.append(": qname already bound to ");
    +        sb.append(oldb);
    +        sb.append(" in file ");
    +        sb.append(oldb.getFirstFile());
    +        reportFailedAssertion(sb.toString());
    +    }
    +
    +    public void putProblem(NNode loc, String msg) {
    +        String file;
    +        if (loc != null && ((file = loc.getFile()) != null)) {
    +            addFileErr(file, loc.start(), loc.end(), msg);
    +        }
    +    }
    +
    +    public void putProblem(String file, int beg, int end, String msg) {
    +        if (file != null) {
    +            addFileErr(file, beg, end, msg);
    +        }
    +    }
    +
    +    void addFileErr(String file, int beg, int end, String msg) {
    +        List msgs = getFileErrs(file, problems);
    +        msgs.add(new Diagnostic(file, Diagnostic.Type.ERROR, beg, end, msg));
    +    }
    +
    +    List getParseErrs(String file) {
    +        return getFileErrs(file, parseErrs);
    +    }
    +
    +    List getFileErrs(String file, Map> map) {
    +        List msgs = map.get(file);
    +        if (msgs == null) {
    +            msgs = new ArrayList();
    +            map.put(file, msgs);
    +        }
    +        return msgs;
    +    }
    +
    +    /**
    +     * Loads a file and all its ancestor packages.
    +     * @see #loadFile(String,boolean)
    +     */
    +    public NModuleType loadFile(String path) throws Exception {
    +        return loadFile(path, false);
    +    }
    +
    +    /**
    +     * Loads a module from a string containing the module contents.
    +     * Idempotent:  looks in the module cache first. Used for simple unit tests.
    +     * @param path a path for reporting/caching purposes.  The filename
    +     *        component is used to construct the module qualified name.
    +     */
    +    public NModuleType loadString(String path, String contents) throws Exception {
    +        NModuleType module = getCachedModule(path);
    +        if (module != null) {
    +            finer("\nusing cached module " + path + " [succeeded]");
    +            return module;
    +        }
    +        return parseAndResolve(path, contents);
    +    }
    +
    +    /**
    +     * Load, parse and analyze a source file given its absolute path.
    +     * By default, loads the entire ancestor package chain.
    +     *
    +     * @param path the absolute path to the file or directory.
    +     *        If it is a directory, it is suffixed with "__init__.py", and
    +     *        only that file is loaded from the directory.
    +     *
    +     * @param noparents {@code true} to skip loading ancestor packages
    +     *
    +     * @return {@code null} if {@code path} could not be loaded
    +     */
    +    public NModuleType loadFile(String path, boolean skipChain) throws Exception {
    +        File f = new File(path);
    +        if (f.isDirectory()) {
    +            finer("\n    loading init file from directory: " + path);
    +            f = Util.joinPath(path, "__init__.py");
    +            path = f.getAbsolutePath();
    +        }
    +
    +        if (!f.canRead()) {
    +            finer("\nfile not not found or cannot be read: " + path);
    +            return null;
    +        }
    +
    +        NModuleType module = getCachedModule(path);
    +        if (module != null) {
    +            finer("\nusing cached module " + path + " [succeeded]");
    +            return module;
    +        }
    +
    +        if (!skipChain) {
    +            loadParentPackage(path);
    +        }
    +        try {
    +            return parseAndResolve(path);
    +        } catch (StackOverflowError soe) {
    +            handleException("Error loading " + path, soe);
    +            return null;
    +        }
    +    }
    +
    +    /**
    +     * When we load a module, load all its parent packages, top-down.
    +     * This is in part because Python does it anyway, and in part so that you
    +     * can click on all parent package components in import statements.
    +     * We load whole ancestor chain top-down, as does Python.
    +     */
    +    private void loadParentPackage(String file) throws Exception {
    +        File f = new File(file);
    +        File parent = f.getParentFile();
    +        if (parent == null || isInLoadPath(parent)) {
    +            return;
    +        }
    +        // the parent package of an __init__.py file is the grandparent dir
    +        if (parent != null && f.isFile() && "__init__.py".equals(f.getName())) {
    +            parent = parent.getParentFile();
    +        }
    +        if (parent == null || isInLoadPath(parent)) {
    +            return;
    +        }
    +        File initpy = Util.joinPath(parent, "__init__.py");
    +        if (!(initpy.isFile() && initpy.canRead())) {
    +            return;
    +        }
    +        loadFile(initpy.getPath());
    +    }
    +
    +    private boolean isInLoadPath(File dir) {
    +        for (String s : getLoadPath()) {
    +            if (new File(s).equals(dir)) {
    +                return true;
    +            }
    +        }
    +        return false;
    +    }
    +
    +    private NModuleType parseAndResolve(String file) throws Exception {
    +        return parseAndResolve(file, null);
    +    }
    +
    +    /**
    +     * Parse a file or string and return its module parse tree.
    +     * @param file the filename
    +     * @param contents optional file contents.  If {@code null}, loads the
    +     *        file contents from disk.
    +     */
    +    @SuppressWarnings("unchecked")
    +    private NModuleType parseAndResolve(String file, String contents) throws Exception {
    +        // Avoid infinite recursion if any caller forgets this check.  (Has happened.)
    +        NModuleType nmt = (NModuleType)moduleTable.lookupType(file);
    +        if (nmt != null) {
    +            return nmt;
    +        }
    +
    +        // Put it in the cache now to prevent circular import from recursing.
    +        NModuleType mod = new NModuleType(Util.moduleNameFor(file), file, globaltable);
    +        moduleTable.put(file, new NUrl("file://" + file), mod, NBinding.Kind.MODULE);
    +
    +        try {
    +            NModule ast = null;
    +            if (contents != null) {
    +                ast = getAstForFile(file, contents);
    +            } else {
    +                ast = getAstForFile(file);
    +            }
    +            if (ast == null) {
    +                return null;
    +            }
    +
    +            finer("resolving: " + file);
    +            ast.resolve(globaltable);
    +            finer("[success]");
    +            loadedFiles++;
    +            return mod;
    +        } catch (OutOfMemoryError e) {
    +            if (astCache != null) {
    +                astCache.clear();
    +            }
    +            System.gc();
    +            return null;
    +        }
    +    }
    +
    +    private AstCache getAstCache() throws Exception {
    +        if (astCache == null) {
    +            astCache = AstCache.get();
    +        }
    +        return astCache;
    +    }
    +
    +    /**
    +     * Returns the syntax tree for {@code file}. 

    + */ + public NModule getAstForFile(String file) throws Exception { + return getAstCache().getAST(file); + } + + /** + * Returns the syntax tree for {@code file}.

    + */ + public NModule getAstForFile(String file, String contents) throws Exception { + return getAstCache().getAST(file, contents); + } + + public NModuleType getBuiltinModule(String qname) throws Exception { + return builtins.get(qname); + } + + /** + * This method searches the module path for the module {@code modname}. + * If found, it is passed to {@link #loadFile}. + * + *

    The mechanisms for importing modules are in general statically + * undecidable. We make a reasonable effort to follow the most common + * lookup rules. + * + * @param modname a module name. Can be a relative path to a directory + * or a file (without the extension) or a possibly-qualified + * module name. If it is a module name, cannot contain leading dots. + * + * @see http://docs.python.org/reference/simple_stmts.html#the-import-statement + */ + public NModuleType loadModule(String modname) throws Exception { + if (failedModules.contains(modname)) { + return null; + } + + NModuleType cached = getCachedModule(modname); // builtin file-less modules + if (cached != null) { + finer("\nusing cached module " + modname); + return cached; + } + + NModuleType mt = getBuiltinModule(modname); + if (mt != null) { + return mt; + } + + finer("looking for module " + modname); + + if (modname.endsWith(".py")) { + modname = modname.substring(0, modname.length() - 3); + } + String modpath = modname.replace('.', '/'); + + // A nasty hack to avoid e.g. python2.5 becoming python2/5. + // Should generalize this for directory components containing '.'. + modpath = modpath.replaceFirst("(/python[23])/([0-9]/)", "$1.$2"); + + List loadPath = getLoadPath(); + + for (String p : loadPath) { + String dirname = p + modpath; + String pyname = dirname + ".py"; + String initname = Util.joinPath(dirname, "__init__.py").getAbsolutePath(); + String name; + + // foo/bar has priority over foo/bar.py + // http://www.python.org/doc/essays/packages.html + if (Util.isReadableFile(initname)) { + name = initname; + } else if (Util.isReadableFile(pyname)) { + name = pyname; + } else { + continue; + } + + name = Util.canonicalize(name); + NModuleType m = loadFile(name); + if (m != null) { + finer("load of module " + modname + "[succeeded]"); + return m; + } + } + finer("failed to find module " + modname + " in load path"); + failedModules.add(modname); + return null; + } + + /** + * Load all Python source files recursively if the given fullname is a + * directory; otherwise just load a file. Looks at file extension to + * determine whether to load a given file. + */ + public void loadFileRecursive(String fullname) throws Exception { + File file_or_dir = new File(fullname); + if (file_or_dir.isDirectory()) { + for (File file : file_or_dir.listFiles()) { + loadFileRecursive(file.getAbsolutePath()); + } + } else { + if (file_or_dir.getAbsolutePath().endsWith(".py")) { + loadFile(file_or_dir.getAbsolutePath()); + } + } + } + + /** + * Performs final indexing-building passes, including marking references to + * undeclared variables. Caller should invoke this method after loading all + * files. + */ + public void ready() { + fine("Checking for undeclared variables"); + + for (Entry> ent : locations.entrySet()) { + Ref ref = ent.getKey(); + List bindings = ent.getValue(); + + convertCallToNew(ref, bindings); + + if (countDefs(bindings) == 0) { + // XXX: fix me: + // if (ref instanceof NName && ((NName)ref).isAttribute()) { + // nunknown++; // not so serious + // } else { + // nunbound++; // more serious + // putProblem(ref, "variable may not be bound: " + ref); + // } + } else { + nloc++; + } + } + + nprob = problems.size(); + nparsing = parseErrs.size(); + + Set removals = new HashSet(); + for (Entry e : allBindings.entrySet()) { + NBinding nb = e.getValue(); + if (nb.isProvisional() || nb.getNumDefs() == 0) { + removals.add(e.getKey()); + } + } + for (String s : removals) { + allBindings.remove(s); + } + + locations.clear(); + } + + private void convertCallToNew(Ref ref, List bindings) { + if (ref.isRef()) { + return; + } + if (bindings.isEmpty()) { + return; + } + NBinding nb = bindings.get(0); + NType t = nb.followType(); + if (t.isUnionType()) { + t = t.asUnionType().firstKnownNonNullAlternate(); + if (t == null) { + return; + } + } + NType tt = t.follow(); + if (!tt.isUnknownType() && !tt.isFuncType()) { + ref.markAsNew(); + } + } + + /** + * Clears the AST cache (to free up memory). Subsequent calls to + * {@link #getAstForFile} will either fetch the serialized AST from a + * disk cache or re-parse the file from scratch. + */ + public void clearAstCache() { + if (astCache != null) { + astCache.clear(); + } + } + + /** + * Clears the module table, discarding all resolved ASTs (modules) + * and their scope information. + */ + public void clearModuleTable() { + moduleTable.clear(); + moduleTable = new Scope(null, Scope.Type.GLOBAL); + clearAstCache(); + } + + private int countDefs(List bindings) { + int count = 0; + for (NBinding b : bindings) { + count += b.getNumDefs(); + } + return count; + } + + private String printBindings() { + StringBuilder sb = new StringBuilder(); + Set sorter = new TreeSet(); + sorter.addAll(allBindings.keySet()); + for (String key : sorter) { + NBinding b = allBindings.get(key); + sb.append(b.toString()).append("\n"); + } + return sb.toString(); + } + + /** + * Reports a failed module or submodule resolution. + * @param qname module qname, e.g. "org.foo.bar" + * @param file the file where the unresolved import occurred + */ + public void recordUnresolvedModule(String qname, String file) { + Set importers = unresolvedModules.get(qname); + if (importers == null) { + importers = new TreeSet(); + unresolvedModules.put(qname, importers); + } + importers.add(file); + } + + /** + * Report resolution rate and other statistics data. + */ + public String getStatusReport() { + int total = nloc + nunbound + nunknown; + if (total == 0) { + total = 1; + } + StringBuilder sb = new StringBuilder(); + sb.append("Summary: \n") + .append("- modules loaded: ") + .append(loadedFiles) + .append("\n- unresolved modules: ") + .append(unresolvedModules.size()) + .append("\n"); + + for (String s : unresolvedModules.keySet()) { + sb.append(s).append(": "); + Set importers = unresolvedModules.get(s); + if (importers.size() > 5) { + sb.append(importers.iterator().next()); + sb.append(" and " ); + sb.append(importers.size()); + sb.append(" more"); + } else { + String files = importers.toString(); + sb.append(files.substring(1, files.length() - 1)); + } + sb.append("\n"); + } + + // XXX: these are no longer accurate, and need to be fixed. + // .append("\nnames resolved: " .append(percent(nloc, total) + // .append("\nunbound: " .append(percent(nunbound, total) + // .append("\nunknown: " .append(percent(nunknown, total) + + sb.append("\nsemantics problems: ").append(nprob); + sb.append("\nparsing problems: ").append(nparsing); + return sb.toString(); + } + + private String percent(int num, int total) { + double pct = (num * 1.0) / total; + pct = Math.round(pct * 10000) / 100.0; + return num + "/" + total + " = " + pct + "%"; + } + + public int numFilesLoaded() { + return loadedFiles; + } + + public List getLoadedFiles() { + List files = new ArrayList(); + for (String file : moduleTable.keySet()) { + if (file.startsWith("/")) { + files.add(file); + } + } + return files; + } + + public void log(Level level, String msg) { + if (logger.isLoggable(level)) { + logger.log(level, msg); + } + } + + public void severe(String msg) { + log(Level.SEVERE, msg); + } + + public void warn(String msg) { + log(Level.WARNING, msg); + } + + public void info(String msg) { + log(Level.INFO, msg); + } + + public void fine(String msg) { + log(Level.FINE, msg); + } + + public void finer(String msg) { + log(Level.FINER, msg); + } + + /** + * Releases all resources for the current indexer. + */ + public void release() { + // Null things out to catch anyone who might still be referencing them. + moduleTable = globaltable = null; + clearAstCache(); + astCache = null; + locations = null; + problems.clear(); + problems = null; + parseErrs.clear(); + parseErrs = null; + path.clear(); + path = null; + failedModules.clear(); + failedModules = null; + unresolvedModules.clear(); + unresolvedModules = null; + builtins = null; + allBindings.clear(); + allBindings = null; + + // Technically this is all that's needed for the garbage collector. + idx = null; + } + + @Override + public String toString() { + return ""; + } +} Index: tests/java/org/python/indexer/data/pkg/mineral/metal/gold.py =================================================================== Index: src/org/python/indexer/ast/NWhile.java =================================================================== --- src/org/python/indexer/ast/NWhile.java (revision 0) +++ src/org/python/indexer/ast/NWhile.java (revision 0) @@ -0,0 +1,55 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; + +public class NWhile extends NNode { + + static final long serialVersionUID = -2419753875936526587L; + + public NNode test; + public NBlock body; + public NBlock orelse; + + public NWhile(NNode test, NBlock body, NBlock orelse) { + this(test, body, orelse, 0, 1); + } + + public NWhile(NNode test, NBlock body, NBlock orelse, int start, int end) { + super(start, end); + this.test = test; + this.body = body; + this.orelse = orelse; + addChildren(test, body, orelse); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(test, s); + if (body != null) { + setType(resolveExpr(body, s)); + } + if (orelse != null) { + addType(resolveExpr(orelse, s)); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(test, v); + visitNode(body, v); + visitNode(orelse, v); + } + } +} Index: tests/java/org/python/indexer/data/yinw/yinw-5.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-5.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-5.py (revision 0) @@ -0,0 +1,3 @@ +a = "hello world !" +b = a.split(' ') +b.reverse() Index: src/org/python/indexer/ast/NReturn.java =================================================================== --- src/org/python/indexer/ast/NReturn.java (revision 0) +++ src/org/python/indexer/ast/NReturn.java (revision 0) @@ -0,0 +1,41 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +public class NReturn extends NNode { + + static final long serialVersionUID = 5795610129307339141L; + + public NNode value; + + public NReturn(NNode n) { + this(n, 0, 1); + } + + public NReturn(NNode n, int start, int end) { + super(start, end); + this.value = n; + addChildren(n); + } + + @Override + public NType resolve(Scope s) throws Exception { + return setType(resolveExpr(value, s)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(value, v); + } + } +} Index: tests/java/org/python/indexer/data/class2.py =================================================================== --- tests/java/org/python/indexer/data/class2.py (revision 0) +++ tests/java/org/python/indexer/data/class2.py (revision 0) @@ -0,0 +1,2 @@ +def hi(msg): + print msg Index: tests/java/org/python/indexer/data/pkg/animal/mammal/dog.py =================================================================== --- tests/java/org/python/indexer/data/pkg/animal/mammal/dog.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/animal/mammal/dog.py (revision 0) @@ -0,0 +1 @@ +bark = "woof!" Index: tests/java/org/python/indexer/data/yinw/yinw-8.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-8.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-8.py (revision 0) @@ -0,0 +1,30 @@ +class A: + a = 1 + +class B: + a = 2 + +def f(o): + print o.a() + +o1 = A() +o2 = B() +f(o1) +f(o2) + + +# locations: +# = [] +# = [] +# = [] +# = [] +# = [] +# = [] +# = [] +# unbound variables: + +# semantic problems: +# unknown variables: +# .a:8:10> = accessing attr of unknown type/unbound variable +# parsing problems: + Index: tests/java/org/python/indexer/data/yinw/yinw-16.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-16.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-16.py (revision 0) @@ -0,0 +1,37 @@ +s = "hello" +s.capitalize() +print s + +x = {} +def f(): + x.clear() + y = x.get('ok') + + + +_cache = {} +_cache_repl = {} + +_pattern_type = type(sre_compile.compile("", 0)) + +_MAXCACHE = 100 + +def _compile(*key): + # internal: compile pattern + cachekey = 'ok' + p = _cache.get(cachekey) + if p is not None: + return p + pattern, flags = key + if isinstance(pattern, _pattern_type): + return pattern + if not sre_compile.isstring(pattern): + raise TypeError, "first argument must be string or compiled pattern" + try: + p = sre_compile.compile(pattern, flags) + except error, v: + raise error, v # invalid expression + if len(_cache) >= _MAXCACHE: + _cache.clear() + _cache[cachekey] = p + return p Index: tests/java/org/python/indexer/data/pkg/other/color/crimson.py =================================================================== --- tests/java/org/python/indexer/data/pkg/other/color/crimson.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/other/color/crimson.py (revision 0) @@ -0,0 +1,3 @@ +from red import * + +print r, g, b Index: src/org/python/indexer/types/NClassType.java =================================================================== --- src/org/python/indexer/types/NClassType.java (revision 0) +++ src/org/python/indexer/types/NClassType.java (revision 0) @@ -0,0 +1,50 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Scope; +import org.python.indexer.Util; + +public class NClassType extends NType { + + private String name; + + public NClassType() { + this("", null); + } + + public NClassType(String name, Scope parent) { + this.name = name; + this.setTable(new Scope(parent, Scope.Type.CLASS)); + if (parent != null) { + this.getTable().setPath(parent.extendPath(name)); + } else { + this.getTable().setPath(name); + } + } + + public NClassType(String name, Scope parent, NClassType superClass) { + this(name, parent); + if (superClass != null) { + addSuper(superClass); + } + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void addSuper(NType sp) { + getTable().addSuper(sp.getTable()); + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + sb.append(name); + } +} Index: tests/java/org/python/indexer/data/pkg/animal/__init__.py =================================================================== --- tests/java/org/python/indexer/data/pkg/animal/__init__.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/animal/__init__.py (revision 0) @@ -0,0 +1,3 @@ +from animaltest import living + +success = True Index: src/org/python/indexer/ast/NBinOp.java =================================================================== --- src/org/python/indexer/ast/NBinOp.java (revision 0) +++ src/org/python/indexer/ast/NBinOp.java (revision 0) @@ -0,0 +1,74 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; +import org.python.indexer.types.NUnionType; + +public class NBinOp extends NNode { + + static final long serialVersionUID = -8961832251782335108L; + + public NNode left; + public NNode right; + public String op; + + public NBinOp(NNode target, NNode value, String op) { + this(target, value, op, 0, 1); + } + + public NBinOp(NNode target, NNode value, String op, int start, int end) { + super(start, end); + this.left = target; + this.right = value; + this.op = op; + addChildren(target, value); + } + + @Override + public NType resolve(Scope s) throws Exception { + NType ltype = null, rtype = null; + if (left != null) { + ltype = resolveExpr(left, s).follow(); + } + if (right != null) { + rtype = resolveExpr(right, s).follow(); + } + + // If either non-null operand is a string, assume the result is a string. + if (ltype == Indexer.idx.builtins.BaseStr || rtype == Indexer.idx.builtins.BaseStr) { + return setType(Indexer.idx.builtins.BaseStr); + } + // If either non-null operand is a number, assume the result is a number. + if (ltype == Indexer.idx.builtins.BaseNum || rtype == Indexer.idx.builtins.BaseNum) { + return setType(Indexer.idx.builtins.BaseNum); + } + + if (ltype == null) { + return setType(rtype == null ? new NUnknownType() : rtype); + } + + if (rtype == null) { + return setType(ltype == null ? new NUnknownType() : ltype); + } + + return setType(NUnionType.union(ltype, rtype)); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(left, v); + visitNode(right, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/animal/reptile/snake.py =================================================================== --- tests/java/org/python/indexer/data/pkg/animal/reptile/snake.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/animal/reptile/snake.py (revision 0) @@ -0,0 +1,9 @@ +import croc + +class Snake: + def eats(self, thing): + return False + +class Python(Snake): + def eats(self, thing): + return isinstance(thing, croc.Gavial) Index: tests/java/org/python/indexer/data/yinw/yinw-4.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-4.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-4.py (revision 0) @@ -0,0 +1,21 @@ +class A: + a = 1 + +class B: + a = 2 + +o1 = (A(), A(), B()) +o2 = o1[1:2] + +for x in o2: + print x.a + + +# locations: +# = [] +# = [] +# = [] +# = [] +# = [] +# = [] +# .a:11:10> = [, ] Index: tests/java/org/python/indexer/data/yinw/yinw-3.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-3.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-3.py (revision 0) @@ -0,0 +1,31 @@ +class A: + a = 1 + +class B: + def baz(self): + self.a = 1 + def aa(me): + me.a = 2 + def foo(self): + return self.a + def bar(self): + return self.a + def __iter__(self): + return self + + +o = B() + +for x in o: + print x.a + + +# locations: +# = [] +# = [] +# = [] +# .b:8:15> = [.b:6:8>] +# = [] +# = [] +# = [] +# .a:13:10> = [] Index: tests/java/org/python/indexer/data/pkg/mineral/__init__.py =================================================================== Index: src/org/python/indexer/ast/NAttribute.java =================================================================== --- src/org/python/indexer/ast/NAttribute.java (revision 0) +++ src/org/python/indexer/ast/NAttribute.java (revision 0) @@ -0,0 +1,172 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +import static org.python.indexer.NBinding.Kind.ATTRIBUTE; + +public class NAttribute extends NNode { + + static final long serialVersionUID = -1120979305017812255L; + + public NNode target; + public NName attr; + + public NAttribute(NNode target, NName attr) { + this(target, attr, 0, 1); + } + + public NAttribute(NNode target, NName attr, int start, int end) { + super(start, end); + setTarget(target); + setAttr(attr); + addChildren(target, attr); + } + + public String getAttributeName() { + return attr.id; + } + + /** + * Sets the attribute node. Used when constructing the AST. + * @throws IllegalArgumentException if the param is null + */ + public void setAttr(NName attr) { + if (attr == null) { + throw new IllegalArgumentException("param cannot be null"); + } + this.attr = attr; + } + + public NName getAttr() { + return attr; + } + + /** + * Sets the target node. Used when constructing the AST. + * @throws IllegalArgumentException if the param is null + */ + public void setTarget(NNode target) { + if (target == null) { + throw new IllegalArgumentException("param cannot be null"); + } + this.target = target; + } + + public NNode getTarget() { + return target; + } + + /** + * Assign some definite value to the attribute. Used during the name + * resolution pass. This method is called when this node is in the lvalue of + * an assignment, in which case it is called in lieu of {@link #resolve}.

    + */ + public void setAttr(Scope s, NType v) throws Exception { + setType(new NUnknownType()); + + NType targetType = resolveExpr(target, s); + if (targetType.isUnionType()) { + targetType = targetType.asUnionType().firstKnownNonNullAlternate(); + if (targetType == null) { + return; + } + } + + targetType = targetType.follow(); + if (targetType == Indexer.idx.builtins.None) { + return; + } + NBinding b = targetType.getTable().putAttr(attr.id, attr, v, ATTRIBUTE); + if (b != null) { + setType(attr.setType(b.followType())); + } + } + + @Override + public NType resolve(Scope s) throws Exception { + setType(new NUnknownType()); + + NType targetType = resolveExpr(target, s); + + if (targetType.isUnionType()) { + NType ret = new NUnknownType(); + for (NType tp : targetType.asUnionType().getTypes()) { + resolveAttributeOnType(tp); + ret = NUnionType.union(ret, getType()); + } + setType(attr.setType(ret.follow())); + } else { + resolveAttributeOnType(targetType); + } + + return getType(); + } + + private void resolveAttributeOnType(NType targetType) { + NType ttype = targetType.follow(); + NBinding b = ttype.getTable().lookupAttr(attr.id); + if (b == null) { + b = makeProvisionalBinding(ttype); + } + if (b != null) { + Indexer.idx.putLocation(attr, b); + setType(attr.setType(b.getType())); + } + } + + /** + * If we can't find the attribute in the target type, create a temp binding + * for the attribute. If later on the target type defines the attribute, + * that definition replaces the temp binding, and any references to the temp + * binding are updated to refer to the new definition. + * + *

    We never create temp bindings for attributes of native types, as + * the attributes of native types are expected to be fully resolved. + * + *

    This strategy is a temporary solution for inferring the types of + * attributes on targets that are not yet resolved. This whole approach + * needs to be replaced by dataflow analysis. + */ + private NBinding makeProvisionalBinding(NType targetType) { + if (targetType.isNative()) { + return null; + } + + Scope targetScope = targetType.getTable(); + + // XXX: Eventually we need to fix out all the cases where the path is + // is empty here. For now, avoid an IndexingException. + if ("".equals(targetScope.getPath())) { + return null; + } + + NType utype = new NUnknownType(); + NBinding b = targetScope.putAttr(attr.id, null, utype, ATTRIBUTE); + if (b != null) { + b.setProvisional(true); + utype.getTable().setPath(b.getQname()); + } + return b; + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(target, v); + visitNode(attr, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/other/color/blue.py =================================================================== --- tests/java/org/python/indexer/data/pkg/other/color/blue.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/other/color/blue.py (revision 0) @@ -0,0 +1,3 @@ +r = 0 +g = 0 +b = 255 Index: tests/java/org/python/indexer/data/pkg/mineral/stone/lapis.py =================================================================== --- tests/java/org/python/indexer/data/pkg/mineral/stone/lapis.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/mineral/stone/lapis.py (revision 0) @@ -0,0 +1,2 @@ + +category = "metamorphic" Index: tests/java/org/python/indexer/data/pkg/plant/poison/eggplant.py =================================================================== --- tests/java/org/python/indexer/data/pkg/plant/poison/eggplant.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/plant/poison/eggplant.py (revision 0) @@ -0,0 +1,9 @@ +""" +"Eggplant causes reflex vomiting." + -- www.veggieboards.com/boards/showthread.php?t=5102 +""" + +facts = {"fact1": ["Eggplant is richer in nicotine than any other edible plant", + "http://en.wikipedia.org/wiki/Eggplant"]} + +adjectives = ["gross", "slimy", "inedible", "nasty"] Index: tests/java/org/python/indexer/data/pkg/plant/poison/__init__.py =================================================================== Index: tests/java/org/python/indexer/data/__init__.py =================================================================== --- tests/java/org/python/indexer/data/__init__.py (revision 0) +++ tests/java/org/python/indexer/data/__init__.py (revision 0) @@ -0,0 +1 @@ +beefwit = [1, 2, 3] Index: tests/java/org/python/indexer/data/pkg/misc/moduleB.py =================================================================== --- tests/java/org/python/indexer/data/pkg/misc/moduleB.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/misc/moduleB.py (revision 0) @@ -0,0 +1,6 @@ +from moduleA import * + +a # bound +b # bound +c # bound +d # should be unbound Index: src/org/python/antlr/ast/ClassDef.java =================================================================== --- src/org/python/antlr/ast/ClassDef.java (revision 6695) +++ src/org/python/antlr/ast/ClassDef.java (working copy) @@ -32,6 +32,10 @@ public String getInternalName() { return name; } + private Name nameNode; + public Name getInternalNameNode() { + return nameNode; + } @ExposedGet(name = "name") public PyObject getName() { if (name == null) return Py.None; @@ -154,6 +158,34 @@ } } + public ClassDef(Token token, Name name, java.util.List bases, java.util.List + body, java.util.List decorator_list) { + super(token); + this.name = name.getText(); + this.nameNode = name; + this.bases = bases; + if (bases == null) { + this.bases = new ArrayList(); + } + for(PythonTree t : this.bases) { + addChild(t); + } + this.body = body; + if (body == null) { + this.body = new ArrayList(); + } + for(PythonTree t : this.body) { + addChild(t); + } + this.decorator_list = decorator_list; + if (decorator_list == null) { + this.decorator_list = new ArrayList(); + } + for(PythonTree t : this.decorator_list) { + addChild(t); + } + } + public ClassDef(Integer ttype, Token token, String name, java.util.List bases, java.util.List body, java.util.List decorator_list) { super(ttype, token); Index: src/org/python/indexer/ast/NAssign.java =================================================================== --- src/org/python/indexer/ast/NAssign.java (revision 0) +++ src/org/python/indexer/ast/NAssign.java (revision 0) @@ -0,0 +1,72 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +public class NAssign extends NNode { + + static final long serialVersionUID = 928890389856851537L; + + public List targets; + public NNode rvalue; + + public NAssign(List targets, NNode rvalue) { + this(targets, rvalue, 0, 1); + } + + public NAssign(List targets, NNode rvalue, int start, int end) { + super(start, end); + this.targets = targets; + this.rvalue = rvalue; + addChildren(targets); + addChildren(rvalue); + } + + @Override + public boolean bindsName() { + return true; + } + + @Override + protected void bindNames(Scope s) throws Exception { + NameBinder binder = NameBinder.make(); + for (NNode target : targets) { + binder.bind(s, target, new NUnknownType()); + } + } + + @Override + public NType resolve(Scope s) throws Exception { + NType valueType = resolveExpr(rvalue, s); + switch (targets.size()) { + case 0: + break; + case 1: + NameBinder.make().bind(s, targets.get(0), valueType); + break; + default: + NameBinder.make().bind(s, targets, valueType); + break; + } + return setType(valueType); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(targets, v); + visitNode(rvalue, v); + } + } +} Index: src/org/python/indexer/demos/DocStringParser.java =================================================================== --- src/org/python/indexer/demos/DocStringParser.java (revision 0) +++ src/org/python/indexer/demos/DocStringParser.java (revision 0) @@ -0,0 +1,159 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.demos; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Ref; +import org.python.indexer.Scope; +import org.python.indexer.StyleRun; +import org.python.indexer.ast.NStr; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Scans doc strings looking for interesting stuff to highlight or hyperlink. + */ +class DocStringParser { + + /** + * Only try to resolve possible qnames of at least this length. + * Helps cut down on noise. + */ + private static final int MIN_TYPE_NAME_LENGTH = 4; + + /** + * Matches an unqualified Python identifier. + */ + private static final String IDENT = "[a-zA-Z_][a-zA-Z0-9_]*"; + + /** + * Matches probable type names. Does loose matching; caller must do more checks. + */ + private static final Pattern TYPE_NAME = + Pattern.compile( + // any two or more identifiers joined with dots. + IDENT + "\\." + IDENT + "(?:\\." + IDENT + ")*\\b" + + // a capitalized word that isn't all-caps + + "|\\b[A-Z][a-zA-Z0-9_]*?[a-z][a-zA-Z0-9_]*\\b" + + // an __identifier__ + + "|(? offsets = new HashSet(); // styles we've already added + private List styles = new ArrayList(); + private Linker linker; + + /** + * Constructor. + * @param start beginning 0-based file offset of the doc string + * @param comment the doc string or doc-comment text + * @param node the AST node for the doc string + */ + public DocStringParser(String comment, NStr node, Linker linker) { + docOffset = node.start(); + docString = comment; + docNode = node; + scope = node.getEnclosingNamespace(); + file = node.getFile(); + this.linker = linker; + } + + /** + * Configures whether to highlight syntactically or semantically. + * + * @param resolve {@code true} to do name resolution, {@code false} + * to guess purely based on syntax in the doc string. + * Pass {@code false} if you're using the highlighter to + * syntax-highlight a file (i.e. no code graph or indexing.) + */ + public void setResolveReferences(boolean resolve) { + resolveReferences = resolve; + } + + public boolean isResolvingReferences() { + return resolveReferences; + } + + /** + * Main entry point. + * + * @return the non-{@code null} but possibly empty list of additional + * styles for the doc string. + */ + public List highlight() { + if (resolveReferences) { + scanCommentForTypeNames(); + } + + return styles; + } + + /** + * Try to match potential type names against the code graph. + * If any match, graph references and styles are added for them. + */ + private void scanCommentForTypeNames() { + Matcher m = TYPE_NAME.matcher(docString); + while (m.find()) { + String qname = m.group(); + int beg = m.start() + docOffset; + + // If we already added a style here, skip this one. + if (offsets.contains(beg)) { + continue; + } + + // Arbitrarily require them to be at least N chars, to reduce noise. + if (qname.length() < MIN_TYPE_NAME_LENGTH) { + continue; + } + + checkForReference(beg, qname); + } + } + + /** + * Look for the name in the current scope. If found, and its + * qname is a valid binding in the graph, record a reference. + */ + private void checkForReference(int offset, String qname) { + NBinding nb; + if (qname.indexOf('.') == -1) { + nb = scope.lookup(qname); + if (nb == null) { + nb = Indexer.idx.globaltable.lookup(qname); + } + } else { + nb = Indexer.idx.lookupQname(qname); + } + + if (nb != null) { + linker.processRef(new Ref(file, offset, qname), nb); + } + } + + private void addStyle(int beg, int len, NBinding nb) { + addStyle(beg, len, StyleRun.Type.TYPE_NAME); + offsets.add(beg); + } + + private void addStyle(int beg, int len, StyleRun.Type type) { + styles.add(new StyleRun(type, beg, len)); + offsets.add(beg); + } +} Index: src/org/python/indexer/Builtins.java =================================================================== --- src/org/python/indexer/Builtins.java (revision 0) +++ src/org/python/indexer/Builtins.java (revision 0) @@ -0,0 +1,2278 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer; + +import org.python.antlr.base.mod; +import org.python.indexer.ast.NUrl; +import org.python.indexer.types.NClassType; +import org.python.indexer.types.NDictType; +import org.python.indexer.types.NFuncType; +import org.python.indexer.types.NInstanceType; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NTupleType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.python.indexer.NBinding.Kind.ATTRIBUTE; +import static org.python.indexer.NBinding.Kind.CLASS; +import static org.python.indexer.NBinding.Kind.CONSTRUCTOR; +import static org.python.indexer.NBinding.Kind.FUNCTION; +import static org.python.indexer.NBinding.Kind.METHOD; +import static org.python.indexer.NBinding.Kind.MODULE; + +/** + * Initializes the built-in types, functions and modules. + * This approach was easy (if tedious) to implement, but longer-term + * it would be better to define these type signatures in python + * "externs files", using a standard type annotation syntax. + */ +public class Builtins { + + public static final String LIBRARY_URL = "http://docs.python.org/library/"; + public static final String TUTORIAL_URL = "http://docs.python.org/tutorial/"; + public static final String REFERENCE_URL = "http://docs.python.org/reference/"; + public static final String DATAMODEL_URL = "http://docs.python.org/reference/datamodel#"; + + public static NUrl newLibUrl(String module, String name) { + return newLibUrl(module + ".html#" + name); + } + + public static NUrl newLibUrl(String path) { + if (!path.endsWith(".html")) { + path += ".html"; + } + return new NUrl(LIBRARY_URL + path); + } + + public static NUrl newRefUrl(String path) { + return new NUrl(REFERENCE_URL + path); + } + + public static NUrl newDataModelUrl(String path) { + return new NUrl(DATAMODEL_URL + path); + } + + public static NUrl newTutUrl(String path) { + return new NUrl(TUTORIAL_URL + path); + } + + // XXX: need to model "types" module and reconcile with these types + public NModuleType Builtin; + public NClassType Object; + public NClassType Type; + public NClassType None; + public NClassType BaseNum; // BaseNum models int, float and long + public NClassType BaseComplex; + public NClassType BaseBool; + public NClassType BaseStr; + public NClassType BaseList; + public NClassType BaseArray; + public NClassType BaseDict; + public NClassType BaseTuple; + public NClassType BaseModule; + public NClassType BaseFile; + public NClassType BaseException; + public NClassType BaseStruct; + public NClassType BaseFunction; // models functions, lambas and methods + public NClassType BaseClass; // models classes and instances + + public NClassType Datetime_datetime; + public NClassType Datetime_date; + public NClassType Datetime_time; + public NClassType Datetime_timedelta; + public NClassType Datetime_tzinfo; + public NClassType Time_struct_time; + + Scope globaltable; + Scope moduleTable; + + String[] builtin_exception_types = { + "ArithmeticError", "AssertionError", "AttributeError", + "BaseException", "Exception", "DeprecationWarning", "EOFError", + "EnvironmentError", "FloatingPointError", "FutureWarning", + "GeneratorExit", "IOError", "ImportError", "ImportWarning", + "IndentationError", "IndexError", "KeyError", "KeyboardInterrupt", + "LookupError", "MemoryError", "NameError", "NotImplemented", + "NotImplementedError", "OSError", "OverflowError", + "PendingDeprecationWarning", "ReferenceError", "RuntimeError", + "RuntimeWarning", "StandardError", "StopIteration", "SyntaxError", + "SyntaxWarning", "SystemError", "SystemExit", "TabError", + "TypeError", "UnboundLocalError", "UnicodeDecodeError", + "UnicodeEncodeError", "UnicodeError", "UnicodeTranslateError", + "UnicodeWarning", "UserWarning", "ValueError", "Warning", + "ZeroDivisionError" + }; + + Set nativeTypes = new HashSet(); + + NClassType newClass(String name, Scope table) { + return newClass(name, table, null); + } + + NClassType newClass(String name, Scope table, + NClassType superClass, NClassType... moreSupers) { + NClassType t = new NClassType(name, table, superClass); + for (NClassType c : moreSupers) { + t.addSuper(c); + } + nativeTypes.add(t); + return t; + } + + NModuleType newModule(String name) { + NModuleType mt = new NModuleType(name, null, globaltable); + nativeTypes.add(mt); + return mt; + } + + NUnknownType unknown() { + NUnknownType t = new NUnknownType(); + nativeTypes.add(t); + return t; + } + + NClassType newException(String name, Scope t) { + return newClass(name, t, BaseException); + } + + NFuncType newFunc() { + NFuncType t = new NFuncType(); + nativeTypes.add(t); + return t; + } + + NFuncType newFunc(NType type) { + NFuncType t = new NFuncType(type); + nativeTypes.add(t); + return t; + } + + NListType newList() { + return newList(unknown()); + } + + NListType newList(NType type) { + NListType t = new NListType(type); + nativeTypes.add(t); + return t; + } + + NDictType newDict(NType ktype, NType vtype) { + NDictType t = new NDictType(ktype, vtype); + nativeTypes.add(t); + return t; + } + + NTupleType newTuple(NType... types) { + NTupleType t = new NTupleType(types); + nativeTypes.add(t); + return t; + } + + NUnionType newUnion(NType... types) { + NUnionType t = new NUnionType(types); + nativeTypes.add(t); + return t; + } + + String[] list(String... names) { + return names; + } + + private abstract class NativeModule { + + protected String name; + protected NModuleType module; + protected Scope table; // the module's symbol table + + NativeModule(String name) { + this.name = name; + modules.put(name, this); + } + + /** Lazily load the module. */ + NModuleType getModule() { + if (module == null) { + createModuleType(); + initBindings(); + } + return module; + } + + protected abstract void initBindings(); + + protected void createModuleType() { + if (module == null) { + module = newModule(name); + table = module.getTable(); + moduleTable.put(name, liburl(), module, MODULE); + } + } + + protected NBinding update(String name, NUrl url, NType type, NBinding.Kind kind) { + return table.update(name, url, type, kind); + } + + protected NBinding addClass(String name, NUrl url, NType type) { + return table.update(name, url, type, CLASS); + } + + protected NBinding addMethod(String name, NUrl url, NType type) { + return table.update(name, url, type, METHOD); + } + + protected NBinding addFunction(String name, NUrl url, NType type) { + return table.update(name, url, newFunc(type), FUNCTION); + } + + // don't use this unless you're sure it's OK to share the type object + protected void addFunctions_beCareful(NType type, String... names) { + for (String name : names) { + addFunction(name, liburl(), type); + } + } + + protected void addNoneFuncs(String... names) { + addFunctions_beCareful(None, names); + } + + protected void addNumFuncs(String... names) { + addFunctions_beCareful(BaseNum, names); + } + + protected void addStrFuncs(String... names) { + addFunctions_beCareful(BaseStr, names); + } + + protected void addUnknownFuncs(String... names) { + for (String name : names) { + addFunction(name, liburl(), unknown()); + } + } + + protected NBinding addAttr(String name, NUrl url, NType type) { + return table.update(name, url, type, ATTRIBUTE); + } + + // don't use this unless you're sure it's OK to share the type object + protected void addAttributes_beCareful(NType type, String... names) { + for (String name : names) { + addAttr(name, liburl(), type); + } + } + + protected void addNumAttrs(String... names) { + addAttributes_beCareful(BaseNum, names); + } + + protected void addStrAttrs(String... names) { + addAttributes_beCareful(BaseStr, names); + } + + protected void addUnknownAttrs(String... names) { + for (String name : names) { + addAttr(name, liburl(), unknown()); + } + } + + protected NUrl liburl() { + return newLibUrl(name); + } + + protected NUrl liburl(String anchor) { + return newLibUrl(name, anchor); + } + + @Override + public String toString() { + return module == null + ? "" + : ""; + } + } + + /** + * The set of top-level native modules. + */ + private Map modules = new HashMap(); + + public Builtins(Scope globals, Scope modules) { + globaltable = globals; + moduleTable = modules; + buildTypes(); + } + + private void buildTypes() { + new BuiltinsModule(); + Scope bt = Builtin.getTable(); + + Object = newClass("object", bt); + None = newClass("None", bt); + Type = newClass("type", bt, Object); + BaseTuple = newClass("tuple", bt, Object); + BaseList = newClass("list", bt, Object); + BaseArray = newClass("array", bt); + BaseDict = newClass("dict", bt, Object); + BaseNum = newClass("float", bt, Object); + BaseComplex = newClass("complex", bt, Object); + BaseBool = newClass("bool", bt, BaseNum); // XXX: useful? + BaseStr = newClass("str", bt, Object); + BaseModule = newClass("module", bt); + BaseFile = newClass("file", bt, Object); + BaseFunction = newClass("function", bt, Object); + BaseClass = newClass("classobj", bt, Object); + } + + void init() { + buildObjectType(); + buildTupleType(); + buildArrayType(); + buildListType(); + buildDictType(); + buildNumTypes(); + buildStrType(); + buildModuleType(); + buildFileType(); + buildFunctionType(); + buildClassType(); + + modules.get("__builtin__").initBindings(); // eagerly load these bindings + + new ArrayModule(); + new AudioopModule(); + new BinasciiModule(); + new Bz2Module(); + new CPickleModule(); + new CStringIOModule(); + new CMathModule(); + new CollectionsModule(); + new CryptModule(); + new CTypesModule(); + new DatetimeModule(); + new DbmModule(); + new ErrnoModule(); + new ExceptionsModule(); + new FcntlModule(); + new FpectlModule(); + new GcModule(); + new GdbmModule(); + new GrpModule(); + new ImpModule(); + new ItertoolsModule(); + new MarshalModule(); + new MathModule(); + new Md5Module(); + new MmapModule(); + new NisModule(); + new OperatorModule(); + new OsModule(); + new ParserModule(); + new PosixModule(); + new PwdModule(); + new PyexpatModule(); + new ReadlineModule(); + new ResourceModule(); + new SelectModule(); + new SignalModule(); + new ShaModule(); + new SpwdModule(); + new StropModule(); + new StructModule(); + new SysModule(); + new SyslogModule(); + new TermiosModule(); + new ThreadModule(); + new TimeModule(); + new UnicodedataModule(); + new ZipimportModule(); + new ZlibModule(); + } + + /** + * Loads (if necessary) and returns the specified built-in module. + */ + public NModuleType get(String name) { + if (name.indexOf(".") == -1) { // unqualified + return getModule(name); + } + + String[] mods = name.split("\\."); + NType type = getModule(mods[0]); + if (type == null) { + return null; + } + for (int i = 1; i < mods.length; i++) { + type = type.getTable().lookupType(mods[i]); + if (!(type instanceof NModuleType)) { + return null; + } + } + return (NModuleType)type; + } + + private NModuleType getModule(String name) { + NativeModule wrap = modules.get(name); + return wrap == null ? null : wrap.getModule(); + } + + public boolean isNative(NType type) { + return nativeTypes.contains(type); + } + + void buildObjectType() { + String[] obj_methods = { + "__delattr__", "__format__", "__getattribute__", "__hash__", + "__init__", "__new__", "__reduce__", "__reduce_ex__", + "__repr__", "__setattr__", "__sizeof__", "__str__", "__subclasshook__" + }; + for (String m : obj_methods) { + Object.getTable().update(m, newLibUrl("stdtypes"), newFunc(), METHOD); + } + Object.getTable().update("__doc__", newLibUrl("stdtypes"), BaseStr, CLASS); + Object.getTable().update("__class__", newLibUrl("stdtypes"), unknown(), CLASS); + } + + void buildTupleType() { + Scope bt = BaseTuple.getTable(); + String[] tuple_methods = { + "__add__", "__contains__", "__eq__", "__ge__", "__getnewargs__", + "__gt__", "__iter__", "__le__", "__len__", "__lt__", "__mul__", + "__ne__", "__new__", "__rmul__", "count", "index" + }; + for (String m : tuple_methods) { + bt.update(m, newLibUrl("stdtypes"), newFunc(), METHOD); + } + NBinding b = bt.update("__getslice__", newDataModelUrl("object.__getslice__"), + newFunc(), METHOD); + b.markDeprecated(); + bt.update("__getitem__", newDataModelUrl("object.__getitem__"), newFunc(), METHOD); + bt.update("__iter__", newDataModelUrl("object.__iter__"), newFunc(), METHOD); + } + + void buildArrayType() { + String[] array_methods_none = { + "append", "buffer_info", "byteswap", "extend", "fromfile", + "fromlist", "fromstring", "fromunicode", "index", "insert", "pop", + "read", "remove", "reverse", "tofile", "tolist", "typecode", "write" + }; + for (String m : array_methods_none) { + BaseArray.getTable().update(m, newLibUrl("array"), newFunc(None), METHOD); + } + String[] array_methods_num = { "count", "itemsize", }; + for (String m : array_methods_num) { + BaseArray.getTable().update(m, newLibUrl("array"), newFunc(BaseNum), METHOD); + } + String[] array_methods_str = { "tostring", "tounicode", }; + for (String m : array_methods_str) { + BaseArray.getTable().update(m, newLibUrl("array"), newFunc(BaseStr), METHOD); + } + } + + void buildListType() { + BaseList.getTable().update("__getslice__", newDataModelUrl("object.__getslice__"), + newFunc(BaseList), METHOD); + BaseList.getTable().update("__getitem__", newDataModelUrl("object.__getitem__"), + newFunc(BaseList), METHOD); + BaseList.getTable().update("__iter__", newDataModelUrl("object.__iter__"), + newFunc(BaseList), METHOD); + + String[] list_methods_none = { + "append", "extend", "index", "insert", "pop", "remove", "reverse", "sort" + }; + for (String m : list_methods_none) { + BaseList.getTable().update(m, newLibUrl("stdtypes"), newFunc(None), METHOD); + } + String[] list_methods_num = { "count" }; + for (String m : list_methods_num) { + BaseList.getTable().update(m, newLibUrl("stdtypes"), newFunc(BaseNum), METHOD); + } + } + + NUrl numUrl() { + return newLibUrl("stdtypes", "typesnumeric"); + } + + void buildNumTypes() { + Scope bnt = BaseNum.getTable(); + String[] num_methods_num = { + "__abs__", "__add__", "__coerce__", "__div__", "__divmod__", + "__eq__", "__float__", "__floordiv__", "__format__", + "__ge__", "__getformat__", "__gt__", "__int__", + "__le__", "__long__", "__lt__", "__mod__", "__mul__", "__ne__", + "__neg__", "__new__", "__nonzero__", "__pos__", "__pow__", + "__radd__", "__rdiv__", "__rdivmod__", "__rfloordiv__", "__rmod__", + "__rmul__", "__rpow__", "__rsub__", "__rtruediv__", "__setformat__", + "__sub__", "__truediv__", "__trunc__", "as_integer_ratio", + "fromhex", "is_integer" + }; + for (String m : num_methods_num) { + bnt.update(m, numUrl(), newFunc(BaseNum), METHOD); + } + bnt.update("__getnewargs__", numUrl(), newFunc(newTuple(BaseNum)), METHOD); + bnt.update("hex", numUrl(), newFunc(BaseStr), METHOD); + bnt.update("conjugate", numUrl(), newFunc(BaseComplex), METHOD); + + Scope bct = BaseComplex.getTable(); + String[] complex_methods = { + "__abs__", "__add__", "__div__", "__divmod__", + "__float__", "__floordiv__", "__format__", "__getformat__", "__int__", + "__long__", "__mod__", "__mul__", "__neg__", "__new__", + "__pos__", "__pow__", "__radd__", "__rdiv__", "__rdivmod__", + "__rfloordiv__", "__rmod__", "__rmul__", "__rpow__", "__rsub__", + "__rtruediv__", "__sub__", "__truediv__", "conjugate" + }; + for (String c : complex_methods) { + bct.update(c, numUrl(), newFunc(BaseComplex), METHOD); + } + String[] complex_methods_num = { + "__eq__", "__ge__", "__gt__", "__le__","__lt__", "__ne__", + "__nonzero__", "__coerce__" + }; + for (String cn : complex_methods_num) { + bct.update(cn, numUrl(), newFunc(BaseNum), METHOD); + } + bct.update("__getnewargs__", numUrl(), newFunc(newTuple(BaseComplex)), METHOD); + bct.update("imag", numUrl(), BaseNum, ATTRIBUTE); + bct.update("real", numUrl(), BaseNum, ATTRIBUTE); + } + + void buildStrType() { + BaseStr.getTable().update("__getslice__", newDataModelUrl("object.__getslice__"), + newFunc(BaseStr), METHOD); + BaseStr.getTable().update("__getitem__", newDataModelUrl("object.__getitem__"), + newFunc(BaseStr), METHOD); + BaseStr.getTable().update("__iter__", newDataModelUrl("object.__iter__"), + newFunc(BaseStr), METHOD); + + String[] str_methods_str = { + "capitalize", "center", "decode", "encode", "expandtabs", "format", + "index", "join", "ljust", "lower", "lstrip", "partition", "replace", + "rfind", "rindex", "rjust", "rpartition", "rsplit", "rstrip", + "strip", "swapcase", "title", "translate", "upper", "zfill" + }; + for (String m : str_methods_str) { + BaseStr.getTable().update(m, newLibUrl("stdtypes.html#str." + m), + newFunc(BaseStr), METHOD); + } + + String[] str_methods_num = { + "count", "isalnum", "isalpha", "isdigit", "islower", "isspace", + "istitle", "isupper", "find", "startswith", "endswith" + }; + for (String m : str_methods_num) { + BaseStr.getTable().update(m, newLibUrl("stdtypes.html#str." + m), + newFunc(BaseNum), METHOD); + } + + String[] str_methods_list = { "split", "splitlines" }; + for (String m : str_methods_list) { + BaseStr.getTable().update(m, newLibUrl("stdtypes.html#str." + m), + newFunc(newList(BaseStr)), METHOD); + } + BaseStr.getTable().update("partition", newLibUrl("stdtypes"), + newFunc(newTuple(BaseStr)), METHOD); + } + + void buildModuleType() { + String[] attrs = { "__doc__", "__file__", "__name__", "__package__" }; + for (String m : attrs) { + BaseModule.getTable().update(m, newTutUrl("modules.html"), BaseStr, ATTRIBUTE); + } + BaseModule.getTable().update("__dict__", newLibUrl("stdtypes", "modules"), + newDict(BaseStr, unknown()), ATTRIBUTE); + } + + void buildDictType() { + String url = "datastructures.html#dictionaries"; + Scope bt = BaseDict.getTable(); + + bt.update("__getitem__", newTutUrl(url), newFunc(), METHOD); + bt.update("__iter__", newTutUrl(url), newFunc(), METHOD); + bt.update("get", newTutUrl(url), newFunc(), METHOD); + + bt.update("items", newTutUrl(url), + newFunc(newList(newTuple(unknown(), unknown()))), METHOD); + + bt.update("keys", newTutUrl(url), newFunc(BaseList), METHOD); + bt.update("values", newTutUrl(url), newFunc(BaseList), METHOD); + + String[] dict_method_unknown = { + "clear", "copy", "fromkeys", "get", "iteritems", "iterkeys", + "itervalues", "pop", "popitem", "setdefault", "update" + }; + for (String m : dict_method_unknown) { + bt.update(m, newTutUrl(url), newFunc(), METHOD); + } + + String[] dict_method_num = { "has_key" }; + for (String m : dict_method_num) { + bt.update(m, newTutUrl(url), newFunc(BaseNum), METHOD); + } + } + + void buildFileType() { + String url = "stdtypes.html#bltin-file-objects"; + Scope table = BaseFile.getTable(); + + String[] methods_unknown = { + "__enter__", "__exit__", "__iter__", "flush", "readinto", "truncate" + }; + for (String m : methods_unknown) { + table.update(m, newLibUrl(url), newFunc(), METHOD); + } + + String[] methods_str = { "next", "read", "readline" }; + for (String m : methods_str) { + table.update(m, newLibUrl(url), newFunc(BaseStr), METHOD); + } + + String[] num = { "fileno", "isatty", "tell" }; + for (String m : num) { + table.update(m, newLibUrl(url), newFunc(BaseNum), METHOD); + } + + String[] methods_none = { "close", "seek", "write", "writelines" }; + for (String m : methods_none) { + table.update(m, newLibUrl(url), newFunc(None), METHOD); + } + + table.update("readlines", newLibUrl(url), newFunc(newList(BaseStr)), METHOD); + table.update("xreadlines", newLibUrl(url), newFunc(BaseFile), METHOD); + table.update("closed", newLibUrl(url), BaseNum, ATTRIBUTE); + table.update("encoding", newLibUrl(url), BaseStr, ATTRIBUTE); + table.update("errors", newLibUrl(url), unknown(), ATTRIBUTE); + table.update("mode", newLibUrl(url), BaseNum, ATTRIBUTE); + table.update("name", newLibUrl(url), BaseStr, ATTRIBUTE); + table.update("softspace", newLibUrl(url), BaseNum, ATTRIBUTE); + table.update("newlines", newLibUrl(url), newUnion(BaseStr, newTuple(BaseStr)), ATTRIBUTE); + } + + private NBinding synthetic(Scope table, String n, NUrl url, NType type, NBinding.Kind k) { + NBinding b = table.update(n, url, type, k); + b.markSynthetic(); + return b; + } + + void buildFunctionType() { + Scope t = BaseFunction.getTable(); + + for (String s : list("func_doc", "__doc__", "func_name", "__name__", "__module__")) { + t.update(s, new NUrl(DATAMODEL_URL), BaseStr, ATTRIBUTE); + } + + NBinding b = synthetic(t, "func_closure", new NUrl(DATAMODEL_URL), newTuple(), ATTRIBUTE); + b.markReadOnly(); + + synthetic(t, "func_code", new NUrl(DATAMODEL_URL), unknown(), ATTRIBUTE); + synthetic(t, "func_defaults", new NUrl(DATAMODEL_URL), newTuple(), ATTRIBUTE); + synthetic(t, "func_globals", new NUrl(DATAMODEL_URL), + new NDictType(BaseStr, new NUnknownType()), ATTRIBUTE); + synthetic(t, "func_dict", new NUrl(DATAMODEL_URL), + new NDictType(BaseStr, new NUnknownType()), ATTRIBUTE); + + // Assume any function can become a method, for simplicity. + for (String s : list("__func__", "im_func")) { + synthetic(t, s, new NUrl(DATAMODEL_URL), new NFuncType(), METHOD); + } + } + + // XXX: finish wiring this up. NClassType needs to inherit from it somehow, + // so we can remove the per-instance attributes from NClassDef. + void buildClassType() { + Scope t = BaseClass.getTable(); + + for (String s : list("__name__", "__doc__", "__module__")) { + synthetic(t, s, new NUrl(DATAMODEL_URL), BaseStr, ATTRIBUTE); + } + + synthetic(t, "__dict__", new NUrl(DATAMODEL_URL), + new NDictType(BaseStr, unknown()), ATTRIBUTE); + } + + class BuiltinsModule extends NativeModule { + public BuiltinsModule() { + super("__builtin__"); + Builtin = module = newModule(name); + table = module.getTable(); + } + @Override + public void initBindings() { + moduleTable.put(name, liburl(), module, MODULE); + table.addSuper(BaseModule.getTable()); + + addClass("None", newLibUrl("constants"), None); + addClass("bool", newLibUrl("functions", "bool"), BaseBool); + addClass("complex", newLibUrl("functions", "complex"), BaseComplex); + addClass("dict", newLibUrl("stdtypes", "typesmapping"), BaseDict); + addClass("file", newLibUrl("functions", "file"), BaseFile); + addClass("float", newLibUrl("functions", "float"), BaseNum); + addClass("int", newLibUrl("functions", "int"), BaseNum); + addClass("list", newLibUrl("functions", "list"), BaseList); + addClass("long", newLibUrl("functions", "long"), BaseNum); + addClass("object", newLibUrl("functions", "object"), Object); + addClass("str", newLibUrl("functions", "str"), BaseStr); + addClass("tuple", newLibUrl("functions", "tuple"), BaseTuple); + addClass("type", newLibUrl("functions", "type"), Type); + + // XXX: need to model the following as built-in class types: + // basestring, bool, buffer, frozenset, property, set, slice, + // staticmethod, super and unicode + String[] builtin_func_unknown = { + "apply", "basestring", "callable", "classmethod", + "coerce", "compile", "copyright", "credits", "delattr", "enumerate", + "eval", "execfile", "exit", "filter", "frozenset", "getattr", + "help", "input", "int", "intern", "iter", "license", "long", + "property", "quit", "raw_input", "reduce", "reload", "reversed", + "set", "setattr", "slice", "sorted", "staticmethod", "super", + "type", "unichr", "unicode", + }; + for (String f : builtin_func_unknown) { + addFunction(f, newLibUrl("functions.html#" + f), unknown()); + } + + String[] builtin_func_num = { + "abs", "all", "any", "cmp", "coerce", "divmod", + "hasattr", "hash", "id", "isinstance", "issubclass", "len", "max", + "min", "ord", "pow", "round", "sum" + }; + for (String f : builtin_func_num) { + addFunction(f, newLibUrl("functions.html#" + f), BaseNum); + } + + for (String f : list("hex", "oct", "repr", "chr")) { + addFunction(f, newLibUrl("functions.html#" + f), BaseStr); + } + + addFunction("dir", newLibUrl("functions", "dir"), newList(BaseStr)); + addFunction("map", newLibUrl("functions", "map"), newList(unknown())); + addFunction("range", newLibUrl("functions", "range"), newList(BaseNum)); + addFunction("xrange", newLibUrl("functions", "range"), newList(BaseNum)); + addFunction("buffer", newLibUrl("functions", "buffer"), newList(unknown())); + addFunction("zip", newLibUrl("functions", "zip"), newList(newTuple(unknown()))); + + + for (String f : list("globals", "vars", "locals")) { + addFunction(f, newLibUrl("functions.html#" + f), newDict(BaseStr, unknown())); + } + + for (String f : builtin_exception_types) { + addClass(f, newDataModelUrl("types"), newClass(f, globaltable, Object)); + } + BaseException = (NClassType)table.lookup("BaseException").getType(); + + for (String f : list("True", "False", "None", "Ellipsis")) { + addAttr(f, newDataModelUrl("types"), unknown()); + } + + addFunction("open", newTutUrl("inputoutput.html#reading-and-writing-files"), BaseFile); + + addFunction("__import__", newLibUrl("functions"), newModule("")); + + globaltable.put("__builtins__", liburl(), module, ATTRIBUTE); + globaltable.merge(table); + } + } + + class ArrayModule extends NativeModule { + public ArrayModule() { + super("array"); + } + @Override + public void initBindings() { + addClass("array", newLibUrl("array", "array"), BaseArray); + addClass("ArrayType", newLibUrl("array", "ArrayType"), BaseArray); + } + } + + class AudioopModule extends NativeModule { + public AudioopModule() { + super("audioop"); + } + @Override + public void initBindings() { + addClass("error", liburl(), newException("error", table)); + + addStrFuncs("add", "adpcm2lin", "alaw2lin", "bias", "lin2alaw", "lin2lin", + "lin2ulaw", "mul", "reverse", "tomono", "ulaw2lin"); + + addNumFuncs("avg", "avgpp", "cross", "findfactor", "findmax", + "getsample", "max", "maxpp", "rms"); + + for (String s : list("adpcm2lin", "findfit", "lin2adpcm", "minmax", "ratecv")) { + addFunction(s, liburl(), newTuple()); + } + } + } + + class BinasciiModule extends NativeModule { + public BinasciiModule() { + super("binascii"); + } + @Override + public void initBindings() { + addStrFuncs( + "a2b_uu", "b2a_uu", "a2b_base64", "b2a_base64", "a2b_qp", + "b2a_qp", "a2b_hqx", "rledecode_hqx", "rlecode_hqx", "b2a_hqx", + "b2a_hex", "hexlify", "a2b_hex", "unhexlify"); + + addNumFuncs("crc_hqx", "crc32"); + + addClass("Error", liburl(), newException("Error", table)); + addClass("Incomplete", liburl(), newException("Incomplete", table)); + } + } + + class Bz2Module extends NativeModule { + public Bz2Module() { + super("bz2"); + } + @Override + public void initBindings() { + NClassType bz2 = newClass("BZ2File", table, BaseFile); // close enough. + addClass("BZ2File", liburl(), bz2); + + NClassType bz2c = newClass("BZ2Compressor", table, Object); + bz2c.getTable().update("compress", newLibUrl("bz2", "sequential-de-compression"), + newFunc(BaseStr), METHOD); + bz2c.getTable().update("flush", newLibUrl("bz2", "sequential-de-compression"), + newFunc(None), METHOD); + addClass("BZ2Compressor", newLibUrl("bz2", "sequential-de-compression"), bz2c); + + NClassType bz2d = newClass("BZ2Decompressor", table, Object); + bz2d.getTable().update("decompress", newLibUrl("bz2", "sequential-de-compression"), + newFunc(BaseStr), METHOD); + addClass("BZ2Decompressor", newLibUrl("bz2", "sequential-de-compression"), bz2d); + + addFunction("compress", newLibUrl("bz2", "one-shot-de-compression"), BaseStr); + addFunction("decompress", newLibUrl("bz2", "one-shot-de-compression"), BaseStr); + } + } + + class CPickleModule extends NativeModule { + public CPickleModule() { + super("cPickle"); + } + @Override + protected NUrl liburl() { + return newLibUrl("pickle", "module-cPickle"); + } + @Override + public void initBindings() { + addUnknownFuncs("dump", "load", "dumps", "loads"); + + addClass("PickleError", liburl(), newException("PickleError", table)); + + NClassType picklingError = newException("PicklingError", table); + addClass("PicklingError", liburl(), picklingError); + update("UnpickleableError", liburl(), + newClass("UnpickleableError", table, picklingError), CLASS); + NClassType unpicklingError = newException("UnpicklingError", table); + addClass("UnpicklingError", liburl(), unpicklingError); + update("BadPickleGet", liburl(), + newClass("BadPickleGet", table, unpicklingError), CLASS); + + NClassType pickler = newClass("Pickler", table, Object); + pickler.getTable().update("dump", liburl(), newFunc(), METHOD); + pickler.getTable().update("clear_memo", liburl(), newFunc(), METHOD); + addClass("Pickler", liburl(), pickler); + + NClassType unpickler = newClass("Unpickler", table, Object); + unpickler.getTable().update("load", liburl(), newFunc(), METHOD); + unpickler.getTable().update("noload", liburl(), newFunc(), METHOD); + addClass("Unpickler", liburl(), unpickler); + } + } + + class CStringIOModule extends NativeModule { + public CStringIOModule() { + super("cStringIO"); + } + @Override + public void initBindings() { + NClassType StringIO = newClass("StringIO", table, BaseFile); + addFunction("StringIO", liburl(), StringIO); + addAttr("InputType", liburl(), Type); + addAttr("OutputType", liburl(), Type); + addAttr("cStringIO_CAPI", liburl(), unknown()); + } + } + + class CMathModule extends NativeModule { + public CMathModule() { + super("cmath"); + } + @Override + public void initBindings() { + addFunction("phase", liburl("conversions-to-and-from-polar-coordinates"), BaseNum); + addFunction("polar", liburl("conversions-to-and-from-polar-coordinates"), + newTuple(BaseNum, BaseNum)); + addFunction("rect", liburl("conversions-to-and-from-polar-coordinates"), + BaseComplex); + + for (String plf : list("exp", "log", "log10", "sqrt")) { + addFunction(plf, liburl("power-and-logarithmic-functions"), BaseNum); + } + + for (String tf : list("acos", "asin", "atan", "cos", "sin", "tan")) { + addFunction(tf, liburl("trigonometric-functions"), BaseNum); + } + + for (String hf : list("acosh", "asinh", "atanh", "cosh", "sinh", "tanh")) { + addFunction(hf, liburl("hyperbolic-functions"), BaseComplex); + } + + for (String cf : list("isinf", "isnan")) { + addFunction(cf, liburl("classification-functions"), BaseBool); + } + + for (String c : list("pi", "e")) { + addAttr(c, liburl("constants"), BaseNum); + } + } + } + + class CollectionsModule extends NativeModule { + public CollectionsModule() { + super("collections"); + } + private NUrl abcUrl() { + return liburl("abcs-abstract-base-classes"); + } + private NUrl dequeUrl() { + return liburl("deque-objects"); + } + @Override + public void initBindings() { + NClassType Callable = newClass("Callable", table, Object); + Callable.getTable().update("__call__", abcUrl(), newFunc(), METHOD); + addClass("Callable", abcUrl(), Callable); + + NClassType Iterable = newClass("Iterable", table, Object); + Iterable.getTable().update("__next__", abcUrl(), newFunc(), METHOD); + Iterable.getTable().update("__iter__", abcUrl(), newFunc(), METHOD); + addClass("Iterable", abcUrl(), Iterable); + + NClassType Hashable = newClass("Hashable", table, Object); + Hashable.getTable().update("__hash__", abcUrl(), newFunc(BaseNum), METHOD); + addClass("Hashable", abcUrl(), Hashable); + + NClassType Sized = newClass("Sized", table, Object); + Sized.getTable().update("__len__", abcUrl(), newFunc(BaseNum), METHOD); + addClass("Sized", abcUrl(), Sized); + + NClassType Container = newClass("Container", table, Object); + Container.getTable().update("__contains__", abcUrl(), newFunc(BaseNum), METHOD); + addClass("Container", abcUrl(), Container); + + NClassType Iterator = newClass("Iterator", table, Iterable); + addClass("Iterator", abcUrl(), Iterator); + + NClassType Sequence = newClass("Sequence", table, Sized, Iterable, Container); + Sequence.getTable().update("__getitem__", abcUrl(), newFunc(), METHOD); + Sequence.getTable().update("reversed", abcUrl(), newFunc(Sequence), METHOD); + Sequence.getTable().update("index", abcUrl(), newFunc(BaseNum), METHOD); + Sequence.getTable().update("count", abcUrl(), newFunc(BaseNum), METHOD); + addClass("Sequence", abcUrl(), Sequence); + + NClassType MutableSequence = newClass("MutableSequence", table, Sequence); + MutableSequence.getTable().update("__setitem__", abcUrl(), newFunc(), METHOD); + MutableSequence.getTable().update("__delitem__", abcUrl(), newFunc(), METHOD); + addClass("MutableSequence", abcUrl(), MutableSequence); + + NClassType Set = newClass("Set", table, Sized, Iterable, Container); + Set.getTable().update("__getitem__", abcUrl(), newFunc(), METHOD); + addClass("Set", abcUrl(), Set); + + NClassType MutableSet = newClass("MutableSet", table, Set); + MutableSet.getTable().update("add", abcUrl(), newFunc(), METHOD); + MutableSet.getTable().update("discard", abcUrl(), newFunc(), METHOD); + addClass("MutableSet", abcUrl(), MutableSet); + + NClassType Mapping = newClass("Mapping", table, Sized, Iterable, Container); + Mapping.getTable().update("__getitem__", abcUrl(), newFunc(), METHOD); + addClass("Mapping", abcUrl(), Mapping); + + NClassType MutableMapping = newClass("MutableMapping", table, Mapping); + MutableMapping.getTable().update("__setitem__", abcUrl(), newFunc(), METHOD); + MutableMapping.getTable().update("__delitem__", abcUrl(), newFunc(), METHOD); + addClass("MutableMapping", abcUrl(), MutableMapping); + + NClassType MappingView = newClass("MappingView", table, Sized); + addClass("MappingView", abcUrl(), MappingView); + + NClassType KeysView = newClass("KeysView", table, Sized); + addClass("KeysView", abcUrl(), KeysView); + + NClassType ItemsView = newClass("ItemsView", table, Sized); + addClass("ItemsView", abcUrl(), ItemsView); + + NClassType ValuesView = newClass("ValuesView", table, Sized); + addClass("ValuesView", abcUrl(), ValuesView); + + NClassType deque = newClass("deque", table, Object); + for (String n : list("append", "appendLeft", "clear", + "extend", "extendLeft", "rotate")) { + deque.getTable().update(n, dequeUrl(), newFunc(None), METHOD); + } + for (String u : list("__getitem__", "__iter__", + "pop", "popleft", "remove")) { + deque.getTable().update(u, dequeUrl(), newFunc(), METHOD); + } + addClass("deque", dequeUrl(), deque); + + NClassType defaultdict = newClass("defaultdict", table, Object); + defaultdict.getTable().update("__missing__", liburl("defaultdict-objects"), + newFunc(), METHOD); + defaultdict.getTable().update("default_factory", liburl("defaultdict-objects"), + newFunc(), METHOD); + addClass("defaultdict", liburl("defaultdict-objects"), defaultdict); + + String argh = "namedtuple-factory-function-for-tuples-with-named-fields"; + NClassType namedtuple = newClass("(namedtuple)", table, BaseTuple); + namedtuple.getTable().update("_fields", liburl(argh), + new NListType(BaseStr), ATTRIBUTE); + addFunction("namedtuple", liburl(argh), namedtuple); + } + } + + class CTypesModule extends NativeModule { + public CTypesModule() { + super("ctypes"); + } + @Override + public void initBindings() { + String[] ctypes_attrs = { + "ARRAY", "ArgumentError", "Array", "BigEndianStructure", "CDLL", + "CFUNCTYPE", "DEFAULT_MODE", "DllCanUnloadNow", "DllGetClassObject", + "FormatError", "GetLastError", "HRESULT", "LibraryLoader", + "LittleEndianStructure", "OleDLL", "POINTER", "PYFUNCTYPE", "PyDLL", + "RTLD_GLOBAL", "RTLD_LOCAL", "SetPointerType", "Structure", "Union", + "WINFUNCTYPE", "WinDLL", "WinError", "_CFuncPtr", "_FUNCFLAG_CDECL", + "_FUNCFLAG_PYTHONAPI", "_FUNCFLAG_STDCALL", "_FUNCFLAG_USE_ERRNO", + "_FUNCFLAG_USE_LASTERROR", "_Pointer", "_SimpleCData", + "_c_functype_cache", "_calcsize", "_cast", "_cast_addr", + "_check_HRESULT", "_check_size", "_ctypes_version", "_dlopen", + "_endian", "_memmove_addr", "_memset_addr", "_os", + "_pointer_type_cache", "_string_at", "_string_at_addr", "_sys", + "_win_functype_cache", "_wstring_at", "_wstring_at_addr", + "addressof", "alignment", "byref", "c_bool", "c_buffer", "c_byte", + "c_char", "c_char_p", "c_double", "c_float", "c_int", "c_int16", + "c_int32", "c_int64", "c_int8", "c_long", "c_longdouble", + "c_longlong", "c_short", "c_size_t", "c_ubyte", "c_uint", + "c_uint16", "c_uint32", "c_uint64", "c_uint8", "c_ulong", + "c_ulonglong", "c_ushort", "c_void_p", "c_voidp", "c_wchar", + "c_wchar_p", "cast", "cdll", "create_string_buffer", + "create_unicode_buffer", "get_errno", "get_last_error", "memmove", + "memset", "oledll", "pointer", "py_object", "pydll", "pythonapi", + "resize", "set_conversion_mode", "set_errno", "set_last_error", + "sizeof", "string_at", "windll", "wstring_at" + }; + for (String attr : ctypes_attrs) { + addAttr(attr, liburl(attr), unknown()); + } + } + } + + class CryptModule extends NativeModule { + public CryptModule() { + super("crypt"); + } + @Override + public void initBindings() { + addStrFuncs("crypt"); + } + } + + class DatetimeModule extends NativeModule { + public DatetimeModule() { + super("datetime"); + } + private NUrl dtUrl(String anchor) { + return liburl("datetime." + anchor); + } + @Override + public void initBindings() { + // XXX: make datetime, time, date, timedelta and tzinfo Base* objects, + // so built-in functions can return them. + + addNumAttrs("MINYEAR", "MAXYEAR"); + + NClassType timedelta = Datetime_timedelta = newClass("timedelta", table, Object); + addClass("timedelta", dtUrl("timedelta"), timedelta); + Scope tdtable = Datetime_timedelta.getTable(); + tdtable.update("min", dtUrl("timedelta"), timedelta, ATTRIBUTE); + tdtable.update("max", dtUrl("timedelta"), timedelta, ATTRIBUTE); + tdtable.update("resolution", dtUrl("timedelta"), timedelta, ATTRIBUTE); + + tdtable.update("days", dtUrl("timedelta"), BaseNum, ATTRIBUTE); + tdtable.update("seconds", dtUrl("timedelta"), BaseNum, ATTRIBUTE); + tdtable.update("microseconds", dtUrl("timedelta"), BaseNum, ATTRIBUTE); + + NClassType tzinfo = Datetime_tzinfo = newClass("tzinfo", table, Object); + addClass("tzinfo", dtUrl("tzinfo"), tzinfo); + Scope tztable = Datetime_tzinfo.getTable(); + tztable.update("utcoffset", dtUrl("tzinfo"), newFunc(timedelta), METHOD); + tztable.update("dst", dtUrl("tzinfo"), newFunc(timedelta), METHOD); + tztable.update("tzname", dtUrl("tzinfo"), newFunc(BaseStr), METHOD); + tztable.update("fromutc", dtUrl("tzinfo"), newFunc(tzinfo), METHOD); + + NClassType date = Datetime_date = newClass("date", table, Object); + addClass("date", dtUrl("date"), date); + Scope dtable = Datetime_date.getTable(); + dtable.update("min", dtUrl("date"), date, ATTRIBUTE); + dtable.update("max", dtUrl("date"), date, ATTRIBUTE); + dtable.update("resolution", dtUrl("date"), timedelta, ATTRIBUTE); + + dtable.update("today", dtUrl("date"), newFunc(date), METHOD); + dtable.update("fromtimestamp", dtUrl("date"), newFunc(date), METHOD); + dtable.update("fromordinal", dtUrl("date"), newFunc(date), METHOD); + + dtable.update("year", dtUrl("date"), BaseNum, ATTRIBUTE); + dtable.update("month", dtUrl("date"), BaseNum, ATTRIBUTE); + dtable.update("day", dtUrl("date"), BaseNum, ATTRIBUTE); + + dtable.update("replace", dtUrl("date"), newFunc(date), METHOD); + dtable.update("timetuple", dtUrl("date"), newFunc(Time_struct_time), METHOD); + + for (String n : list("toordinal", "weekday", "isoweekday")) { + dtable.update(n, dtUrl("date"), newFunc(BaseNum), METHOD); + } + for (String r : list("ctime", "strftime", "isoformat")) { + dtable.update(r, dtUrl("date"), newFunc(BaseStr), METHOD); + } + dtable.update("isocalendar", dtUrl("date"), + newFunc(newTuple(BaseNum, BaseNum, BaseNum)), METHOD); + + NClassType time = Datetime_time = newClass("time", table, Object); + addClass("time", dtUrl("time"), date); + Scope ttable = Datetime_time.getTable(); + + ttable.update("min", dtUrl("time"), time, ATTRIBUTE); + ttable.update("max", dtUrl("time"), time, ATTRIBUTE); + ttable.update("resolution", dtUrl("time"), timedelta, ATTRIBUTE); + + ttable.update("hour", dtUrl("time"), BaseNum, ATTRIBUTE); + ttable.update("minute", dtUrl("time"), BaseNum, ATTRIBUTE); + ttable.update("second", dtUrl("time"), BaseNum, ATTRIBUTE); + ttable.update("microsecond", dtUrl("time"), BaseNum, ATTRIBUTE); + ttable.update("tzinfo", dtUrl("time"), tzinfo, ATTRIBUTE); + + ttable.update("replace", dtUrl("time"), newFunc(time), METHOD); + + for (String l : list("isoformat", "strftime", "tzname")) { + ttable.update(l, dtUrl("time"), newFunc(BaseStr), METHOD); + } + for (String f : list("utcoffset", "dst")) { + ttable.update(f, dtUrl("time"), newFunc(timedelta), METHOD); + } + + NClassType datetime = Datetime_datetime = newClass("datetime", table, date, time); + addClass("datetime", dtUrl("datetime"), datetime); + Scope dttable = Datetime_datetime.getTable(); + + for (String c : list("combine", "fromordinal", "fromtimestamp", "now", + "strptime", "today", "utcfromtimestamp", "utcnow")) { + dttable.update(c, dtUrl("datetime"), newFunc(datetime), METHOD); + } + + dttable.update("min", dtUrl("datetime"), datetime, ATTRIBUTE); + dttable.update("max", dtUrl("datetime"), datetime, ATTRIBUTE); + dttable.update("resolution", dtUrl("datetime"), timedelta, ATTRIBUTE); + + dttable.update("date", dtUrl("datetime"), newFunc(date), METHOD); + + for (String x : list("time", "timetz")) { + dttable.update(x, dtUrl("datetime"), newFunc(time), METHOD); + } + + for (String y : list("replace", "astimezone")) { + dttable.update(y, dtUrl("datetime"), newFunc(datetime), METHOD); + } + + dttable.update("utctimetuple", dtUrl("datetime"), newFunc(Time_struct_time), METHOD); + } + } + + class DbmModule extends NativeModule { + public DbmModule() { + super("dbm"); + } + @Override + public void initBindings() { + NClassType dbm = new NClassType("dbm", table, BaseDict); + addClass("dbm", liburl(), dbm); + addClass("error", liburl(), newException("error", table)); + addStrAttrs("library"); + addFunction("open", liburl(), dbm); + } + } + + class ErrnoModule extends NativeModule { + public ErrnoModule() { + super("errno"); + } + @Override + public void initBindings() { + addNumAttrs( + "E2BIG", "EACCES", "EADDRINUSE", "EADDRNOTAVAIL", "EAFNOSUPPORT", + "EAGAIN", "EALREADY", "EBADF", "EBUSY", "ECHILD", "ECONNABORTED", + "ECONNREFUSED", "ECONNRESET", "EDEADLK", "EDEADLOCK", + "EDESTADDRREQ", "EDOM", "EDQUOT", "EEXIST", "EFAULT", "EFBIG", + "EHOSTDOWN", "EHOSTUNREACH", "EILSEQ", "EINPROGRESS", "EINTR", + "EINVAL", "EIO", "EISCONN", "EISDIR", "ELOOP", "EMFILE", "EMLINK", + "EMSGSIZE", "ENAMETOOLONG", "ENETDOWN", "ENETRESET", "ENETUNREACH", + "ENFILE", "ENOBUFS", "ENODEV", "ENOENT", "ENOEXEC", "ENOLCK", + "ENOMEM", "ENOPROTOOPT", "ENOSPC", "ENOSYS", "ENOTCONN", "ENOTDIR", + "ENOTEMPTY", "ENOTSOCK", "ENOTTY", "ENXIO", "EOPNOTSUPP", "EPERM", + "EPFNOSUPPORT", "EPIPE", "EPROTONOSUPPORT", "EPROTOTYPE", "ERANGE", + "EREMOTE", "EROFS", "ESHUTDOWN", "ESOCKTNOSUPPORT", "ESPIPE", + "ESRCH", "ESTALE", "ETIMEDOUT", "ETOOMANYREFS", "EUSERS", + "EWOULDBLOCK", "EXDEV", "WSABASEERR", "WSAEACCES", "WSAEADDRINUSE", + "WSAEADDRNOTAVAIL", "WSAEAFNOSUPPORT", "WSAEALREADY", "WSAEBADF", + "WSAECONNABORTED", "WSAECONNREFUSED", "WSAECONNRESET", + "WSAEDESTADDRREQ", "WSAEDISCON", "WSAEDQUOT", "WSAEFAULT", + "WSAEHOSTDOWN", "WSAEHOSTUNREACH", "WSAEINPROGRESS", "WSAEINTR", + "WSAEINVAL", "WSAEISCONN", "WSAELOOP", "WSAEMFILE", "WSAEMSGSIZE", + "WSAENAMETOOLONG", "WSAENETDOWN", "WSAENETRESET", "WSAENETUNREACH", + "WSAENOBUFS", "WSAENOPROTOOPT", "WSAENOTCONN", "WSAENOTEMPTY", + "WSAENOTSOCK", "WSAEOPNOTSUPP", "WSAEPFNOSUPPORT", "WSAEPROCLIM", + "WSAEPROTONOSUPPORT", "WSAEPROTOTYPE", "WSAEREMOTE", "WSAESHUTDOWN", + "WSAESOCKTNOSUPPORT", "WSAESTALE", "WSAETIMEDOUT", + "WSAETOOMANYREFS", "WSAEUSERS", "WSAEWOULDBLOCK", + "WSANOTINITIALISED", "WSASYSNOTREADY", "WSAVERNOTSUPPORTED"); + + addAttr("errorcode", liburl("errorcode"), newDict(BaseNum, BaseStr)); + } + } + + class ExceptionsModule extends NativeModule { + public ExceptionsModule() { + super("exceptions"); + } + @Override + public void initBindings() { + NModuleType builtins = get("__builtin__"); + for (String s : builtin_exception_types) { + NBinding b = builtins.getTable().lookup(s); + table.update(b.getName(), b.getSignatureNode(), b.getType(), b.getKind()); + } + } + } + + class FcntlModule extends NativeModule { + public FcntlModule() { + super("fcntl"); + } + @Override + public void initBindings() { + for (String s : list("fcntl", "ioctl")) { + addFunction(s, liburl(), newUnion(BaseNum, BaseStr)); + } + addNumFuncs("flock"); + addUnknownFuncs("lockf"); + + addNumAttrs( + "DN_ACCESS", "DN_ATTRIB", "DN_CREATE", "DN_DELETE", "DN_MODIFY", + "DN_MULTISHOT", "DN_RENAME", "FASYNC", "FD_CLOEXEC", "F_DUPFD", + "F_EXLCK", "F_GETFD", "F_GETFL", "F_GETLEASE", "F_GETLK", "F_GETLK64", + "F_GETOWN", "F_GETSIG", "F_NOTIFY", "F_RDLCK", "F_SETFD", "F_SETFL", + "F_SETLEASE", "F_SETLK", "F_SETLK64", "F_SETLKW", "F_SETLKW64", + "F_SETOWN", "F_SETSIG", "F_SHLCK", "F_UNLCK", "F_WRLCK", "I_ATMARK", + "I_CANPUT", "I_CKBAND", "I_FDINSERT", "I_FIND", "I_FLUSH", + "I_FLUSHBAND", "I_GETBAND", "I_GETCLTIME", "I_GETSIG", "I_GRDOPT", + "I_GWROPT", "I_LINK", "I_LIST", "I_LOOK", "I_NREAD", "I_PEEK", + "I_PLINK", "I_POP", "I_PUNLINK", "I_PUSH", "I_RECVFD", "I_SENDFD", + "I_SETCLTIME", "I_SETSIG", "I_SRDOPT", "I_STR", "I_SWROPT", + "I_UNLINK", "LOCK_EX", "LOCK_MAND", "LOCK_NB", "LOCK_READ", "LOCK_RW", + "LOCK_SH", "LOCK_UN", "LOCK_WRITE"); + } + } + + class FpectlModule extends NativeModule { + public FpectlModule() { + super("fpectl"); + } + @Override + public void initBindings() { + addNoneFuncs("turnon_sigfpe", "turnoff_sigfpe"); + addClass("FloatingPointError", liburl(), newException("FloatingPointError", table)); + } + } + + class GcModule extends NativeModule { + public GcModule() { + super("gc"); + } + @Override + public void initBindings() { + addNoneFuncs("enable", "disable", "set_debug", "set_threshold"); + addNumFuncs("isenabled", "collect", "get_debug", "get_count", "get_threshold"); + for (String s : list("get_objects", "get_referrers", "get_referents")) { + addFunction(s, liburl(), newList()); + } + addAttr("garbage", liburl(), newList()); + addNumAttrs("DEBUG_STATS", "DEBUG_COLLECTABLE", "DEBUG_UNCOLLECTABLE", + "DEBUG_INSTANCES", "DEBUG_OBJECTS", "DEBUG_SAVEALL", "DEBUG_LEAK"); + } + } + + class GdbmModule extends NativeModule { + public GdbmModule() { + super("gdbm"); + } + @Override + public void initBindings() { + addClass("error", liburl(), newException("error", table)); + + NClassType gdbm = new NClassType("gdbm", table, BaseDict); + gdbm.getTable().update("firstkey", liburl(), newFunc(BaseStr), METHOD); + gdbm.getTable().update("nextkey", liburl(), newFunc(BaseStr), METHOD); + gdbm.getTable().update("reorganize", liburl(), newFunc(None), METHOD); + gdbm.getTable().update("sync", liburl(), newFunc(None), METHOD); + + addFunction("open", liburl(), gdbm); + } + + } + class GrpModule extends NativeModule { + public GrpModule() { + super("grp"); + } + @Override + public void initBindings() { + Builtins.this.get("struct"); + NClassType struct_group = newClass("struct_group", table, BaseStruct); + struct_group.getTable().update("gr_name", liburl(), BaseStr, ATTRIBUTE); + struct_group.getTable().update("gr_passwd", liburl(), BaseStr, ATTRIBUTE); + struct_group.getTable().update("gr_gid", liburl(), BaseNum, ATTRIBUTE); + struct_group.getTable().update("gr_mem", liburl(), newList(BaseStr), ATTRIBUTE); + + addClass("struct_group", liburl(), struct_group); + + for (String s : list("getgrgid", "getgrnam")) { + addFunction(s, liburl(), struct_group); + } + addFunction("getgrall", liburl(), new NListType(struct_group)); + } + } + + class ImpModule extends NativeModule { + public ImpModule() { + super("imp"); + } + @Override + public void initBindings() { + addStrFuncs("get_magic"); + addFunction("get_suffixes", liburl(), newList(newTuple(BaseStr, BaseStr, BaseNum))); + addFunction("find_module", liburl(), newTuple(BaseStr, BaseStr, BaseNum)); + + String[] module_methods = { + "load_module", "new_module", "init_builtin", "init_frozen", + "load_compiled", "load_dynamic", "load_source" + }; + for (String mm : module_methods) { + addFunction(mm, liburl(), newModule("")); + } + + addUnknownFuncs("acquire_lock", "release_lock"); + + addNumAttrs("PY_SOURCE", "PY_COMPILED", "C_EXTENSION", + "PKG_DIRECTORY", "C_BUILTIN", "PY_FROZEN", "SEARCH_ERROR"); + + addNumFuncs("lock_held", "is_builtin", "is_frozen"); + + NClassType impNullImporter = newClass("NullImporter", table, Object); + impNullImporter.getTable().update("find_module", liburl(), newFunc(None), FUNCTION); + addClass("NullImporter", liburl(), impNullImporter); + } + } + + class ItertoolsModule extends NativeModule { + public ItertoolsModule() { + super("itertools"); + } + @Override + public void initBindings() { + NClassType iterator = newClass("iterator", table, Object); + iterator.getTable().update("from_iterable", liburl("itertool-functions"), + newFunc(iterator), METHOD); + iterator.getTable().update("next", liburl(), newFunc(), METHOD); + + for (String s : list("chain", "combinations", "count", "cycle", + "dropwhile", "groupby", "ifilter", + "ifilterfalse", "imap", "islice", "izip", + "izip_longest", "permutations", "product", + "repeat", "starmap", "takewhile", "tee")) { + addClass(s, liburl("itertool-functions"), iterator); + } + } + } + + class MarshalModule extends NativeModule { + public MarshalModule() { + super("marshal"); + } + @Override + public void initBindings() { + addNumAttrs("version"); + addStrFuncs("dumps"); + addUnknownFuncs("dump", "load", "loads"); + } + } + + class MathModule extends NativeModule { + public MathModule() { + super("math"); + } + @Override + public void initBindings() { + addNumFuncs( + "acos", "acosh", "asin", "asinh", "atan", "atan2", "atanh", "ceil", + "copysign", "cos", "cosh", "degrees", "exp", "fabs", "factorial", + "floor", "fmod", "frexp", "fsum", "hypot", "isinf", "isnan", + "ldexp", "log", "log10", "log1p", "modf", "pow", "radians", "sin", + "sinh", "sqrt", "tan", "tanh", "trunc"); + addNumAttrs("pi", "e"); + } + } + + class Md5Module extends NativeModule { + public Md5Module() { + super("md5"); + } + @Override + public void initBindings() { + addNumAttrs("blocksize", "digest_size"); + + NClassType md5 = newClass("md5", table, Object); + md5.getTable().update("update", liburl(), newFunc(), METHOD); + md5.getTable().update("digest", liburl(), newFunc(BaseStr), METHOD); + md5.getTable().update("hexdigest", liburl(), newFunc(BaseStr), METHOD); + md5.getTable().update("copy", liburl(), newFunc(md5), METHOD); + + update("new", liburl(), newFunc(md5), CONSTRUCTOR); + update("md5", liburl(), newFunc(md5), CONSTRUCTOR); + } + } + + class MmapModule extends NativeModule { + public MmapModule() { + super("mmap"); + } + @Override + public void initBindings() { + NClassType mmap = newClass("mmap", table, Object); + + for (String s : list("ACCESS_COPY", "ACCESS_READ", "ACCESS_WRITE", + "ALLOCATIONGRANULARITY", "MAP_ANON", "MAP_ANONYMOUS", + "MAP_DENYWRITE", "MAP_EXECUTABLE", "MAP_PRIVATE", + "MAP_SHARED", "PAGESIZE", "PROT_EXEC", "PROT_READ", + "PROT_WRITE")) { + mmap.getTable().update(s, liburl(), BaseNum, ATTRIBUTE); + } + + for (String fstr : list("read", "read_byte", "readline")) { + mmap.getTable().update(fstr, liburl(), newFunc(BaseStr), METHOD); + } + + for (String fnum : list("find", "rfind", "tell")) { + mmap.getTable().update(fnum, liburl(), newFunc(BaseNum), METHOD); + } + + for (String fnone : list("close", "flush", "move", "resize", "seek", + "write", "write_byte")) { + mmap.getTable().update(fnone, liburl(), newFunc(None), METHOD); + } + + addClass("mmap", liburl(), mmap); + } + } + + class NisModule extends NativeModule { + public NisModule() { + super("nis"); + } + @Override + public void initBindings() { + addStrFuncs("match", "cat", "get_default_domain"); + addFunction("maps", liburl(), newList(BaseStr)); + addClass("error", liburl(), newException("error", table)); + } + } + + class OsModule extends NativeModule { + public OsModule() { + super("os"); + } + @Override + public void initBindings() { + addAttr("name", liburl(), BaseStr); + addClass("error", liburl(), newException("error", table)); // XXX: OSError + + initProcBindings(); + initProcMgmtBindings(); + initFileBindings(); + initFileAndDirBindings(); + initMiscSystemInfo(); + initOsPathModule(); + + addAttr("errno", liburl(), newModule("errno")); + + addFunction("urandom", liburl("miscellaneous-functions"), BaseStr); + addAttr("NGROUPS_MAX", liburl(), BaseNum); + + for (String s : list("_Environ", "_copy_reg", "_execvpe", "_exists", + "_get_exports_list", "_make_stat_result", + "_make_statvfs_result", "_pickle_stat_result", + "_pickle_statvfs_result", "_spawnvef")) { + addFunction(s, liburl(), unknown()); + } + } + + private void initProcBindings() { + String a = "process-parameters"; + + addAttr("environ", liburl(a), newDict(BaseStr, BaseStr)); + + for (String s : list("chdir", "fchdir", "putenv", "setegid", "seteuid", + "setgid", "setgroups", "setpgrp", "setpgid", + "setreuid", "setregid", "setuid", "unsetenv")) { + addFunction(s, liburl(a), None); + } + + for (String s : list("getegid", "getgid", "getpgid", "getpgrp", + "getppid", "getuid", "getsid", "umask")) { + addFunction(s, liburl(a), BaseNum); + } + + for (String s : list("getcwd", "ctermid", "getlogin", "getenv", "strerror")) { + addFunction(s, liburl(a), BaseStr); + } + + addFunction("getgroups", liburl(a), newList(BaseStr)); + addFunction("uname", liburl(a), newTuple(BaseStr, BaseStr, BaseStr, + BaseStr, BaseStr)); + } + + private void initProcMgmtBindings() { + String a = "process-management"; + + for (String s : list("EX_CANTCREAT", "EX_CONFIG", "EX_DATAERR", + "EX_IOERR", "EX_NOHOST", "EX_NOINPUT", + "EX_NOPERM", "EX_NOUSER", "EX_OK", "EX_OSERR", + "EX_OSFILE", "EX_PROTOCOL", "EX_SOFTWARE", + "EX_TEMPFAIL", "EX_UNAVAILABLE", "EX_USAGE", + "P_NOWAIT", "P_NOWAITO", "P_WAIT", "P_DETACH", + "P_OVERLAY", "WCONTINUED", "WCOREDUMP", + "WEXITSTATUS", "WIFCONTINUED", "WIFEXITED", + "WIFSIGNALED", "WIFSTOPPED", "WNOHANG", "WSTOPSIG", + "WTERMSIG", "WUNTRACED")) { + addAttr(s, liburl(a), BaseNum); + } + + for (String s : list("abort", "execl", "execle", "execlp", "execlpe", + "execv", "execve", "execvp", "execvpe", "_exit", + "kill", "killpg", "plock", "startfile")) { + addFunction(s, liburl(a), None); + } + + for (String s : list("nice", "spawnl", "spawnle", "spawnlp", "spawnlpe", + "spawnv", "spawnve", "spawnvp", "spawnvpe", "system")) { + addFunction(s, liburl(a), BaseNum); + } + + addFunction("fork", liburl(a), newUnion(BaseFile, BaseNum)); + addFunction("times", liburl(a), newTuple(BaseNum, BaseNum, BaseNum, BaseNum, BaseNum)); + + for (String s : list("forkpty", "wait", "waitpid")) { + addFunction(s, liburl(a), newTuple(BaseNum, BaseNum)); + } + + for (String s : list("wait3", "wait4")) { + addFunction(s, liburl(a), newTuple(BaseNum, BaseNum, BaseNum)); + } + } + + private void initFileBindings() { + String a = "file-object-creation"; + + for (String s : list("fdopen", "popen", "tmpfile")) { + addFunction(s, liburl(a), BaseFile); + } + + addFunction("popen2", liburl(a), newTuple(BaseFile, BaseFile)); + addFunction("popen3", liburl(a), newTuple(BaseFile, BaseFile, BaseFile)); + addFunction("popen4", liburl(a), newTuple(BaseFile, BaseFile)); + + a = "file-descriptor-operations"; + + addFunction("open", liburl(a), BaseFile); + + for (String s : list("close", "closerange", "dup2", "fchmod", + "fchown", "fdatasync", "fsync", "ftruncate", + "lseek", "tcsetpgrp", "write")) { + addFunction(s, liburl(a), None); + } + + for (String s : list("dup2", "fpathconf", "fstat", "fstatvfs", + "isatty", "tcgetpgrp")) { + addFunction(s, liburl(a), BaseNum); + } + + for (String s : list("read", "ttyname")) { + addFunction(s, liburl(a), BaseStr); + } + + for (String s : list("openpty", "pipe", "fstat", "fstatvfs", + "isatty")) { + addFunction(s, liburl(a), newTuple(BaseNum, BaseNum)); + } + + for (String s : list("O_APPEND", "O_CREAT", "O_DIRECT", "O_DIRECTORY", + "O_DSYNC", "O_EXCL", "O_LARGEFILE", "O_NDELAY", + "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK", "O_RDONLY", + "O_RDWR", "O_RSYNC", "O_SYNC", "O_TRUNC", "O_WRONLY", + "SEEK_CUR", "SEEK_END", "SEEK_SET")) { + addAttr(s, liburl(a), BaseNum); + } + } + + private void initFileAndDirBindings() { + String a = "files-and-directories"; + + for (String s : list("F_OK", "R_OK", "W_OK", "X_OK")) { + addAttr(s, liburl(a), BaseNum); + } + + for (String s : list("chflags", "chroot", "chmod", "chown", "lchflags", + "lchmod", "lchown", "link", "mknod", "mkdir", + "mkdirs", "remove", "removedirs", "rename", "renames", + "rmdir", "symlink", "unlink", "utime")) { + addAttr(s, liburl(a), None); + } + + for (String s : list("access", "lstat", "major", "minor", + "makedev", "pathconf", "stat_float_times")) { + addFunction(s, liburl(a), BaseNum); + } + + for (String s : list("getcwdu", "readlink", "tempnam", "tmpnam")) { + addFunction(s, liburl(a), BaseStr); + } + + for (String s : list("listdir")) { + addFunction(s, liburl(a), newList(BaseStr)); + } + + addFunction("mkfifo", liburl(a), BaseFile); + + addFunction("stat", liburl(a), newList(BaseNum)); // XXX: posix.stat_result + addFunction("statvfs", liburl(a), newList(BaseNum)); // XXX: pos.statvfs_result + + addAttr("pathconf_names", liburl(a), newDict(BaseStr, BaseNum)); + addAttr("TMP_MAX", liburl(a), BaseNum); + + addFunction("walk", liburl(a), newList(newTuple(BaseStr, BaseStr, BaseStr))); + } + + private void initMiscSystemInfo() { + String a = "miscellaneous-system-information"; + + addAttr("confstr_names", liburl(a), newDict(BaseStr, BaseNum)); + addAttr("sysconf_names", liburl(a), newDict(BaseStr, BaseNum)); + + for (String s : list("curdir", "pardir", "sep", "altsep", "extsep", + "pathsep", "defpath", "linesep", "devnull")) { + addAttr(s, liburl(a), BaseStr); + } + + for (String s : list("getloadavg", "sysconf")) { + addFunction(s, liburl(a), BaseNum); + } + + addFunction("confstr", liburl(a), BaseStr); + } + + private void initOsPathModule() { + NModuleType m = newModule("path"); + Scope ospath = m.getTable(); + ospath.setPath("os.path"); // make sure global qnames are correct + + update("path", newLibUrl("os.path.html#module-os.path"), m, MODULE); + + String[] str_funcs = { + "_resolve_link", "abspath", "basename", "commonprefix", + "dirname","expanduser", "expandvars", "join", + "normcase", "normpath", "realpath", "relpath", + }; + for (String s : str_funcs) { + ospath.update(s, newLibUrl("os.path", s), newFunc(BaseStr), FUNCTION); + } + + String[] num_funcs = { + "exists", "lexists", "getatime", "getctime", "getmtime", "getsize", + "isabs", "isdir", "isfile", "islink", "ismount", "samefile", + "sameopenfile", "samestat", "supports_unicode_filenames", + }; + for (String s : num_funcs) { + ospath.update(s, newLibUrl("os.path", s), newFunc(BaseNum), FUNCTION); + } + + for (String s : list("split", "splitdrive", "splitext", "splitunc")) { + ospath.update(s, newLibUrl("os.path", s), + newFunc(newTuple(BaseStr, BaseStr)), FUNCTION); + } + + NBinding b = ospath.update("walk", newLibUrl("os.path"), newFunc(None), FUNCTION); + b.markDeprecated(); + + String[] str_attrs = { + "altsep", "curdir", "devnull", "defpath", "pardir", "pathsep", "sep", + }; + for (String s : str_attrs) { + ospath.update(s, newLibUrl("os.path", s), BaseStr, ATTRIBUTE); + } + + ospath.update("os", liburl(), this.module, ATTRIBUTE); + ospath.update("stat", newLibUrl("stat"), + // moduleTable.lookupLocal("stat").getType(), + newModule(""), ATTRIBUTE); + + // XXX: this is an re object, I think + ospath.update("_varprog", newLibUrl("os.path"), unknown(), ATTRIBUTE); + } + } + + class OperatorModule extends NativeModule { + public OperatorModule() { + super("operator"); + } + @Override + public void initBindings() { + // XXX: mark __getslice__, __setslice__ and __delslice__ as deprecated. + addNumFuncs( + "__abs__", "__add__", "__and__", "__concat__", "__contains__", + "__div__", "__doc__", "__eq__", "__floordiv__", "__ge__", + "__getitem__", "__getslice__", "__gt__", "__iadd__", "__iand__", + "__iconcat__", "__idiv__", "__ifloordiv__", "__ilshift__", + "__imod__", "__imul__", "__index__", "__inv__", "__invert__", + "__ior__", "__ipow__", "__irepeat__", "__irshift__", "__isub__", + "__itruediv__", "__ixor__", "__le__", "__lshift__", "__lt__", + "__mod__", "__mul__", "__name__", "__ne__", "__neg__", "__not__", + "__or__", "__package__", "__pos__", "__pow__", "__repeat__", + "__rshift__", "__setitem__", "__setslice__", "__sub__", + "__truediv__", "__xor__", "abs", "add", "and_", "concat", + "contains", "countOf", "div", "eq", "floordiv", "ge", "getitem", + "getslice", "gt", "iadd", "iand", "iconcat", "idiv", "ifloordiv", + "ilshift", "imod", "imul", "index", "indexOf", "inv", "invert", + "ior", "ipow", "irepeat", "irshift", "isCallable", + "isMappingType", "isNumberType", "isSequenceType", "is_", + "is_not", "isub", "itruediv", "ixor", "le", "lshift", "lt", "mod", + "mul", "ne", "neg", "not_", "or_", "pos", "pow", "repeat", + "rshift", "sequenceIncludes", "setitem", "setslice", "sub", + "truediv", "truth", "xor"); + + addUnknownFuncs("attrgetter", "itemgetter", "methodcaller"); + addNoneFuncs("__delitem__", "__delslice__", "delitem", "delclice"); + } + } + + class ParserModule extends NativeModule { + public ParserModule() { + super("parser"); + } + @Override + public void initBindings() { + NClassType st = newClass("st", table, Object); + st.getTable().update("compile", newLibUrl("parser", "st-objects"), + newFunc(), METHOD); + st.getTable().update("isexpr", newLibUrl("parser", "st-objects"), + newFunc(BaseNum), METHOD); + st.getTable().update("issuite", newLibUrl("parser", "st-objects"), + newFunc(BaseNum), METHOD); + st.getTable().update("tolist", newLibUrl("parser", "st-objects"), + newFunc(newList()), METHOD); + st.getTable().update("totuple", newLibUrl("parser", "st-objects"), + newFunc(newTuple()), METHOD); + + addAttr("STType", liburl("st-objects"), Type); + + for (String s : list("expr", "suite", "sequence2st", "tuple2st")) { + addFunction(s, liburl("creating-st-objects"), st); + } + + addFunction("st2list", liburl("converting-st-objects"), newList()); + addFunction("st2tuple", liburl("converting-st-objects"), newTuple()); + addFunction("compilest", liburl("converting-st-objects"), unknown()); + + addFunction("isexpr", liburl("queries-on-st-objects"), BaseBool); + addFunction("issuite", liburl("queries-on-st-objects"), BaseBool); + + addClass("ParserError", liburl("exceptions-and-error-handling"), + newException("ParserError", table)); + } + } + + class PosixModule extends NativeModule { + public PosixModule() { + super("posix"); + } + @Override + public void initBindings() { + addAttr("environ", liburl(), newDict(BaseStr, BaseStr)); + } + } + + class PwdModule extends NativeModule { + public PwdModule() { + super("pwd"); + } + @Override + public void initBindings() { + NClassType struct_pwd = newClass("struct_pwd", table, Object); + for (String s : list("pw_nam", "pw_passwd", "pw_uid", "pw_gid", + "pw_gecos", "pw_dir", "pw_shell")) { + struct_pwd.getTable().update(s, liburl(), BaseNum, ATTRIBUTE); + } + addAttr("struct_pwd", liburl(), struct_pwd); + + addFunction("getpwuid", liburl(), struct_pwd); + addFunction("getpwnam", liburl(), struct_pwd); + addFunction("getpwall", liburl(), newList(struct_pwd)); + } + } + + class PyexpatModule extends NativeModule { + public PyexpatModule() { + super("pyexpat"); + } + @Override + public void initBindings() { + // XXX + } + } + + class ReadlineModule extends NativeModule { + public ReadlineModule() { + super("readline"); + } + @Override + public void initBindings() { + addNoneFuncs("parse_and_bind", "insert_text", "read_init_file", + "read_history_file", "write_history_file", + "clear_history", "set_history_length", + "remove_history_item", "replace_history_item", + "redisplay", "set_startup_hook", "set_pre_input_hook", + "set_completer", "set_completer_delims", + "set_completion_display_matches_hook", "add_history"); + + addNumFuncs("get_history_length", "get_current_history_length", + "get_begidx", "get_endidx"); + + addStrFuncs("get_line_buffer", "get_history_item"); + + addUnknownFuncs("get_completion_type"); + + addFunction("get_completer", liburl(), newFunc()); + addFunction("get_completer_delims", liburl(), newList(BaseStr)); + } + } + + class ResourceModule extends NativeModule { + public ResourceModule() { + super("resource"); + } + @Override + public void initBindings() { + addFunction("getrlimit", liburl(), newTuple(BaseNum, BaseNum)); + addFunction("getrlimit", liburl(), unknown()); + + String[] constants = { + "RLIMIT_CORE", "RLIMIT_CPU", "RLIMIT_FSIZE", "RLIMIT_DATA", + "RLIMIT_STACK", "RLIMIT_RSS", "RLIMIT_NPROC", "RLIMIT_NOFILE", + "RLIMIT_OFILE", "RLIMIT_MEMLOCK", "RLIMIT_VMEM", "RLIMIT_AS" + }; + for (String c : constants) { + addAttr(c, liburl("resource-limits"), BaseNum); + } + + NClassType ru = newClass("struct_rusage", table, Object); + String[] ru_fields = { + "ru_utime", "ru_stime", "ru_maxrss", "ru_ixrss", "ru_idrss", + "ru_isrss", "ru_minflt", "ru_majflt", "ru_nswap", "ru_inblock", + "ru_oublock", "ru_msgsnd", "ru_msgrcv", "ru_nsignals", + "ru_nvcsw", "ru_nivcsw" + }; + for (String ruf : ru_fields) { + ru.getTable().update(ruf, liburl("resource-usage"), BaseNum, ATTRIBUTE); + } + + addFunction("getrusage", liburl("resource-usage"), ru); + addFunction("getpagesize", liburl("resource-usage"), BaseNum); + + for (String s : list("RUSAGE_SELF", "RUSAGE_CHILDREN", "RUSAGE_BOTH")) { + addAttr(s, liburl("resource-usage"), BaseNum); + } + } + } + + class SelectModule extends NativeModule { + public SelectModule() { + super("select"); + } + @Override + public void initBindings() { + addClass("error", liburl(), newException("error", table)); + + addFunction("select", liburl(), newTuple(newList(), newList(), newList())); + + String a = "edge-and-level-trigger-polling-epoll-objects"; + + NClassType epoll = newClass("epoll", table, Object); + epoll.getTable().update("close", newLibUrl("select", a), newFunc(None), METHOD); + epoll.getTable().update("fileno", newLibUrl("select", a), newFunc(BaseNum), METHOD); + epoll.getTable().update("fromfd", newLibUrl("select", a), newFunc(epoll), METHOD); + for (String s : list("register", "modify", "unregister", "poll")) { + epoll.getTable().update(s, newLibUrl("select", a), newFunc(), METHOD); + } + addClass("epoll", liburl(a), epoll); + + for (String s : list("EPOLLERR", "EPOLLET", "EPOLLHUP", "EPOLLIN", "EPOLLMSG", + "EPOLLONESHOT", "EPOLLOUT", "EPOLLPRI", "EPOLLRDBAND", + "EPOLLRDNORM", "EPOLLWRBAND", "EPOLLWRNORM")) { + addAttr(s, liburl(a), BaseNum); + } + + a = "polling-objects"; + + NClassType poll = newClass("poll", table, Object); + poll.getTable().update("register", newLibUrl("select", a), newFunc(), METHOD); + poll.getTable().update("modify", newLibUrl("select", a), newFunc(), METHOD); + poll.getTable().update("unregister", newLibUrl("select", a), newFunc(), METHOD); + poll.getTable().update("poll", newLibUrl("select", a), + newFunc(newList(newTuple(BaseNum, BaseNum))), METHOD); + addClass("poll", liburl(a), poll); + + for (String s : list("POLLERR", "POLLHUP", "POLLIN", "POLLMSG", + "POLLNVAL", "POLLOUT","POLLPRI", "POLLRDBAND", + "POLLRDNORM", "POLLWRBAND", "POLLWRNORM")) { + addAttr(s, liburl(a), BaseNum); + } + + a = "kqueue-objects"; + + NClassType kqueue = newClass("kqueue", table, Object); + kqueue.getTable().update("close", newLibUrl("select", a), newFunc(None), METHOD); + kqueue.getTable().update("fileno", newLibUrl("select", a), newFunc(BaseNum), METHOD); + kqueue.getTable().update("fromfd", newLibUrl("select", a), newFunc(kqueue), METHOD); + kqueue.getTable().update("control", newLibUrl("select", a), + newFunc(newList(newTuple(BaseNum, BaseNum))), METHOD); + addClass("kqueue", liburl(a), kqueue); + + a = "kevent-objects"; + + NClassType kevent = newClass("kevent", table, Object); + for (String s : list("ident", "filter", "flags", "fflags", "data", "udata")) { + kevent.getTable().update(s, newLibUrl("select", a), unknown(), ATTRIBUTE); + } + addClass("kevent", liburl(a), kevent); + } + } + + class SignalModule extends NativeModule { + public SignalModule() { + super("signal"); + } + @Override + public void initBindings() { + addNumAttrs( + "NSIG", "SIGABRT", "SIGALRM", "SIGBUS", "SIGCHLD", "SIGCLD", + "SIGCONT", "SIGFPE", "SIGHUP", "SIGILL", "SIGINT", "SIGIO", + "SIGIOT", "SIGKILL", "SIGPIPE", "SIGPOLL", "SIGPROF", "SIGPWR", + "SIGQUIT", "SIGRTMAX", "SIGRTMIN", "SIGSEGV", "SIGSTOP", "SIGSYS", + "SIGTERM", "SIGTRAP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", + "SIGUSR1", "SIGUSR2", "SIGVTALRM", "SIGWINCH", "SIGXCPU", "SIGXFSZ", + "SIG_DFL", "SIG_IGN"); + + addUnknownFuncs("default_int_handler", "getsignal", "set_wakeup_fd", "signal"); + } + } + + class ShaModule extends NativeModule { + public ShaModule() { + super("sha"); + } + @Override + public void initBindings() { + addNumAttrs("blocksize", "digest_size"); + + NClassType sha = newClass("sha", table, Object); + sha.getTable().update("update", liburl(), newFunc(), METHOD); + sha.getTable().update("digest", liburl(), newFunc(BaseStr), METHOD); + sha.getTable().update("hexdigest", liburl(), newFunc(BaseStr), METHOD); + sha.getTable().update("copy", liburl(), newFunc(sha), METHOD); + addClass("sha", liburl(), sha); + + update("new", liburl(), newFunc(sha), CONSTRUCTOR); + } + } + + class SpwdModule extends NativeModule { + public SpwdModule() { + super("spwd"); + } + @Override + public void initBindings() { + NClassType struct_spwd = newClass("struct_spwd", table, Object); + for (String s : list("sp_nam", "sp_pwd", "sp_lstchg", "sp_min", + "sp_max", "sp_warn", "sp_inact", "sp_expire", + "sp_flag")) { + struct_spwd.getTable().update(s, liburl(), BaseNum, ATTRIBUTE); + } + addAttr("struct_spwd", liburl(), struct_spwd); + + addFunction("getspnam", liburl(), struct_spwd); + addFunction("getspall", liburl(), newList(struct_spwd)); + } + } + + class StropModule extends NativeModule { + public StropModule() { + super("strop"); + } + @Override + public void initBindings() { + table.merge(BaseStr.getTable()); + } + } + + class StructModule extends NativeModule { + public StructModule() { + super("struct"); + } + @Override + public void initBindings() { + addClass("error", liburl(), newException("error", table)); + addStrFuncs("pack"); + addUnknownFuncs("pack_into"); + addNumFuncs("calcsize"); + addFunction("unpack", liburl(), newTuple()); + addFunction("unpack_from", liburl(), newTuple()); + + BaseStruct = newClass("Struct", table, Object); + addClass("Struct", liburl("struct-objects"), BaseStruct); + Scope t = BaseStruct.getTable(); + t.update("pack", liburl("struct-objects"), newFunc(BaseStr), METHOD); + t.update("pack_into", liburl("struct-objects"), newFunc(), METHOD); + t.update("unpack", liburl("struct-objects"), newFunc(newTuple()), METHOD); + t.update("unpack_from", liburl("struct-objects"), newFunc(newTuple()), METHOD); + t.update("format", liburl("struct-objects"), BaseStr, ATTRIBUTE); + t.update("size", liburl("struct-objects"), BaseNum, ATTRIBUTE); + } + } + + class SysModule extends NativeModule { + public SysModule() { + super("sys"); + } + @Override + public void initBindings() { + addUnknownFuncs( + "_clear_type_cache", "call_tracing", "callstats", "_current_frames", + "_getframe", "displayhook", "dont_write_bytecode", "exitfunc", + "exc_clear", "exc_info", "excepthook", "exit", + "last_traceback", "last_type", "last_value", "modules", + "path_hooks", "path_importer_cache", "getprofile", "gettrace", + "setcheckinterval", "setprofile", "setrecursionlimit", "settrace"); + + addAttr("exc_type", liburl(), None); + + addUnknownAttrs("__stderr__", "__stdin__", "__stdout__", + "stderr", "stdin", "stdout", "version_info"); + + addNumAttrs("api_version", "hexversion", "winver", "maxint", "maxsize", + "maxunicode", "py3kwarning", "dllhandle"); + + addStrAttrs("platform", "byteorder", "copyright", "prefix", "version", + "exec_prefix", "executable"); + + addNumFuncs("getrecursionlimit", "getwindowsversion", "getrefcount", + "getsizeof", "getcheckinterval"); + + addStrFuncs("getdefaultencoding", "getfilesystemencoding"); + + for (String s : list("argv", "builtin_module_names", "path", + "meta_path", "subversion")) { + addAttr(s, liburl(), newList(BaseStr)); + } + + for (String s : list("flags", "warnoptions", "float_info")) { + addAttr(s, liburl(), newDict(BaseStr, BaseNum)); + } + } + } + + class SyslogModule extends NativeModule { + public SyslogModule() { + super("syslog"); + } + @Override + public void initBindings() { + addNoneFuncs("syslog", "openlog", "closelog", "setlogmask"); + addNumAttrs("LOG_ALERT", "LOG_AUTH", "LOG_CONS", "LOG_CRIT", "LOG_CRON", + "LOG_DAEMON", "LOG_DEBUG", "LOG_EMERG", "LOG_ERR", "LOG_INFO", + "LOG_KERN", "LOG_LOCAL0", "LOG_LOCAL1", "LOG_LOCAL2", "LOG_LOCAL3", + "LOG_LOCAL4", "LOG_LOCAL5", "LOG_LOCAL6", "LOG_LOCAL7", "LOG_LPR", + "LOG_MAIL", "LOG_MASK", "LOG_NDELAY", "LOG_NEWS", "LOG_NOTICE", + "LOG_NOWAIT", "LOG_PERROR", "LOG_PID", "LOG_SYSLOG", "LOG_UPTO", + "LOG_USER", "LOG_UUCP", "LOG_WARNING"); + } + } + + class TermiosModule extends NativeModule { + public TermiosModule() { + super("termios"); + } + @Override + public void initBindings() { + addFunction("tcgetattr", liburl(), newList()); + addUnknownFuncs("tcsetattr", "tcsendbreak", "tcdrain", "tcflush", "tcflow"); + } + } + + class ThreadModule extends NativeModule { + public ThreadModule() { + super("thread"); + } + @Override + public void initBindings() { + addClass("error", liburl(), newException("error", table)); + + NClassType lock = newClass("lock", table, Object); + lock.getTable().update("acquire", liburl(), BaseNum, METHOD); + lock.getTable().update("locked", liburl(), BaseNum, METHOD); + lock.getTable().update("release", liburl(), None, METHOD); + addAttr("LockType", liburl(), Type); + + addNoneFuncs("interrupt_main", "exit", "exit_thread"); + addNumFuncs("start_new", "start_new_thread", "get_ident", "stack_size"); + + addFunction("allocate", liburl(), lock); + addFunction("allocate_lock", liburl(), lock); // synonym + + addAttr("_local", liburl(), Type); + } + } + + class TimeModule extends NativeModule { + public TimeModule() { + super("time"); + } + @Override + public void initBindings() { + NClassType struct_time = Time_struct_time = newClass("datetime", table, Object); + addAttr("struct_time", liburl(), struct_time); + + String[] struct_time_attrs = { + "n_fields", "n_sequence_fields", "n_unnamed_fields", + "tm_hour", "tm_isdst", "tm_mday", "tm_min", + "tm_mon", "tm_wday", "tm_yday", "tm_year", + }; + for (String s : struct_time_attrs) { + struct_time.getTable().update(s, liburl("struct_time"), BaseNum, ATTRIBUTE); + } + + addNumAttrs("accept2dyear", "altzone", "daylight", "timezone"); + + addAttr("tzname", liburl(), newTuple(BaseStr, BaseStr)); + addNoneFuncs("sleep", "tzset"); + + addNumFuncs("clock", "mktime", "time", "tzname"); + addStrFuncs("asctime", "ctime", "strftime"); + + addFunctions_beCareful(struct_time, "gmtime", "localtime", "strptime"); + } + } + + class UnicodedataModule extends NativeModule { + public UnicodedataModule() { + super("unicodedata"); + } + @Override + public void initBindings() { + addNumFuncs("decimal", "digit", "numeric", "combining", + "east_asian_width", "mirrored"); + addStrFuncs("lookup", "name", "category", "bidirectional", + "decomposition", "normalize"); + addNumAttrs("unidata_version"); + addUnknownAttrs("ucd_3_2_0"); + } + } + + class ZipimportModule extends NativeModule { + public ZipimportModule() { + super("zipimport"); + } + @Override + public void initBindings() { + addClass("ZipImportError", liburl(), newException("ZipImportError", table)); + + NClassType zipimporter = newClass("zipimporter", table, Object); + Scope t = zipimporter.getTable(); + t.update("find_module", liburl(), zipimporter, METHOD); + t.update("get_code", liburl(), unknown(), METHOD); // XXX: code object + t.update("get_data", liburl(), unknown(), METHOD); + t.update("get_source", liburl(), BaseStr, METHOD); + t.update("is_package", liburl(), BaseNum, METHOD); + t.update("load_module", liburl(), newModule(""), METHOD); + t.update("archive", liburl(), BaseStr, ATTRIBUTE); + t.update("prefix", liburl(), BaseStr, ATTRIBUTE); + + addClass("zipimporter", liburl(), zipimporter); + addAttr("_zip_directory_cache", liburl(), newDict(BaseStr, unknown())); + } + } + + class ZlibModule extends NativeModule { + public ZlibModule() { + super("zlib"); + } + @Override + public void initBindings() { + NClassType Compress = newClass("Compress", table, Object); + for (String s : list("compress", "flush")) { + Compress.getTable().update(s, newLibUrl("zlib"), BaseStr, METHOD); + } + Compress.getTable().update("copy", newLibUrl("zlib"), Compress, METHOD); + addClass("Compress", liburl(), Compress); + + NClassType Decompress = newClass("Decompress", table, Object); + for (String s : list("unused_data", "unconsumed_tail")) { + Decompress.getTable().update(s, newLibUrl("zlib"), BaseStr, ATTRIBUTE); + } + for (String s : list("decompress", "flush")) { + Decompress.getTable().update(s, newLibUrl("zlib"), BaseStr, METHOD); + } + Decompress.getTable().update("copy", newLibUrl("zlib"), Decompress, METHOD); + addClass("Decompress", liburl(), Decompress); + + addFunction("adler32", liburl(), BaseNum); + addFunction("compress", liburl(), BaseStr); + addFunction("compressobj", liburl(), Compress); + addFunction("crc32", liburl(), BaseNum); + addFunction("decompress", liburl(), BaseStr); + addFunction("decompressobj", liburl(), Decompress); + } + } +} Index: src/org/python/indexer/ast/BindingFinder.java =================================================================== --- src/org/python/indexer/ast/BindingFinder.java (revision 0) +++ src/org/python/indexer/ast/BindingFinder.java (revision 0) @@ -0,0 +1,35 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; + +/** + * A visitor that looks for name-binding constructs in a given scope. + */ +class BindingFinder extends GenericNodeVisitor { + + private Scope scope; // starting scope + + public BindingFinder(Scope scope) { + this.scope = scope; + } + + @Override + public boolean dispatch(NNode n) { + if (n.bindsName()) { + try { + n.bindNames(scope); + } catch (Exception x) { + Indexer.idx.handleException("error binding names for " + n, x); + } + } + // Do not descend into new scopes. + if (n.isFunctionDef() || n.isClassDef()) { + return false; + } + return super.dispatch(n); + } +} Index: tests/java/org/python/indexer/data/pkg/plant/garden/catnip.py =================================================================== --- tests/java/org/python/indexer/data/pkg/plant/garden/catnip.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/plant/garden/catnip.py (revision 0) @@ -0,0 +1,2 @@ +# reserved for circular import test +import pkg.animal.mammal.cat Index: src/org/python/antlr/ast/Name.java =================================================================== --- src/org/python/antlr/ast/Name.java (revision 6695) +++ src/org/python/antlr/ast/Name.java (working copy) @@ -1,4 +1,4 @@ -// Autogenerated AST node +// Autogenerated AST node -*- c-basic-offset:4 -*- package org.python.antlr.ast; import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; @@ -55,7 +55,6 @@ this.ctx = AstAdapters.py2expr_context(ctx); } - private final static PyString[] fields = new PyString[] {new PyString("id"), new PyString("ctx")}; @ExposedGet(name = "_fields") Index: src/org/python/indexer/ast/NGlobal.java =================================================================== --- src/org/python/indexer/ast/NGlobal.java (revision 0) +++ src/org/python/indexer/ast/NGlobal.java (revision 0) @@ -0,0 +1,58 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +public class NGlobal extends NNode { + + static final long serialVersionUID = 5978320165592263568L; + + public List names; + + public NGlobal(List names) { + this(names, 0, 1); + } + + public NGlobal(List names, int start, int end) { + super(start, end); + this.names = names; + addChildren(names); + } + + @Override + public NType resolve(Scope s) throws Exception { + Scope moduleTable = s.getGlobalTable(); + for (NName name : names) { + if (s.isGlobalName(name.id)) { + continue; // already bound by this (or another) global stmt + } + s.addGlobalName(name.id); + NBinding b = moduleTable.lookup(name); + if (b == null) { + b = moduleTable.put(name.id, null, new NUnknownType(), NBinding.Kind.SCOPE); + } + Indexer.idx.putLocation(name, b); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNodeList(names, v); + } + } +} Index: src/org/python/indexer/ast/NYield.java =================================================================== --- src/org/python/indexer/ast/NYield.java (revision 0) +++ src/org/python/indexer/ast/NYield.java (revision 0) @@ -0,0 +1,42 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NListType; +import org.python.indexer.types.NType; + +public class NYield extends NNode { + + static final long serialVersionUID = 2639481204205358048L; + + public NNode value; + + public NYield(NNode n) { + this(n, 0, 1); + } + + public NYield(NNode n, int start, int end) { + super(start, end); + this.value = n; + addChildren(n); + } + + @Override + public NType resolve(Scope s) throws Exception { + return setType(new NListType(resolveExpr(value, s))); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(value, v); + } + } +} Index: tests/java/org/python/indexer/data/hello.py =================================================================== --- tests/java/org/python/indexer/data/hello.py (revision 0) +++ tests/java/org/python/indexer/data/hello.py (revision 0) @@ -0,0 +1,2 @@ +# Don't modify this file except for comments (the unit test will break). +"""Hello""" Index: tests/java/org/python/indexer/data/pkg/mineral/metal/lead.py =================================================================== --- tests/java/org/python/indexer/data/pkg/mineral/metal/lead.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/mineral/metal/lead.py (revision 0) @@ -0,0 +1,6 @@ +import pkg.plant.poison.eggplant + +print pkg +print pkg.plant +print pkg.plant.poison +print pkg.plant.poison.eggplant Index: tests/java/org/python/indexer/data/pkg/plant/food/__init__.py =================================================================== Index: tests/java/org/python/indexer/data/testfileload.py =================================================================== --- tests/java/org/python/indexer/data/testfileload.py (revision 0) +++ tests/java/org/python/indexer/data/testfileload.py (revision 0) @@ -0,0 +1,3 @@ +# Smoke test for loading a file. +# Don't put any imports in here. +foo = "bar" Index: tests/java/org/python/indexer/data/pkg/mineral/metal/iron.py =================================================================== --- tests/java/org/python/indexer/data/pkg/mineral/metal/iron.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/mineral/metal/iron.py (revision 0) @@ -0,0 +1,3 @@ +import pkg.plant.poison.eggplant as aubergine + +print "eggplants are %s" % ", and ".join(aubergine.adjectives) Index: src/org/python/indexer/ast/NCall.java =================================================================== --- src/org/python/indexer/ast/NCall.java (revision 0) +++ src/org/python/indexer/ast/NCall.java (revision 0) @@ -0,0 +1,90 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NClassType; +import org.python.indexer.types.NFuncType; +import org.python.indexer.types.NInstanceType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +import java.util.ArrayList; +import java.util.List; + +public class NCall extends NNode { + + static final long serialVersionUID = 5212954751978100639L; + + public NNode func; + public List args; + public List keywords; + public NNode kwargs; + public NNode starargs; + + public NCall(NNode func, List args, List keywords, + NNode kwargs, NNode starargs) { + this(func, args, keywords, kwargs, starargs, 0, 1); + } + + public NCall(NNode func, List args, List keywords, + NNode kwargs, NNode starargs, int start, int end) { + super(start, end); + this.func = func; + this.args = args; + this.keywords = keywords; + this.kwargs = kwargs; + this.starargs = starargs; + addChildren(func, kwargs, starargs); + addChildren(args); + addChildren(keywords); + } + + @Override + public NType resolve(Scope s) throws Exception { + NType ft = resolveExpr(func, s); + List argTypes = new ArrayList(); + for (NNode a : args) { + argTypes.add(resolveExpr(a, s)); + } + resolveList(keywords, s); + resolveExpr(starargs, s); + resolveExpr(kwargs, s); + + if (ft.isClassType()) { + return setType(ft); // XXX: was new NInstanceType(ft) + } + + if (ft.isFuncType()) { + return setType(ft.asFuncType().getReturnType().follow()); + } + + if (ft.isUnknownType()) { + NUnknownType to = new NUnknownType(); + NFuncType at = new NFuncType(to); + NUnionType.union(ft, at); + return setType(to); + } + + addWarning("calling non-function " + ft); + return setType(new NUnknownType()); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(func, v); + visitNodeList(args, v); + visitNodeList(keywords, v); + visitNode(kwargs, v); + visitNode(starargs, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/plant/garden/weed.py =================================================================== Index: src/org/python/indexer/ast/NContinue.java =================================================================== --- src/org/python/indexer/ast/NContinue.java (revision 0) +++ src/org/python/indexer/ast/NContinue.java (revision 0) @@ -0,0 +1,26 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +public class NContinue extends NNode { + + static final long serialVersionUID = 1646681898280823606L; + + public NContinue() { + } + + public NContinue(int start, int end) { + super(start, end); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: tests/java/org/python/indexer/data/yinw/yinw-13.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-13.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-13.py (revision 0) @@ -0,0 +1,13 @@ +x = 1 + +def f(y): + y = x + x = 3 + print x + +def g(): + x = 3 + print x + +f(1) +g() Index: src/org/python/indexer/types/NFuncType.java =================================================================== --- src/org/python/indexer/types/NFuncType.java (revision 0) +++ src/org/python/indexer/types/NFuncType.java (revision 0) @@ -0,0 +1,46 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Indexer; + +public class NFuncType extends NType { + + private NType returnType; + + public NFuncType() { + this(new NUnknownType()); + } + + public NFuncType(NType from, NType to) { + this(to); + } + + public NFuncType(NType from1, NType from2, NType to) { + this(to); + } + + public NFuncType(NType to) { + setReturnType(to); + getTable().addSuper(Indexer.idx.builtins.BaseFunction.getTable()); + getTable().setPath(Indexer.idx.builtins.BaseFunction.getTable().getPath()); + } + + public void setReturnType(NType to) { + if (to == null) { + to = new NUnknownType(); + } + this.returnType = to; + } + + public NType getReturnType() { + return returnType; + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + sb.append("_:"); // XXX: placeholder for fromType + returnType.print(ctr, sb); + } +} Index: src/org/python/indexer/ast/NPrint.java =================================================================== --- src/org/python/indexer/ast/NPrint.java (revision 0) +++ src/org/python/indexer/ast/NPrint.java (revision 0) @@ -0,0 +1,49 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +import java.util.List; + +public class NPrint extends NNode { + + static final long serialVersionUID = 689872588518071148L; + + public NNode dest; + public List values; + + public NPrint(NNode dest, List elts) { + this(dest, elts, 0, 1); + } + + public NPrint(NNode dest, List elts, int start, int end) { + super(start, end); + this.dest = dest; + this.values = elts; + addChildren(dest); + addChildren(elts); + } + + @Override + public NType resolve(Scope s) throws Exception { + resolveExpr(dest, s); + resolveList(values, s); + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(dest, v); + visitNodeList(values, v); + } + } +} Index: src/org/python/indexer/ast/NNode.java =================================================================== --- src/org/python/indexer/ast/NNode.java (revision 0) +++ src/org/python/indexer/ast/NNode.java (revision 0) @@ -0,0 +1,387 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.IndexingException; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.types.NClassType; +import org.python.indexer.types.NFuncType; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; +import org.python.indexer.types.NUnknownType; + +import java.util.List; + +public abstract class NNode implements java.io.Serializable { + + static final long serialVersionUID = 3682719481356964898L; + + private int start = 0; + private int end = 1; + + protected NNode parent = null; + + /** + * This is marked transient to prevent serialization. We re-resolve ASTs + * after deserializing them. It is private to ensure that the type is never + * {@code null}, as much code in the indexer assumes this precondition. + */ + private transient NType type = Indexer.idx.builtins.None; + + public NNode() { + } + + public NNode(int start, int end) { + setStart(start); + setEnd(end); + } + + public void setParent(NNode parent) { + this.parent = parent; + } + + public NNode getParent() { + return parent; + } + + public NNode getAstRoot() { + if (parent == null) { + return this; + } + return parent.getAstRoot(); + } + + public void setStart(int start) { + this.start = start; + } + + public void setEnd(int end) { + this.end = end; + } + + public int start() { + return start; + } + + public int end() { + return end; + } + + public int length() { + return end - start; + } + + /** + * Utility alias for {@code getType().getTable()}. + */ + public Scope getTable() { + return getType().getTable(); + } + + /** + * Returns the type for this node. It is never {@code null}. + * If the node has not been resolved, the type will default to + * {@link Indexer.idx.builtins.None}. + */ + public NType getType() { + if (type == null) { + type = Indexer.idx.builtins.None; + } + return type; + } + + /** + * Sets the type for the node. + * @param newType the new type + * @return {@code newType} + * @throws IllegalArgumentException if {@code newType} is {@code null} + */ + public NType setType(NType newType) { + if (newType == null) { + throw new IllegalArgumentException(); + } + return type = newType; + } + + /** + * Adds a new type for the node, creating a union of the previous type + * and the new type. + * @param newType the new type + * @return the resulting type for the node + * @throws IllegalArgumentException if {@code newType} is {@code null} + */ + public NType addType(NType newType) { + if (newType == null) { + throw new IllegalArgumentException(); + } + return type = NUnionType.union(getType(), newType); + } + + /** + * Returns {@code true} if this is a name-binding node. + * Includes functions/lambdas, function/lambda params, classes, + * assignments, imports, and implicit assignment via for statements + * and except clauses. + * @see http://www.python.org/dev/peps/pep-0227 + */ + public boolean bindsName() { + return false; + } + + /** + * Called by resolver to bind names into the passed scope. + */ + protected void bindNames(Scope s) throws Exception { + throw new UnsupportedOperationException("Not a name-binding node type"); + } + + /** + * @return the path to the code that generated this AST + */ + public String getFile() { + return parent != null ? parent.getFile() : null; + } + + public void addChildren(NNode... nodes) { + if (nodes != null) { + for (NNode n : nodes) { + if (n != null) { + n.setParent(this); + } + } + } + } + + public void addChildren(List nodes) { + if (nodes != null) { + for (NNode n : nodes) { + if (n != null) { + n.setParent(this); + } + } + } + } + + private static NType handleExceptionInResolve(NNode n, Throwable t) { + Indexer.idx.handleException("Unable to resolve: " + n + " in " + n.getFile(), t); + return new NUnknownType(); + } + + public static NType resolveExpr(NNode n, Scope s) { + if (n == null) { + return new NUnknownType(); + } + // This try-catch enables error recovery when there are bugs in + // the indexer. Rather than unwinding all the way up to the module + // level (and failing to load the module), we record an error for this + // node and continue. + try { + NType result = n.resolve(s); + if (result == null) { + Indexer.idx.warn(n + " resolved to a null type"); + return n.setType(new NUnknownType()); + } + return result; + } catch (IndexingException ix) { + throw ix; + } catch (Exception x) { + return handleExceptionInResolve(n, x); + } catch (StackOverflowError soe) { + String msg = "Unable to resolve: " + n + " in " + n.getFile() + " (stack overflow)"; + Indexer.idx.warn(msg); + return handleExceptionInResolve(n, soe); + } + } + + /** + * Node should set the resolved type in its {@link #type} field + * and also return it. + */ + public NType resolve(Scope s) throws Exception { + return getType(); + } + + public boolean isCall() { + return this instanceof NCall; + } + + public boolean isModule() { + return this instanceof NModule; + } + + public boolean isClassDef() { + return false; + } + + public boolean isFunctionDef() { + return false; + } + + public boolean isLambda() { + return false; + } + + public boolean isName() { + return this instanceof NName; + } + + protected void visitNode(NNode n, NNodeVisitor v) { + if (n != null) { + n.visit(v); + } + } + + protected void visitNodeList(List nodes, NNodeVisitor v) { + if (nodes != null) { + for (NNode n : nodes) { + if (n != null) { + n.visit(v); + } + } + } + } + + /** + * Visits this node and optionally its children.

    + * + * @param visitor the object to call with this node. + * If the visitor returns {@code true}, the node also + * passes its children to the visitor. + */ + public abstract void visit(NNodeVisitor visitor); + + /** + * Returns the innermost enclosing scope for doing (non-attribute) name + * lookups. If the current node defines a scope, it returns the parent + * scope for name lookups. + * + * @return the enclosing function, class, instance, module or builtin scope. + * If this node has not yet been resolved, returns the builtin + * namespace. + */ + public Scope getEnclosingNamespace() { + if (parent == null || this.isModule()) { + return Indexer.idx.globaltable; + } + NNode up = this; + while ((up = up.parent) != null) { + if (up.isFunctionDef() || up.isClassDef() || up.isModule()) { + NType type = up.getType(); + if (type == null || type.getTable() == null) { + return Indexer.idx.globaltable; + } + return type.getTable(); + } + } + return Indexer.idx.globaltable; + } + + protected void addWarning(String msg) { + Indexer.idx.putProblem(this, msg); + } + + protected void addWarning(NNode loc, String msg) { + Indexer.idx.putProblem(loc, msg); + } + + protected void addError(String msg) { + Indexer.idx.putProblem(this, msg); + } + + protected void addError(NNode loc, String msg) { + Indexer.idx.putProblem(loc, msg); + } + + /** + * Utility method to resolve every node in {@code nodes} and + * return the union of their types. If {@code nodes} is empty or + * {@code null}, returns a new {@link NUnknownType}. + */ + protected NType resolveListAsUnion(List nodes, Scope s) { + if (nodes == null || nodes.isEmpty()) { + return new NUnknownType(); + } + + NType result = null; + for (NNode node : nodes) { + NType nodeType = resolveExpr(node, s); + if (result == null) { + result = nodeType; + } else { + result = NUnionType.union(result, nodeType); + } + } + return result; + } + + /** + * Resolves each element of a node list in the passed scope. + * Node list may be empty or {@code null}. + */ + protected void resolveList(List nodes, Scope s) { + if (nodes != null) { + for (NNode n : nodes) { + resolveExpr(n, s); + } + } + } + + /** + * Assumes nodes are always traversed in increasing order of their start + * positions. + */ + static class DeepestOverlappingNodeFinder extends GenericNodeVisitor { + private int offset; + private NNode deepest; + + public DeepestOverlappingNodeFinder(int offset) { + this.offset = offset; + } + + /** + * Returns the deepest node overlapping the desired source offset. + * @return the node, or {@code null} if no node overlaps the offset + */ + public NNode getNode() { + return deepest; + } + + public boolean dispatch(NNode node) { + // This node ends before the offset, so don't look inside it. + if (offset > node.end) { + return false; // don't traverse children, but do keep going + } + + if (offset >= node.start && offset <= node.end) { + deepest = node; + return true; // visit kids + } + + // this node starts after the offset, so we're done + throw new NNodeVisitor.StopIterationException(); + } + } + + /** + * Searches the AST for the deepest node that overlaps the specified source + * offset. Can be called from any node in the AST, as it traverses to the + * parent before beginning the search. + * @param sourceOffset the spot at which to look for a node + * @return the deepest AST node whose start is greater than or equal to the offset, + * and whose end is less than or equal to the offset. Returns {@code null} + * if no node overlaps {@code sourceOffset}. + */ + public NNode getDeepestNodeAtOffset(int sourceOffset) { + NNode ast = getAstRoot(); + DeepestOverlappingNodeFinder finder = new DeepestOverlappingNodeFinder(sourceOffset); + try { + ast.visit(finder); + } catch (NNodeVisitor.StopIterationException six) { + // expected + } + return finder.getNode(); + } +} Index: tests/java/org/python/indexer/data/yinw/yinw-1.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-1.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-1.py (revision 0) @@ -0,0 +1,19 @@ +class A: + a = 1 + +class B: + a = 2 + +l = [A(), A(), B()] + +for x in l: + print x.a + + +# locations: +# = [] +# = [] +# = [] +# = [] +# = [] +# .a:10:10> = [, ] Index: tests/java/org/python/indexer/data/testload.py =================================================================== --- tests/java/org/python/indexer/data/testload.py (revision 0) +++ tests/java/org/python/indexer/data/testload.py (revision 0) @@ -0,0 +1,3 @@ +"""Make sure we can load and parse a file.""" + +x = 6 Index: tests/java/org/python/indexer/data/pkg/other/color/bleu.py =================================================================== --- tests/java/org/python/indexer/data/pkg/other/color/bleu.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/other/color/bleu.py (revision 0) @@ -0,0 +1 @@ +from blue import * Index: src/org/python/indexer/ast/NBody.java =================================================================== --- src/org/python/indexer/ast/NBody.java (revision 0) +++ src/org/python/indexer/ast/NBody.java (revision 0) @@ -0,0 +1,71 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +import java.util.List; + +/** + * An {@link NBlock} variant used for the bodies of functions, lambdas, + * classes and modules. + */ +public class NBody extends NBlock { + + static final long serialVersionUID = 1518962862898927516L; + + public NBody(NBlock block) { + this(block == null ? null : block.seq); + } + + public NBody(List seq) { + super(seq); + } + + public NBody(List seq, int start, int end) { + super(seq, start, end); + } + + private class GlobalFinder extends DefaultNodeVisitor { + private Scope scope; // starting scope + + public GlobalFinder(Scope scope) { + this.scope = scope; + } + + @Override + public boolean visit(NGlobal n) { + resolveExpr(n, scope); + return false; + } + + // Do not descend into new scopes. + @Override + public boolean visit(NFunctionDef n) { + return false; + } + @Override + public boolean visit(NLambda n) { + return false; + } + @Override + public boolean visit(NClassDef n) { + return false; + } + } + + @Override + public NType resolve(Scope scope) throws Exception { + try { + scope.setNameBindingPhase(true); + visit(new GlobalFinder(scope)); + visit(new BindingFinder(scope)); + } finally { + scope.setNameBindingPhase(false); + } + return super.resolve(scope); + } +} Index: src/org/python/indexer/types/NModuleType.java =================================================================== --- src/org/python/indexer/types/NModuleType.java (revision 0) +++ src/org/python/indexer/types/NModuleType.java (revision 0) @@ -0,0 +1,77 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.types; + +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.Util; +import org.python.indexer.ast.NAssign; +import org.python.indexer.ast.NList; +import org.python.indexer.ast.NName; +import org.python.indexer.ast.NNode; +import org.python.indexer.ast.NStr; + +import java.util.ArrayList; +import java.util.List; + +public class NModuleType extends NType { + + private String file; + private String name; + private String qname; + + public NModuleType() { + } + + public NModuleType(String name, String file, Scope parent) { + this.name = name; + this.file = file; // null for builtin modules + if (file != null) { + // This will return null iff specified file is not prefixed by + // any path in the module search path -- i.e., the caller asked + // the indexer to load a file not in the search path. + qname = Util.moduleQname(file); + } + if (qname == null) { + qname = name; + } + setTable(new Scope(parent, Scope.Type.MODULE)); + getTable().setPath(qname); + + // null during bootstrapping of built-in types + if (Indexer.idx.builtins != null) { + getTable().addSuper(Indexer.idx.builtins.BaseModule.getTable()); + } + } + + public void setFile(String file) { + this.file = file; + } + + public String getFile() { + return file; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setQname(String qname) { + this.qname = qname; + } + + public String getQname() { + return qname; + } + + @Override + public void printKids(CyclicTypeRecorder ctr, StringBuilder sb) { + sb.append(qname); + } +} Index: tests/java/org/python/indexer/data/pkg/misc/m1.py =================================================================== --- tests/java/org/python/indexer/data/pkg/misc/m1.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/misc/m1.py (revision 0) @@ -0,0 +1,9 @@ +# test in python shell with execfile: +# >>> sys.path.insert(0, "/abs/path/to/this/m1.py") +# >>> execfile("/abs/path/to/this/m1.py") + +import distutils.command + +del distutils + +print dir() Index: src/org/python/indexer/ast/DefaultNodeVisitor.java =================================================================== --- src/org/python/indexer/ast/DefaultNodeVisitor.java (revision 0) +++ src/org/python/indexer/ast/DefaultNodeVisitor.java (revision 0) @@ -0,0 +1,243 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +/** + * A visitor that by default visits every node in the tree. + * Subclasses can override specific node visiting methods + * and decide whether to visit the children. + */ +public class DefaultNodeVisitor implements NNodeVisitor { + + protected boolean traverseIntoNodes = true; + + /** + * Once this is called, all {@code visit} methods will return {@code false}. + * If the current node's children are being visited, all remaining top-level + * children of the node will be visited (without visiting their children), + * and then tree traversal halts.

    + * + * If the traversal should be halted immediately without visiting any further + * nodes, the visitor can throw a {@link StopIterationException}. + */ + public void stopTraversal() { + traverseIntoNodes = false; + } + + public boolean visit(NAlias n) { + return traverseIntoNodes; + } + + public boolean visit(NAssert n) { + return traverseIntoNodes; + } + + public boolean visit(NAssign n) { + return traverseIntoNodes; + } + + public boolean visit(NAttribute n) { + return traverseIntoNodes; + } + + public boolean visit(NAugAssign n) { + return traverseIntoNodes; + } + + public boolean visit(NBinOp n) { + return traverseIntoNodes; + } + + public boolean visit(NBlock n) { + return traverseIntoNodes; + } + + public boolean visit(NBoolOp n) { + return traverseIntoNodes; + } + + public boolean visit(NBreak n) { + return traverseIntoNodes; + } + + public boolean visit(NCall n) { + return traverseIntoNodes; + } + + public boolean visit(NClassDef n) { + return traverseIntoNodes; + } + + public boolean visit(NCompare n) { + return traverseIntoNodes; + } + + public boolean visit(NComprehension n) { + return traverseIntoNodes; + } + + public boolean visit(NContinue n) { + return traverseIntoNodes; + } + + public boolean visit(NDelete n) { + return traverseIntoNodes; + } + + public boolean visit(NDict n) { + return traverseIntoNodes; + } + + public boolean visit(NEllipsis n) { + return traverseIntoNodes; + } + + public boolean visit(NExceptHandler n) { + return traverseIntoNodes; + } + + public boolean visit(NExec n) { + return traverseIntoNodes; + } + + public boolean visit(NFor n) { + return traverseIntoNodes; + } + + public boolean visit(NFunctionDef n) { + return traverseIntoNodes; + } + + public boolean visit(NGeneratorExp n) { + return traverseIntoNodes; + } + + public boolean visit(NGlobal n) { + return traverseIntoNodes; + } + + public boolean visit(NIf n) { + return traverseIntoNodes; + } + + public boolean visit(NIfExp n) { + return traverseIntoNodes; + } + + public boolean visit(NImport n) { + return traverseIntoNodes; + } + + public boolean visit(NImportFrom n) { + return traverseIntoNodes; + } + + public boolean visit(NIndex n) { + return traverseIntoNodes; + } + + public boolean visit(NKeyword n) { + return traverseIntoNodes; + } + + public boolean visit(NLambda n) { + return traverseIntoNodes; + } + + public boolean visit(NList n) { + return traverseIntoNodes; + } + + public boolean visit(NListComp n) { + return traverseIntoNodes; + } + + public boolean visit(NModule n) { + return traverseIntoNodes; + } + + public boolean visit(NName n) { + return traverseIntoNodes; + } + + public boolean visit(NNum n) { + return traverseIntoNodes; + } + + public boolean visit(NPass n) { + return traverseIntoNodes; + } + + public boolean visit(NPlaceHolder n) { + return traverseIntoNodes; + } + + public boolean visit(NPrint n) { + return traverseIntoNodes; + } + + public boolean visit(NQname n) { + return traverseIntoNodes; + } + + public boolean visit(NRaise n) { + return traverseIntoNodes; + } + + public boolean visit(NRepr n) { + return traverseIntoNodes; + } + + public boolean visit(NReturn n) { + return traverseIntoNodes; + } + + public boolean visit(NExprStmt n) { + return traverseIntoNodes; + } + + public boolean visit(NSlice n) { + return traverseIntoNodes; + } + + public boolean visit(NStr n) { + return traverseIntoNodes; + } + + public boolean visit(NSubscript n) { + return traverseIntoNodes; + } + + public boolean visit(NTryExcept n) { + return traverseIntoNodes; + } + + public boolean visit(NTryFinally n) { + return traverseIntoNodes; + } + + public boolean visit(NTuple n) { + return traverseIntoNodes; + } + + public boolean visit(NUnaryOp n) { + return traverseIntoNodes; + } + + public boolean visit(NUrl n) { + return traverseIntoNodes; + } + + public boolean visit(NWhile n) { + return traverseIntoNodes; + } + + public boolean visit(NWith n) { + return traverseIntoNodes; + } + + public boolean visit(NYield n) { + return traverseIntoNodes; + } +} Index: tests/java/org/python/indexer/data/pkg/animal/mammal/ape.py =================================================================== --- tests/java/org/python/indexer/data/pkg/animal/mammal/ape.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/animal/mammal/ape.py (revision 0) @@ -0,0 +1 @@ +ape = True Index: tests/java/org/python/indexer/data/yinw/yinw-19.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-19.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-19.py (revision 0) @@ -0,0 +1,10 @@ +def f(): + try: + p = f(1) + except error, v: + raise error, v # invalid expression + +# try: +# this_fails() +# except ZeroDivisionError as detail: +# print 'Handling run-time error:', detail Index: src/org/python/indexer/ast/NTryFinally.java =================================================================== --- src/org/python/indexer/ast/NTryFinally.java (revision 0) +++ src/org/python/indexer/ast/NTryFinally.java (revision 0) @@ -0,0 +1,51 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Scope; +import org.python.indexer.types.NType; +import org.python.indexer.types.NUnionType; + +public class NTryFinally extends NNode { + + static final long serialVersionUID = 136428581711609107L; + + public NBlock body; + public NBlock finalbody; + + public NTryFinally(NBlock body, NBlock orelse) { + this(body, orelse, 0, 1); + } + + public NTryFinally(NBlock body, NBlock orelse, int start, int end) { + super(start, end); + this.body = body; + this.finalbody = orelse; + addChildren(body, orelse); + } + + @Override + public NType resolve(Scope s) throws Exception { + if (body != null) { + setType(resolveExpr(body, s)); + } + if (finalbody != null) { + addType(resolveExpr(finalbody, s)); + } + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(body, v); + visitNode(finalbody, v); + } + } +} Index: src/org/python/indexer/ast/NBreak.java =================================================================== --- src/org/python/indexer/ast/NBreak.java (revision 0) +++ src/org/python/indexer/ast/NBreak.java (revision 0) @@ -0,0 +1,26 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +public class NBreak extends NNode { + + static final long serialVersionUID = 2114759731430768793L; + + public NBreak() { + } + + public NBreak(int start, int end) { + super(start, end); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + v.visit(this); + } +} Index: tests/java/org/python/indexer/data/pkg/animal/animaltest.py =================================================================== --- tests/java/org/python/indexer/data/pkg/animal/animaltest.py (revision 0) +++ tests/java/org/python/indexer/data/pkg/animal/animaltest.py (revision 0) @@ -0,0 +1 @@ +living = True Index: src/org/python/indexer/ast/NCompare.java =================================================================== --- src/org/python/indexer/ast/NCompare.java (revision 0) +++ src/org/python/indexer/ast/NCompare.java (revision 0) @@ -0,0 +1,55 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Indexer; +import org.python.indexer.Scope; +import org.python.indexer.types.NType; + +import java.util.List; + +public class NCompare extends NNode { + + static final long serialVersionUID = 1013460919393985064L; + + public NNode left; + public List ops; + public List comparators; + + public NCompare(NNode left, List ops, List comparators) { + this(left, ops, comparators, 0, 1); + } + + public NCompare(NNode left, List ops, List comparators, int start, int end) { + super(start, end); + this.left = left; + this.ops = ops; + this.comparators = comparators; + addChildren(left); + addChildren(ops); + addChildren(comparators); + } + + @Override + public NType resolve(Scope s) throws Exception { + setType(Indexer.idx.builtins.BaseNum); + resolveExpr(left, s); + resolveList(comparators, s); + return getType(); + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(left, v); + visitNodeList(ops, v); + visitNodeList(comparators, v); + } + } +} Index: src/org/python/indexer/ast/NModule.java =================================================================== --- src/org/python/indexer/ast/NModule.java (revision 0) +++ src/org/python/indexer/ast/NModule.java (revision 0) @@ -0,0 +1,200 @@ +/** + * Copyright 2009, Google Inc. All rights reserved. + */ +package org.python.indexer.ast; + +import org.python.indexer.Def; +import org.python.indexer.Indexer; +import org.python.indexer.NBinding; +import org.python.indexer.Scope; +import org.python.indexer.Util; +import org.python.indexer.types.NModuleType; +import org.python.indexer.types.NType; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class NModule extends NNode { + + static final long serialVersionUID = -7737089963380450802L; + + public String name; + public NBody body; + + private String file; // input source file path + private String md5; // input source file md5 + + public NModule() { + } + + public NModule(String name) { + this.name = name; + } + + public NModule(NBlock body, int start, int end) { + super(start, end); + this.body = new NBody(body); + addChildren(this.body); + } + + public void setFile(String file) throws Exception { + this.file = file; + this.name = Util.moduleNameFor(file); + this.md5 = Util.getMD5(new File(file)); + } + + public void setFile(File path) throws Exception { + file = path.getCanonicalPath(); + name = Util.moduleNameFor(file); + md5 = Util.getMD5(path); + } + + /** + * Used when module is parsed from an in-memory string. + * @param path file path + * @param md5 md5 message digest for source contents + */ + public void setFileAndMD5(String path, String md5) throws Exception { + file = path; + name = Util.moduleNameFor(file); + this.md5 = md5; + } + + @Override + public String getFile() { + return file; + } + + public String getMD5() { + return md5; + } + + @Override + public NType resolve(Scope s) throws Exception { + NBinding mb = Indexer.idx.moduleTable.lookupLocal(file); + if (mb == null ) { + Indexer.idx.reportFailedAssertion("No module for " + name + ": " + file); + setType(new NModuleType(name, file, s)); + } else { + setType(mb.getType()); + } + + resolveExpr(body, getTable()); + + resolveExportedNames(); + return getType(); + } + + /** + * If the module defines an {@code __all__} variable, resolve references + * to each of the elements. + */ + private void resolveExportedNames() throws Exception { + NModuleType mtype = null; + NType thisType = getType(); + if (thisType.isModuleType()) { + mtype = thisType.asModuleType(); + } else if (thisType.isUnionType()) { + for (NType u : thisType.asUnionType().getTypes()) { + if (u.isModuleType()) { + mtype = u.asModuleType(); + break; + } + } + } + + if (mtype == null) { + Indexer.idx.reportFailedAssertion("Found non-module type for " + + this + " in " + getFile() + ": " + thisType); + return; + } + + Scope table = mtype.getTable(); + for (NStr nstr : getExportedNameNodes()) { + String name = nstr.n.toString(); + NBinding b = table.lookupLocal(name); + if (b != null) { + Indexer.idx.putLocation(nstr, b); + } + } + } + + /** + * Attempt to determine the actual value of the "__all__" variable in the + * target module. If we can parse it, return the list of exported names.

    + * + * @return the list of exported names. Returns {@code null} if __all__ is + * missing, or if its initializer is not a simple list of strings. + * We don't generate a warning, since complex expressions such as + * {@code __all__ = [name for name in dir() if name[0] == "e"]} + * are valid provided the expr result is a string list. + */ + public List getExportedNames() throws Exception { + List exports = new ArrayList(); + if (!getType().isModuleType()) { + return exports; + } + for (NStr nstr : getExportedNameNodes()) { + exports.add(nstr.n.toString()); + } + return exports; + } + + /** + * If the module defines an {@code __all__} variable, returns the string + * elements of the variable's list value. + * @return any exported name nodes found, or an empty list if none found + */ + public List getExportedNameNodes() throws Exception { + List exports = new ArrayList(); + if (!getType().isModuleType()) { + return exports; + } + NBinding all = getTable().lookupLocal("__all__"); + if (all== null) { + return exports; + } + Def def = all.getSignatureNode(); + if (def == null) { + return exports; + } + NNode __all__ = getDeepestNodeAtOffset(def.start()); + if (!(__all__ instanceof NName)) { + return exports; + } + NNode assign = __all__.getParent(); + if (!(assign instanceof NAssign)) { + return exports; + } + NNode rvalue = ((NAssign)assign).rvalue; + if (!(rvalue instanceof NList)) { + return exports; + } + for (NNode elt : ((NList)rvalue).elts) { + if (elt instanceof NStr) { + NStr nstr = (NStr)elt; + if (nstr.n != null) { + exports.add(nstr); + } + } + } + return exports; + } + + public String toLongString() { + return ""; + } + + @Override + public String toString() { + return ""; + } + + @Override + public void visit(NNodeVisitor v) { + if (v.visit(this)) { + visitNode(body, v); + } + } +} Index: tests/java/org/python/indexer/data/pkg/mineral/metal/__init__.py =================================================================== Index: tests/java/org/python/indexer/data/yinw/yinw-11.py =================================================================== --- tests/java/org/python/indexer/data/yinw/yinw-11.py (revision 0) +++ tests/java/org/python/indexer/data/yinw/yinw-11.py (revision 0) @@ -0,0 +1,2 @@ +x = 1 +print x