Tools for analyzing use of built-in functions.

go.mod

@@ -34,14 +34,16 @@ require (
 	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
 	golang.org/x/net v0.0.0-20211208012354-db4efeb81f4b
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/tools v0.1.9
 	google.golang.org/grpc v1.45.0
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
 	pgregory.net/rapid v0.4.7
 )
 
+require github.com/google/go-cmp v0.5.7
+
 require (
 	github.com/creachadair/atomicfile v0.2.4
-	github.com/google/go-cmp v0.5.7
 	gotest.tools v2.2.0+incompatible
 )
 
@@ -83,9 +85,9 @@ require (
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/esimonov/ifshort v1.0.4 // indirect
 	github.com/ettle/strcase v0.1.1 // indirect
-	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
+	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
+	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fatih/structtag v1.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect

go.sum

@@ -267,12 +267,12 @@ github.com/esimonov/ifshort v1.0.4 h1:6SID4yGWfRae/M7hkVDVVyppy8q/v9OuxNdmjLQStB
 github.com/esimonov/ifshort v1.0.4/go.mod h1:Pe8zjlRrJ80+q2CxHLfEOfTwxCZ4O+MuhcHcfgNWTk0=
 github.com/ettle/strcase v0.1.1 h1:htFueZyVeE1XNnMEfbqp5r67qAN/4r6ya1ysq8Q+Zcw=
 github.com/ettle/strcase v0.1.1/go.mod h1:hzDLsPC7/lwKyBOywSHEP89nt2pDgdy+No1NBA9o9VY=
-github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
-github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
-github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
-github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=

tools/panic/bicall/bicall.go

@@ -0,0 +1,128 @@
+// Package bicall locates calls of built-in functions in Go source.
+package bicall
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io"
+	"strings"
+)
+
+// See https://golang.org/ref/spec#Built-in_functions
+var isBuiltin = map[string]bool{
+	"append":  true,
+	"cap":     true,
+	"close":   true,
+	"complex": true,
+	"copy":    true,
+	"delete":  true,
+	"imag":    true,
+	"len":     true,
+	"make":    true,
+	"new":     true,
+	"panic":   true,
+	"print":   true,
+	"println": true,
+	"real":    true,
+	"recover": true,
+}
+
+// Call represents a call to a built-in function in a source program.
+type Call struct {
+	Name     string         // the name of the built-in function
+	Call     *ast.CallExpr  // the call expression in the AST
+	Site     token.Position // the location of the call
+	Path     []ast.Node     // the AST path to the call
+	Comments []string       // comments attributed to the call site by the parser
+}
+
+// flattenComments extracts the text of the given comment groups, and removes
+// leading and trailing whitespace from them.
+func flattenComments(cgs []*ast.CommentGroup) (out []string) {
+	for _, cg := range cgs {
+		out = append(out, strings.TrimSuffix(cg.Text(), "\n"))
+	}
+	return
+}
+
+// Parse parses the contents of r as a Go source file, and calls f for each
+// call expression targeting a built-in function that occurs in the resulting
+// AST. Location information about each call is attributed to the specified
+// filename.
+//
+// If f reports an error, traversal stops and that error is reported to the
+// caller of Parse.
+func Parse(r io.Reader, filename string, f func(Call) error) error {
+	fset := token.NewFileSet()
+	file, err := parser.ParseFile(fset, filename, r, parser.ParseComments)
+	if err != nil {
+		return fmt.Errorf("parsing %q: %w", filename, err)
+	}
+	cmap := ast.NewCommentMap(fset, file, file.Comments)
+
+	var path []ast.Node
+
+	// Find the comments associated with node. If the node does not have its own
+	// comments, scan upward for a statement containing the node.
+	commentsFor := func(node ast.Node) []string {
+		if cgs := cmap[node]; cgs != nil {
+			return flattenComments(cgs)
+		}
+		for i := len(path) - 2; i >= 0; i-- {
+			if _, ok := path[i].(ast.Stmt); !ok {
+				continue
+			}
+			if cgs := cmap[path[i]]; cgs != nil {
+				return flattenComments(cgs)
+			} else {
+				break
+			}
+		}
+		return nil
+	}
+	v := &visitor{
+		visit: func(node ast.Node) error {
+			if node == nil {
+				path = path[:len(path)-1]
+				return nil
+			}
+			path = append(path, node)
+			if call, ok := node.(*ast.CallExpr); ok {
+				id, ok := call.Fun.(*ast.Ident)
+				if !ok || !isBuiltin[id.Name] {
+					return nil
+				}
+
+				if err := f(Call{
+					Name:     id.Name,
+					Call:     call,
+					Site:     fset.Position(call.Pos()),
+					Path:     path,
+					Comments: commentsFor(node),
+				}); err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+	}
+	ast.Walk(v, file)
+	return v.err
+}
+
+type visitor struct {
+	err   error
+	visit func(ast.Node) error
+}
+
+func (v *visitor) Visit(node ast.Node) ast.Visitor {
+	if v.err == nil {
+		v.err = v.visit(node)
+	}
+	if v.err != nil {
+		return nil
+	}
+	return v
+}

tools/panic/bicall/bicall_test.go

@@ -0,0 +1,101 @@
+package bicall_test
+
+import (
+	"log"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/tendermint/tendermint/tools/panic/bicall"
+)
+
+func TestParse(t *testing.T) {
+	want := mustFindNeedles(testInput)
+	sortByLocation(want)
+
+	var got []needle
+	err := bicall.Parse(strings.NewReader(testInput), "testinput.go", func(c bicall.Call) error {
+		got = append(got, needle{
+			Name: c.Name,
+			Line: c.Site.Line,
+			Col:  c.Site.Column - 1,
+		})
+		t.Logf("Found call site for %q at %v", c.Name, c.Site)
+
+		// Verify that the indicator comment shows up attributed to the site.
+		tag := "@" + c.Name
+		if len(c.Comments) != 1 || c.Comments[0] != tag {
+			t.Errorf("Wrong comment at %v: got %+q, want [%q]", c.Site, c.Comments, tag)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Parse unexpectedly failed: %v", err)
+	}
+	sortByLocation(got)
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("Call site mismatch: (-want, +got)\n%s", diff)
+	}
+}
+
+// sortByLocation permutes ns in-place to be ordered by line and column.
+// The specific ordering rule is not important; we just need a consistent order
+// for comparison of test results.
+func sortByLocation(ns []needle) {
+	sort.Slice(ns, func(i, j int) bool {
+		if ns[i].Line == ns[j].Line {
+			return ns[i].Col < ns[j].Col
+		}
+		return ns[i].Line < ns[j].Line
+	})
+}
+
+// To add call sites to the test, include a trailing line comment having the
+// form "//@name", where "name" is a built-in function name.
+// The first offset of that name on the line prior to the comment will become
+// an expected call site for that function.
+const testInput = `
+package testinput
+func main() {
+   defer func() {
+      x := recover() //@recover
+      if x != nil {
+         println("whoa") //@println
+      }
+   }()
+   ip := new(int)                  //@new
+   *ip = 3 + copy([]byte{}, "")    //@copy
+   panic(fmt.Sprintf("ip=%p", ip)) //@panic
+}
+`
+
+// A needle is a name at a location in the source, that is expected to be
+// located in a scan of the input for built-in calls.
+// N.B. Fields are exported to allow comparison by the cmp package.
+type needle struct {
+	Name      string
+	Line, Col int
+}
+
+func mustFindNeedles(src string) []needle {
+	var needles []needle
+	for i, raw := range strings.Split(src, "\n") {
+		tag := strings.SplitN(raw, "//@", 2)
+		if len(tag) == 1 {
+			continue // no needle on this line
+		}
+		name := strings.TrimSpace(tag[1])
+		col := strings.Index(tag[0], name)
+		if col < 0 {
+			log.Panicf("No match for %q on line %d of test input", name, i+1)
+		}
+		needles = append(needles, needle{
+			Name: name,
+			Line: i + 1,
+			Col:  col,
+		})
+	}
+	return needles
+}

tools/panic/callgraph/callgraph.go

@@ -0,0 +1,115 @@
+package callgraph
+
+import (
+	"fmt"
+	"go/ast"
+	"go/types"
+	"sort"
+
+	"golang.org/x/tools/go/loader"
+)
+
+type Triple struct {
+	Caller Entry
+	Target Entry
+	Site   Location
+}
+
+type Entry struct {
+	Package string // canonical import path
+	Name    string // name relative to the package ("" for calls at file scope)
+}
+
+type Location struct {
+	Path      string
+	Offset    int // 0-based
+	Line, Col int // 1-based line, 0-based byte offset
+}
+
+type Graph struct {
+	cfg *loader.Config
+}
+
+func New() *Graph {
+	cfg := new(loader.Config)
+	cfg.TypeCheckFuncBodies = func(ip string) bool {
+		_, ok := cfg.ImportPkgs[ip]
+		return ok
+	}
+	return &Graph{cfg: cfg}
+}
+
+func (g *Graph) Import(ipath string) { g.cfg.Import(ipath) }
+
+func (g *Graph) ImportWithTests(ipath string) { g.cfg.ImportWithTests(ipath) }
+
+func (g *Graph) Process(f func(*Triple)) error {
+	pgm, err := g.cfg.Load()
+	if err != nil {
+		return fmt.Errorf("loading program: %v", err)
+	}
+	var pkgs []*loader.PackageInfo
+	for _, pkg := range pgm.Imported {
+		pkgs = append(pkgs, pkg)
+	}
+	sort.Slice(pkgs, func(i, j int) bool {
+		return pkgs[i].Pkg.Path() < pkgs[j].Pkg.Path()
+	})
+
+	for _, pkg := range pkgs {
+		for _, file := range pkg.Files {
+			fname := g.cfg.Fset.Position(file.Pos()).Filename
+
+			var nodes []ast.Node
+			parent := func() string {
+				for i := len(nodes) - 1; i >= 0; i-- {
+					switch t := nodes[i].(type) {
+					case *ast.FuncDecl:
+						return t.Name.Name
+					}
+				}
+				return fname
+			}
+
+			ast.Walk(visitFunc(func(node ast.Node) {
+				if node == nil {
+					nodes = nodes[:len(nodes)-1]
+					return
+				}
+				nodes = append(nodes, node)
+
+				switch t := node.(type) {
+				case *ast.Ident:
+					ref := pkg.Info.Uses[t]
+					if ref == nil {
+						return // no referent
+					}
+					var refPath string
+					if _, ok := ref.Type().(*types.Signature); ok {
+						refPath = ref.Pkg().Path() // OK, function
+					} else if _, ok := ref.(*types.Builtin); ok {
+						// OK, builtin
+					} else {
+						return // not a function call or reference
+					}
+					pos := g.cfg.Fset.Position(t.Pos())
+					f(&Triple{
+						Caller: Entry{Package: pkg.Pkg.Path(), Name: parent()},
+						Target: Entry{Package: refPath, Name: ref.Name()},
+						Site: Location{
+							Path:   pos.Filename,
+							Offset: pos.Offset,
+							Line:   pos.Line,
+							Col:    pos.Column - 1,
+						},
+					})
+				}
+			}), file)
+		}
+	}
+	return nil
+}
+
+type visitFunc func(ast.Node)
+
+func (v visitFunc) Visit(n ast.Node) ast.Visitor { v(n); return v }

tools/panic/callgraph/callgraph_test.go

@@ -0,0 +1,19 @@
+package callgraph_test
+
+import (
+	"testing"
+
+	"github.com/tendermint/tendermint/tools/panic/callgraph"
+)
+
+func TestStub(t *testing.T) {
+	g := callgraph.New()
+	g.ImportWithTests("github.com/tendermint/tendermint/internal/consensus")
+	if err := g.Process(func(cg *callgraph.Triple) {
+		if cg.Target.Name == "panic" {
+			t.Logf("Panic call at %v", cg.Site)
+		}
+	}); err != nil {
+		t.Fatal(err)
+	}
+}

tools/panic/cmd/findbuiltin/findbuiltin.go

@@ -0,0 +1,131 @@
+// Program findbuiltin locates calls to built-in functions in Go source
+// code, and writes a machine-readable log of where those calls occur.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/tendermint/tendermint/tools/panic/bicall"
+)
+
+var (
+	matchNames    []string // function names to match
+	doPipe        bool     // read paths from stdin
+	doSkipMissing bool     // do not fail for missing files
+)
+
+func init() {
+	flag.Var(stringList{&matchNames}, "match", `Comma-separated function names to select ("" for all)`)
+	flag.BoolVar(&doPipe, "pipe", false, "Read paths from stdin, one per line")
+	flag.BoolVar(&doSkipMissing, "skip-missing", false, "Ignore input paths that are not found")
+
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, `Usage: %[1]s [options] path...  : process the named source files
+       %[1]s [options] -pipe    : process paths from stdin
+
+Scan the specified Go source files for calls to built-in functions, and print a
+log of those calls to stdout.
+
+With -pipe, the program reads paths from stdin, one path per line.
+In this case, paths given on the command-line are consumed first.
+
+Options:
+`, filepath.Base(os.Args[0]))
+		flag.PrintDefaults()
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	// The default input consists of the command-line arguments.
+	// If -pipe is given, also concatenate the contents of stdin.
+	var in io.Reader = strings.NewReader(strings.Join(flag.Args(), "\n"))
+	if doPipe {
+		in = io.MultiReader(in, os.Stdin)
+	}
+
+	lines := bufio.NewScanner(in)
+	for lines.Scan() {
+		mustProcessFile(lines.Text())
+	}
+	if err := lines.Err(); err != nil {
+		log.Fatalf("Error scanning: %v", err)
+	}
+}
+
+func mustProcessFile(path string) {
+	f, err := os.Open(path)
+	if os.IsNotExist(err) && doSkipMissing {
+		log.Printf("File not found: %q [skipped]", path)
+		return
+	} else if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	if err := bicall.Parse(f, path, func(c bicall.Call) error {
+		if !wantFunction(c.Name) {
+			return nil
+		}
+		bits, err := json.Marshal(struct {
+			Name string   `json:"name"`
+			Path string   `json:"path"`
+			Line int      `json:"line"`
+			Col  int      `json:"col"`
+			Com  []string `json:"comments"`
+		}{
+			Name: c.Name,
+			Path: c.Site.Filename,
+			Line: c.Site.Line,
+			Col:  c.Site.Column - 1,
+			Com:  c.Comments,
+		})
+		if err != nil {
+			log.Fatalf("Marshaling output: %v", err)
+		}
+		fmt.Println(string(bits))
+		return nil
+	}); err != nil {
+		log.Fatalf("Parsing %q failed: %v", path, err)
+	}
+}
+
+func wantFunction(name string) bool {
+	if len(matchNames) == 0 {
+		return true
+	}
+	for _, want := range matchNames {
+		if want == name {
+			return true
+		}
+	}
+	return false
+}
+
+type stringList struct{ v *[]string }
+
+func (s stringList) Set(v string) error {
+	ss := strings.Split(v, ",")
+	if len(ss) == 1 && ss[0] == "" {
+		*s.v = nil
+	} else {
+		*s.v = ss
+	}
+	return nil
+}
+
+func (s stringList) String() string {
+	if s.v == nil {
+		return ""
+	}
+	return strings.Join(*s.v, ",")
+}