cmd/jsonimports: add static analyzer for consistent "json" imports (#17669)
This migrates an internal tool to open source so that we can run it on the tailscale.com module as well. We add the "util/safediff" also as a dependency of the tool. This PR does not yet set up a CI to run this analyzer. Updates tailscale/corp#791 Signed-off-by: Joe Tsai <joetsai@digital-static.net>
This commit is contained in:
Joe Tsai
GitHub
@@ -0,0 +1,175 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"go/ast"
|
||||
"go/format"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"go/types"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"tailscale.com/util/must"
|
||||
)
|
||||
|
||||
// mustFormatFile formats a Go source file and adjust "json" imports.
|
||||
// It panics if there are any parsing errors.
|
||||
//
|
||||
// - "encoding/json" is imported under the name "jsonv1" or "jsonv1std"
|
||||
// - "encoding/json/v2" is rewritten to import "github.com/go-json-experiment/json" instead
|
||||
// - "encoding/json/jsontext" is rewritten to import "github.com/go-json-experiment/json/jsontext" instead
|
||||
// - "github.com/go-json-experiment/json" is imported under the name "jsonv2"
|
||||
// - "github.com/go-json-experiment/json/v1" is imported under the name "jsonv1"
|
||||
//
|
||||
// If no changes to the file is made, it returns input.
|
||||
func mustFormatFile(in []byte) (out []byte) {
|
||||
fset := token.NewFileSet()
|
||||
f := must.Get(parser.ParseFile(fset, "", in, parser.ParseComments))
|
||||
|
||||
// Check for the existence of "json" imports.
|
||||
jsonImports := make(map[string][]*ast.ImportSpec)
|
||||
for _, imp := range f.Imports {
|
||||
switch pkgPath := must.Get(strconv.Unquote(imp.Path.Value)); pkgPath {
|
||||
case
|
||||
"encoding/json",
|
||||
"encoding/json/v2",
|
||||
"encoding/json/jsontext",
|
||||
"github.com/go-json-experiment/json",
|
||||
"github.com/go-json-experiment/json/v1",
|
||||
"github.com/go-json-experiment/json/jsontext":
|
||||
jsonImports[pkgPath] = append(jsonImports[pkgPath], imp)
|
||||
}
|
||||
}
|
||||
if len(jsonImports) == 0 {
|
||||
return in
|
||||
}
|
||||
|
||||
// Best-effort local type-check of the file
|
||||
// to resolve local declarations to detect shadowed variables.
|
||||
typeInfo := &types.Info{Uses: make(map[*ast.Ident]types.Object)}
|
||||
(&types.Config{
|
||||
Error: func(err error) {},
|
||||
}).Check("", fset, []*ast.File{f}, typeInfo)
|
||||
|
||||
// Rewrite imports to instead use "github.com/go-json-experiment/json".
|
||||
// This ensures that code continues to build even if
|
||||
// goexperiment.jsonv2 is *not* specified.
|
||||
// As of https://github.com/go-json-experiment/json/pull/186,
|
||||
// imports to "github.com/go-json-experiment/json" are identical
|
||||
// to the standard library if built with goexperiment.jsonv2.
|
||||
for fromPath, toPath := range map[string]string{
|
||||
"encoding/json/v2": "github.com/go-json-experiment/json",
|
||||
"encoding/json/jsontext": "github.com/go-json-experiment/json/jsontext",
|
||||
} {
|
||||
for _, imp := range jsonImports[fromPath] {
|
||||
imp.Path.Value = strconv.Quote(toPath)
|
||||
jsonImports[toPath] = append(jsonImports[toPath], imp)
|
||||
}
|
||||
delete(jsonImports, fromPath)
|
||||
}
|
||||
|
||||
// While in a transitory state, where both v1 and v2 json imports
|
||||
// may exist in our codebase, always explicitly import with
|
||||
// either jsonv1 or jsonv2 in the package name to avoid ambiguities
|
||||
// when looking at a particular Marshal or Unmarshal call site.
|
||||
renames := make(map[string]string) // mapping of old names to new names
|
||||
deletes := make(map[*ast.ImportSpec]bool) // set of imports to delete
|
||||
for pkgPath, imps := range jsonImports {
|
||||
var newName string
|
||||
switch pkgPath {
|
||||
case "encoding/json":
|
||||
newName = "jsonv1"
|
||||
// If "github.com/go-json-experiment/json/v1" is also imported,
|
||||
// then use jsonv1std for "encoding/json" to avoid a conflict.
|
||||
if len(jsonImports["github.com/go-json-experiment/json/v1"]) > 0 {
|
||||
newName += "std"
|
||||
}
|
||||
case "github.com/go-json-experiment/json":
|
||||
newName = "jsonv2"
|
||||
case "github.com/go-json-experiment/json/v1":
|
||||
newName = "jsonv1"
|
||||
}
|
||||
|
||||
// Rename the import if different than expected.
|
||||
if oldName := importName(imps[0]); oldName != newName && newName != "" {
|
||||
renames[oldName] = newName
|
||||
pos := imps[0].Pos() // preserve original positioning
|
||||
imps[0].Name = ast.NewIdent(newName)
|
||||
imps[0].Name.NamePos = pos
|
||||
}
|
||||
|
||||
// For all redundant imports, use the first imported name.
|
||||
for _, imp := range imps[1:] {
|
||||
renames[importName(imp)] = importName(imps[0])
|
||||
deletes[imp] = true
|
||||
}
|
||||
}
|
||||
if len(deletes) > 0 {
|
||||
f.Imports = slices.DeleteFunc(f.Imports, func(imp *ast.ImportSpec) bool {
|
||||
return deletes[imp]
|
||||
})
|
||||
for _, decl := range f.Decls {
|
||||
if genDecl, ok := decl.(*ast.GenDecl); ok && genDecl.Tok == token.IMPORT {
|
||||
genDecl.Specs = slices.DeleteFunc(genDecl.Specs, func(spec ast.Spec) bool {
|
||||
return deletes[spec.(*ast.ImportSpec)]
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(renames) > 0 {
|
||||
ast.Walk(astVisitor(func(n ast.Node) bool {
|
||||
if sel, ok := n.(*ast.SelectorExpr); ok {
|
||||
if id, ok := sel.X.(*ast.Ident); ok {
|
||||
// Just because the selector looks like "json.Marshal"
|
||||
// does not mean that it is referencing the "json" package.
|
||||
// There could be a local "json" declaration that shadows
|
||||
// the package import. Check partial type information
|
||||
// to see if there was a local declaration.
|
||||
if obj, ok := typeInfo.Uses[id]; ok {
|
||||
if _, ok := obj.(*types.PkgName); !ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if newName, ok := renames[id.String()]; ok {
|
||||
id.Name = newName
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}), f)
|
||||
}
|
||||
|
||||
bb := new(bytes.Buffer)
|
||||
must.Do(format.Node(bb, fset, f))
|
||||
return must.Get(format.Source(bb.Bytes()))
|
||||
}
|
||||
|
||||
// importName is the local package name used for an import.
|
||||
// If no explicit local name is used, then it uses string parsing
|
||||
// to derive the package name from the path, relying on the convention
|
||||
// that the package name is the base name of the package path.
|
||||
func importName(imp *ast.ImportSpec) string {
|
||||
if imp.Name != nil {
|
||||
return imp.Name.String()
|
||||
}
|
||||
pkgPath, _ := strconv.Unquote(imp.Path.Value)
|
||||
pkgPath = strings.TrimRight(pkgPath, "/v0123456789") // exclude version directories
|
||||
return path.Base(pkgPath)
|
||||
}
|
||||
|
||||
// astVisitor is a function that implements [ast.Visitor].
|
||||
type astVisitor func(ast.Node) bool
|
||||
|
||||
func (f astVisitor) Visit(node ast.Node) ast.Visitor {
|
||||
if !f(node) {
|
||||
return nil
|
||||
}
|
||||
return f
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/format"
|
||||
"testing"
|
||||
|
||||
"tailscale.com/util/must"
|
||||
"tailscale.com/util/safediff"
|
||||
)
|
||||
|
||||
func TestFormatFile(t *testing.T) {
|
||||
tests := []struct{ in, want string }{{
|
||||
in: `package foobar
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
jsonv2exp "github.com/go-json-experiment/json"
|
||||
)
|
||||
|
||||
func main() {
|
||||
json.Marshal()
|
||||
jsonv2exp.Marshal()
|
||||
{
|
||||
var json T // deliberately shadow "json" package name
|
||||
json.Marshal() // should not be re-written
|
||||
}
|
||||
}
|
||||
`,
|
||||
want: `package foobar
|
||||
|
||||
import (
|
||||
jsonv1 "encoding/json"
|
||||
jsonv2 "github.com/go-json-experiment/json"
|
||||
)
|
||||
|
||||
func main() {
|
||||
jsonv1.Marshal()
|
||||
jsonv2.Marshal()
|
||||
{
|
||||
var json T // deliberately shadow "json" package name
|
||||
json.Marshal() // should not be re-written
|
||||
}
|
||||
}
|
||||
`,
|
||||
}, {
|
||||
in: `package foobar
|
||||
|
||||
import (
|
||||
"github.com/go-json-experiment/json"
|
||||
jsonv2exp "github.com/go-json-experiment/json"
|
||||
)
|
||||
|
||||
func main() {
|
||||
json.Marshal()
|
||||
jsonv2exp.Marshal()
|
||||
}
|
||||
`,
|
||||
want: `package foobar
|
||||
import (
|
||||
jsonv2 "github.com/go-json-experiment/json"
|
||||
)
|
||||
func main() {
|
||||
jsonv2.Marshal()
|
||||
jsonv2.Marshal()
|
||||
}
|
||||
`,
|
||||
}, {
|
||||
in: `package foobar
|
||||
import "github.com/go-json-experiment/json/v1"
|
||||
func main() {
|
||||
json.Marshal()
|
||||
}
|
||||
`,
|
||||
want: `package foobar
|
||||
import jsonv1 "github.com/go-json-experiment/json/v1"
|
||||
func main() {
|
||||
jsonv1.Marshal()
|
||||
}
|
||||
`,
|
||||
}, {
|
||||
in: `package foobar
|
||||
import (
|
||||
"encoding/json"
|
||||
jsonv1in2 "github.com/go-json-experiment/json/v1"
|
||||
)
|
||||
func main() {
|
||||
json.Marshal()
|
||||
jsonv1in2.Marshal()
|
||||
}
|
||||
`,
|
||||
want: `package foobar
|
||||
import (
|
||||
jsonv1std "encoding/json"
|
||||
jsonv1 "github.com/go-json-experiment/json/v1"
|
||||
)
|
||||
func main() {
|
||||
jsonv1std.Marshal()
|
||||
jsonv1.Marshal()
|
||||
}
|
||||
`,
|
||||
}, {
|
||||
in: `package foobar
|
||||
import (
|
||||
"encoding/json"
|
||||
jsonv1in2 "github.com/go-json-experiment/json/v1"
|
||||
)
|
||||
func main() {
|
||||
json.Marshal()
|
||||
jsonv1in2.Marshal()
|
||||
}
|
||||
`,
|
||||
want: `package foobar
|
||||
import (
|
||||
jsonv1std "encoding/json"
|
||||
jsonv1 "github.com/go-json-experiment/json/v1"
|
||||
)
|
||||
func main() {
|
||||
jsonv1std.Marshal()
|
||||
jsonv1.Marshal()
|
||||
}
|
||||
`,
|
||||
}, {
|
||||
in: `package foobar
|
||||
import (
|
||||
"encoding/json"
|
||||
j2 "encoding/json/v2"
|
||||
"encoding/json/jsontext"
|
||||
)
|
||||
func main() {
|
||||
json.Marshal()
|
||||
j2.Marshal()
|
||||
jsontext.NewEncoder
|
||||
}
|
||||
`,
|
||||
want: `package foobar
|
||||
import (
|
||||
jsonv1 "encoding/json"
|
||||
jsonv2 "github.com/go-json-experiment/json"
|
||||
"github.com/go-json-experiment/json/jsontext"
|
||||
)
|
||||
func main() {
|
||||
jsonv1.Marshal()
|
||||
jsonv2.Marshal()
|
||||
jsontext.NewEncoder
|
||||
}
|
||||
`,
|
||||
}}
|
||||
for _, tt := range tests {
|
||||
got := string(must.Get(format.Source([]byte(tt.in))))
|
||||
got = string(mustFormatFile([]byte(got)))
|
||||
want := string(must.Get(format.Source([]byte(tt.want))))
|
||||
if got != want {
|
||||
diff, _ := safediff.Lines(got, want, -1)
|
||||
t.Errorf("mismatch (-got +want)\n%s", diff)
|
||||
t.Error(got)
|
||||
t.Error(want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
// The jsonimports tool formats all Go source files in the repository
|
||||
// to enforce that "json" imports are consistent.
|
||||
//
|
||||
// With Go 1.25, the "encoding/json/v2" and "encoding/json/jsontext"
|
||||
// packages are now available under goexperiment.jsonv2.
|
||||
// This leads to possible confusion over the following:
|
||||
//
|
||||
// - "encoding/json"
|
||||
// - "encoding/json/v2"
|
||||
// - "encoding/json/jsontext"
|
||||
// - "github.com/go-json-experiment/json/v1"
|
||||
// - "github.com/go-json-experiment/json"
|
||||
// - "github.com/go-json-experiment/json/jsontext"
|
||||
//
|
||||
// In order to enforce consistent usage, we apply the following rules:
|
||||
//
|
||||
// - Until the Go standard library formally accepts "encoding/json/v2"
|
||||
// and "encoding/json/jsontext" into the standard library
|
||||
// (i.e., they are no longer considered experimental),
|
||||
// we forbid any code from directly importing those packages.
|
||||
// Go code should instead import "github.com/go-json-experiment/json"
|
||||
// and "github.com/go-json-experiment/json/jsontext".
|
||||
// The latter packages contain aliases to the standard library
|
||||
// if built on Go 1.25 with the goexperiment.jsonv2 tag specified.
|
||||
//
|
||||
// - Imports of "encoding/json" or "github.com/go-json-experiment/json/v1"
|
||||
// must be explicitly imported under the package name "jsonv1".
|
||||
// If both packages need to be imported, then the former should
|
||||
// be imported under the package name "jsonv1std".
|
||||
//
|
||||
// - Imports of "github.com/go-json-experiment/json"
|
||||
// must be explicitly imported under the package name "jsonv2".
|
||||
//
|
||||
// The latter two rules exist to provide clarity when reading code.
|
||||
// Without them, it is unclear whether "json.Marshal" refers to v1 or v2.
|
||||
// With them, however, it is clear that "jsonv1.Marshal" is calling v1 and
|
||||
// that "jsonv2.Marshal" is calling v2.
|
||||
//
|
||||
// TODO(@joetsai): At this present moment, there is no guidance given on
|
||||
// whether to use v1 or v2 for newly written Go source code.
|
||||
// I will write a document in the near future providing more guidance.
|
||||
// Feel free to continue using v1 "encoding/json" as you are accustomed to.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"tailscale.com/syncs"
|
||||
"tailscale.com/util/must"
|
||||
"tailscale.com/util/safediff"
|
||||
)
|
||||
|
||||
func main() {
|
||||
update := flag.Bool("update", false, "update all Go source files")
|
||||
flag.Parse()
|
||||
|
||||
// Change working directory to Git repository root.
|
||||
repoRoot := strings.TrimSuffix(string(must.Get(exec.Command(
|
||||
"git", "rev-parse", "--show-toplevel",
|
||||
).Output())), "\n")
|
||||
must.Do(os.Chdir(repoRoot))
|
||||
|
||||
// Iterate over all indexed files in the Git repository.
|
||||
var printMu sync.Mutex
|
||||
var group sync.WaitGroup
|
||||
sema := syncs.NewSemaphore(runtime.NumCPU())
|
||||
var numDiffs int
|
||||
files := string(must.Get(exec.Command("git", "ls-files").Output()))
|
||||
for file := range strings.Lines(files) {
|
||||
sema.Acquire()
|
||||
group.Go(func() {
|
||||
defer sema.Release()
|
||||
|
||||
// Ignore non-Go source files.
|
||||
file = strings.TrimSuffix(file, "\n")
|
||||
if !strings.HasSuffix(file, ".go") {
|
||||
return
|
||||
}
|
||||
|
||||
// Format all "json" imports in the Go source file.
|
||||
srcIn := must.Get(os.ReadFile(file))
|
||||
srcOut := mustFormatFile(srcIn)
|
||||
|
||||
// Print differences with each formatted file.
|
||||
if !bytes.Equal(srcIn, srcOut) {
|
||||
numDiffs++
|
||||
|
||||
printMu.Lock()
|
||||
fmt.Println(file)
|
||||
lines, _ := safediff.Lines(string(srcIn), string(srcOut), -1)
|
||||
for line := range strings.Lines(lines) {
|
||||
fmt.Print("\t", line)
|
||||
}
|
||||
fmt.Println()
|
||||
printMu.Unlock()
|
||||
|
||||
// If -update is specified, write out the changes.
|
||||
if *update {
|
||||
mode := must.Get(os.Stat(file)).Mode()
|
||||
must.Do(os.WriteFile(file, srcOut, mode))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
|
||||
// Report whether any differences were detected.
|
||||
if numDiffs > 0 && !*update {
|
||||
fmt.Printf(`%d files with "json" imports that need formatting`+"\n", numDiffs)
|
||||
fmt.Println("Please run:")
|
||||
fmt.Println("\t./tool/go run tailscale.com/cmd/jsonimports -update")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user