tool/updateflakes, cmd/nardump: replace update-flake.sh with Go tool
Consolidate go.mod.sri and go.toolchain.rev.sri into a single flakehashes.json file at the repo root, owned by a new Go program at tool/updateflakes. The JSON is consumed by flake.nix via builtins.fromJSON and by any future Go code via the FlakeHashes struct that defines its schema. Each block records its input fingerprint alongside the SRI it produced: the goModSum (a sha256 over go.mod and go.sum) for the vendor block, and the literal rev string from go.toolchain.rev for the toolchain block. updateflakes regenerates a block only when its recorded fingerprint disagrees with the current input. Doing the gating by content rather than file mtimes avoids the usual mtime hazards across git checkouts, clones, and merges. It also means re-runs with no input changes are essentially free, and a re-run that touches only one input pays only for that one block. The two blocks have no shared state -- vendor invokes go mod vendor into one tempdir, toolchain fetches and extracts a tarball into another -- so they run concurrently via errgroup. Cold time is bounded by the slower of the two rather than their sum. Also takes the opportunity to fold the toolchain fetch into a single curl|tar pipeline (no intermediate .tar.gz on disk). Split cmd/nardump into a thin package main and a new package nardump library at cmd/nardump/nardump that holds the NAR encoder and SRI helper. tool/updateflakes imports the library directly rather than building and exec'ing the nardump binary at runtime. The library uses fs.ReadLink (Go 1.25+) instead of os.Readlink, so it no longer requires the caller to chdir into the FS root for symlink targets to resolve. WriteNAR now wraps its writer in a bufio.Writer internally (unless the caller already passed one) and flushes on return, so callers don't pay for tiny writes against slow underlying writers. The cache-busting line in flake.nix and shell.nix is known to live at end of file, so updateCacheBust walks the lines in reverse. make tidy timings on this machine, before: ~14s every run. After: warm (no input changes): 0.05s vendor block stale only: 1.4s toolchain block stale only: 5.0s cold (no flakehashes.json): 5.0s Updates #6845 Change-Id: I0340608798f1614abf147a491bf7c68a198a0db4 Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
This commit is contained in:
Brad Fitzpatrick
committed by
Brad Fitzpatrick
Brad Fitzpatrick
parent
33714211c8
commit
88cb6f58f8
+7
-167
@@ -9,22 +9,13 @@
|
||||
// git-pull-oss.sh having Nix available.
|
||||
package main
|
||||
|
||||
// For the format, see:
|
||||
// See https://gist.github.com/jbeda/5c79d2b1434f0018d693
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
|
||||
"tailscale.com/cmd/nardump/nardump"
|
||||
)
|
||||
|
||||
var sri = flag.Bool("sri", false, "print SRI")
|
||||
@@ -34,167 +25,16 @@ func main() {
|
||||
if flag.NArg() != 1 {
|
||||
log.Fatal("usage: nardump <dir>")
|
||||
}
|
||||
arg := flag.Arg(0)
|
||||
if err := os.Chdir(arg); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fsys := os.DirFS(flag.Arg(0))
|
||||
if *sri {
|
||||
hash := sha256.New()
|
||||
if err := writeNAR(hash, os.DirFS(".")); err != nil {
|
||||
s, err := nardump.SRI(fsys)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("sha256-%s\n", base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
||||
fmt.Println(s)
|
||||
return
|
||||
}
|
||||
bw := bufio.NewWriter(os.Stdout)
|
||||
if err := writeNAR(bw, os.DirFS(".")); err != nil {
|
||||
if err := nardump.WriteNAR(os.Stdout, fsys); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
bw.Flush()
|
||||
}
|
||||
|
||||
// writeNARError is a sentinel panic type that's recovered by writeNAR
|
||||
// and converted into the wrapped error.
|
||||
type writeNARError struct{ err error }
|
||||
|
||||
// narWriter writes NAR files.
|
||||
type narWriter struct {
|
||||
w io.Writer
|
||||
fs fs.FS
|
||||
}
|
||||
|
||||
// writeNAR writes a NAR file to w from the root of fs.
|
||||
func writeNAR(w io.Writer, fs fs.FS) (err error) {
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
if we, ok := e.(writeNARError); ok {
|
||||
err = we.err
|
||||
return
|
||||
}
|
||||
panic(e)
|
||||
}
|
||||
}()
|
||||
nw := &narWriter{w: w, fs: fs}
|
||||
nw.str("nix-archive-1")
|
||||
return nw.writeDir(".")
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeDir(dirPath string) error {
|
||||
ents, err := fs.ReadDir(nw.fs, dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(ents, func(i, j int) bool {
|
||||
return ents[i].Name() < ents[j].Name()
|
||||
})
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("directory")
|
||||
for _, ent := range ents {
|
||||
nw.str("entry")
|
||||
nw.str("(")
|
||||
nw.str("name")
|
||||
nw.str(ent.Name())
|
||||
nw.str("node")
|
||||
mode := ent.Type()
|
||||
sub := path.Join(dirPath, ent.Name())
|
||||
var err error
|
||||
switch {
|
||||
case mode.IsDir():
|
||||
err = nw.writeDir(sub)
|
||||
case mode.IsRegular():
|
||||
err = nw.writeRegular(sub)
|
||||
case mode&os.ModeSymlink != 0:
|
||||
err = nw.writeSymlink(sub)
|
||||
default:
|
||||
return fmt.Errorf("unsupported file type %v at %q", sub, mode)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(")")
|
||||
}
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeRegular(path string) error {
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("regular")
|
||||
fi, err := fs.Stat(nw.fs, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fi.Mode()&0111 != 0 {
|
||||
nw.str("executable")
|
||||
nw.str("")
|
||||
}
|
||||
contents, err := fs.ReadFile(nw.fs, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str("contents")
|
||||
if err := writeBytes(nw.w, contents); err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeSymlink(path string) error {
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("symlink")
|
||||
nw.str("target")
|
||||
// broken symlinks are valid in a nar
|
||||
// given we do os.chdir(dir) and os.dirfs(".") above
|
||||
// readlink now resolves relative links even if they are broken
|
||||
link, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(link)
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) str(s string) {
|
||||
if err := writeString(nw.w, s); err != nil {
|
||||
panic(writeNARError{err})
|
||||
}
|
||||
}
|
||||
|
||||
func writeString(w io.Writer, s string) error {
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(len(s)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w, s); err != nil {
|
||||
return err
|
||||
}
|
||||
return writePad(w, len(s))
|
||||
}
|
||||
|
||||
func writeBytes(w io.Writer, b []byte) error {
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(len(b)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
return writePad(w, len(b))
|
||||
}
|
||||
|
||||
func writePad(w io.Writer, n int) error {
|
||||
pad := n % 8
|
||||
if pad == 0 {
|
||||
return nil
|
||||
}
|
||||
var zeroes [8]byte
|
||||
_, err := w.Write(zeroes[:8-pad])
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
// Package nardump writes a NAR (Nix Archive) representation of an
|
||||
// fs.FS to an io.Writer, or summarizes it as a Subresource Integrity
|
||||
// hash, as used by Nix flake.nix vendor and toolchain hashes.
|
||||
//
|
||||
// For the format, see:
|
||||
// https://gist.github.com/jbeda/5c79d2b1434f0018d693
|
||||
package nardump
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// WriteNAR writes a NAR-encoded representation of fsys, rooted at
|
||||
// the FS root, to w.
|
||||
//
|
||||
// The encoder issues many small writes; if w is not already a
|
||||
// *bufio.Writer, WriteNAR wraps it in one and flushes on return so
|
||||
// the caller doesn't have to.
|
||||
//
|
||||
// fsys must implement fs.ReadLinkFS to encode any symlinks it
|
||||
// contains; os.DirFS satisfies this on Go 1.25+.
|
||||
func WriteNAR(w io.Writer, fsys fs.FS) (err error) {
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
if we, ok := e.(writeNARError); ok {
|
||||
err = we.err
|
||||
return
|
||||
}
|
||||
panic(e)
|
||||
}
|
||||
}()
|
||||
bw, ok := w.(*bufio.Writer)
|
||||
if !ok {
|
||||
bw = bufio.NewWriter(w)
|
||||
defer func() {
|
||||
if flushErr := bw.Flush(); err == nil {
|
||||
err = flushErr
|
||||
}
|
||||
}()
|
||||
}
|
||||
nw := &narWriter{w: bw, fs: fsys}
|
||||
nw.str("nix-archive-1")
|
||||
return nw.writeDir(".")
|
||||
}
|
||||
|
||||
// SRI returns the Subresource Integrity hash of the NAR encoding of
|
||||
// fsys, in the form "sha256-<base64>". This is the format Nix
|
||||
// expects for vendorHash and similar fields.
|
||||
func SRI(fsys fs.FS) (string, error) {
|
||||
h := sha256.New()
|
||||
if err := WriteNAR(h, fsys); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "sha256-" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// writeNARError is a sentinel panic type that's recovered by
|
||||
// WriteNAR and converted into the wrapped error.
|
||||
type writeNARError struct{ err error }
|
||||
|
||||
// narWriter writes NAR files.
|
||||
type narWriter struct {
|
||||
w io.Writer
|
||||
fs fs.FS
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeDir(dirPath string) error {
|
||||
ents, err := fs.ReadDir(nw.fs, dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(ents, func(i, j int) bool {
|
||||
return ents[i].Name() < ents[j].Name()
|
||||
})
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("directory")
|
||||
for _, ent := range ents {
|
||||
nw.str("entry")
|
||||
nw.str("(")
|
||||
nw.str("name")
|
||||
nw.str(ent.Name())
|
||||
nw.str("node")
|
||||
mode := ent.Type()
|
||||
sub := path.Join(dirPath, ent.Name())
|
||||
var err error
|
||||
switch {
|
||||
case mode.IsDir():
|
||||
err = nw.writeDir(sub)
|
||||
case mode.IsRegular():
|
||||
err = nw.writeRegular(sub)
|
||||
case mode&fs.ModeSymlink != 0:
|
||||
err = nw.writeSymlink(sub)
|
||||
default:
|
||||
return fmt.Errorf("unsupported file type %v at %q", sub, mode)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(")")
|
||||
}
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeRegular(p string) error {
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("regular")
|
||||
fi, err := fs.Stat(nw.fs, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fi.Mode()&0111 != 0 {
|
||||
nw.str("executable")
|
||||
nw.str("")
|
||||
}
|
||||
contents, err := fs.ReadFile(nw.fs, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str("contents")
|
||||
if err := writeBytes(nw.w, contents); err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeSymlink(p string) error {
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("symlink")
|
||||
nw.str("target")
|
||||
link, err := fs.ReadLink(nw.fs, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(link)
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) str(s string) {
|
||||
if err := writeString(nw.w, s); err != nil {
|
||||
panic(writeNARError{err})
|
||||
}
|
||||
}
|
||||
|
||||
func writeString(w io.Writer, s string) error {
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(len(s)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w, s); err != nil {
|
||||
return err
|
||||
}
|
||||
return writePad(w, len(s))
|
||||
}
|
||||
|
||||
func writeBytes(w io.Writer, b []byte) error {
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(len(b)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
return writePad(w, len(b))
|
||||
}
|
||||
|
||||
func writePad(w io.Writer, n int) error {
|
||||
pad := n % 8
|
||||
if pad == 0 {
|
||||
return nil
|
||||
}
|
||||
var zeroes [8]byte
|
||||
_, err := w.Write(zeroes[:8-pad])
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package nardump
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// setupTmpdir sets up a known golden layout, covering all allowed file/folder types in a nar.
|
||||
func setupTmpdir(t *testing.T) string {
|
||||
t.Helper()
|
||||
tmpdir := t.TempDir()
|
||||
must := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
must(os.MkdirAll(filepath.Join(tmpdir, "sub/dir"), 0755))
|
||||
must(os.Symlink("brokenfile", filepath.Join(tmpdir, "brokenlink")))
|
||||
must(os.Symlink("sub/dir", filepath.Join(tmpdir, "dirl")))
|
||||
must(os.Symlink("/abs/nonexistentdir", filepath.Join(tmpdir, "dirb")))
|
||||
f, err := os.Create(filepath.Join(tmpdir, "sub/dir/file1"))
|
||||
must(err)
|
||||
f.Close()
|
||||
f, err = os.Create(filepath.Join(tmpdir, "file2m"))
|
||||
must(err)
|
||||
must(f.Truncate(2 * 1024 * 1024))
|
||||
f.Close()
|
||||
must(os.Symlink("../file2m", filepath.Join(tmpdir, "sub/goodlink")))
|
||||
return tmpdir
|
||||
}
|
||||
|
||||
func TestWriteNAR(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
// Skip test on Windows as the Nix package manager is not supported on this platform
|
||||
t.Skip("nix package manager is not available on Windows")
|
||||
}
|
||||
dir := setupTmpdir(t)
|
||||
// obtained via `nix-store --dump /tmp/... | sha256sum` of the above test dir
|
||||
const expected = "727613a36f41030e93a4abf2649c3ec64a2757ccff364e3f6f7d544eb976e442"
|
||||
h := sha256.New()
|
||||
if err := WriteNAR(h, os.DirFS(dir)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fmt.Sprintf("%x", h.Sum(nil)); got != expected {
|
||||
t.Fatalf("sha256sum of nar: got %s, want %s", got, expected)
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// setupTmpdir sets up a known golden layout, covering all allowed file/folder types in a nar
|
||||
func setupTmpdir(t *testing.T) string {
|
||||
tmpdir := t.TempDir()
|
||||
pwd, _ := os.Getwd()
|
||||
os.Chdir(tmpdir)
|
||||
defer os.Chdir(pwd)
|
||||
os.MkdirAll("sub/dir", 0755)
|
||||
os.Symlink("brokenfile", "brokenlink")
|
||||
os.Symlink("sub/dir", "dirl")
|
||||
os.Symlink("/abs/nonexistentdir", "dirb")
|
||||
os.Create("sub/dir/file1")
|
||||
f, _ := os.Create("file2m")
|
||||
_ = f.Truncate(2 * 1024 * 1024)
|
||||
f.Close()
|
||||
os.Symlink("../file2m", "sub/goodlink")
|
||||
return tmpdir
|
||||
}
|
||||
|
||||
func TestWriteNar(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
// Skip test on Windows as the Nix package manager is not supported on this platform
|
||||
t.Skip("nix package manager is not available on Windows")
|
||||
}
|
||||
dir := setupTmpdir(t)
|
||||
t.Run("nar", func(t *testing.T) {
|
||||
// obtained via `nix-store --dump /tmp/... | sha256sum` of the above test dir
|
||||
expected := "727613a36f41030e93a4abf2649c3ec64a2757ccff364e3f6f7d544eb976e442"
|
||||
h := sha256.New()
|
||||
os.Chdir(dir)
|
||||
err := writeNAR(h, os.DirFS("."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash := fmt.Sprintf("%x", h.Sum(nil))
|
||||
if expected != hash {
|
||||
t.Fatal("sha256sum of nar not matched", hash, expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user