// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package build builds the Tailscale Appliance Gokrazy image and, // optionally, an AWS AMI from it. // // It is the reusable core behind the gokrazy/build.go command: a // [Builder] runs monogok to produce a disk image (or GAF), formats the // ext4 /perm filesystem via gokrazy/mkfs, and can then upload the image // to S3 and register an AMI by shelling out to the "aws" CLI. Callers // that only want the image (e.g. flash-appliance tooling) can call // [Builder.BuildImage] alone. // // Tracking issue is https://github.com/tailscale/tailscale/issues/1866 package build import ( "context" "encoding/json" "fmt" "io" "log" "os" "os/exec" "path/filepath" "strings" "time" "github.com/mattn/go-isatty" "tailscale.com/gokrazy/mkfs" "tailscale.com/types/logger" ) // Result is the machine-readable outcome of a build. Fields are // populated as the run progresses; per docs/cli.md, existing fields // keep their meaning and consumers must tolerate new ones. type Result struct { App string `json:"app"` Arch string `json:"arch"` Name string `json:"name,omitempty"` Image string `json:"image,omitempty"` GAF string `json:"gaf,omitempty"` Region string `json:"region,omitempty"` Snapshot string `json:"snapshot,omitempty"` AMI string `json:"ami,omitempty"` // Error is the error that ended the build, if any. It lets --json // consumers see why a run failed rather than only its exit code. Error string `json:"error,omitempty"` } // Config configures a [Builder]. Only App is required; New fills in the // rest with defaults. It holds build inputs only; what to produce is // chosen by which build method you call ([Builder.BuildImage], // [Builder.BuildGAF], or [Builder.BuildAMI]). type Config struct { // App is the appliance name, e.g. "tsapp". It must be a // subdirectory of Dir containing a config.json. App string // Dir is the directory holding the appliance subdirectories. // Empty means the current working directory. Dir string // Bucket is the S3 bucket that BuildAMI uploads the disk image to // while registering the AMI. Unused by BuildImage and BuildGAF. Bucket string // Region is the AWS region BuildAMI imports and registers in. Empty // means ResolveRegion("", $AWS_REGION). Region string // Logf receives human-readable progress. If nil, log.Printf is used. Logf logger.Logf // Stderr receives the output of subprocesses (monogok, aws). If nil, // os.Stderr is used. Keeping this off stdout lets callers reserve // stdout for machine-readable output. Stderr io.Writer } // Builder builds one appliance image, GAF, or AMI per its Config. // Create one with [New]; it is not safe for concurrent use. type Builder struct { Config conf gokrazyConfig // parsed //config.json res Result } // baseImageSizeBytes is the size of the disk image we ask monogok to // produce (and that the AWS AMI import expects). It has to be large // enough to fit gokrazy's standard partition layout (see // github.com/bradfitz/monogok/disklayout): // // 4 MiB gap before the first partition // 100 MiB boot (FAT) // 500 MiB root A (squashfs; the partition OTA updates write into) // 500 MiB root B (squashfs) // ~96 MiB /perm (ext4; rest of the disk minus the secondary GPT) // // Bump this to give /perm more room (and to make the produced .img // file larger). The same value is passed to monogok via // --target_storage_bytes and to mkfs.Perm so the GPT and the ext4 // inside it agree on the disk's size. // // imageSizeBytesFor may round this up; callers should use that helper // instead of this constant. const baseImageSizeBytes = 1258299392 // gokrazyConfig is the subset of gokrazy/internal/config.Struct // that we care about. type gokrazyConfig struct { // Environment is os.Environment pairs to use when // building userspace. // See https://gokrazy.org/userguide/instance-config/#environment Environment []string } func (c *gokrazyConfig) GOARCH() string { for _, e := range c.Environment { if v, ok := strings.CutPrefix(e, "GOARCH="); ok { return v } } return "" } // New validates cfg, fills in defaults, reads and parses // //config.json, and validates its GOARCH. func New(cfg Config) (*Builder, error) { if cfg.App == "" || strings.Contains(cfg.App, "/") { return nil, fmt.Errorf("App must be a non-empty name such as 'tsapp' or 'natlabapp'; got %q", cfg.App) } if cfg.Dir == "" { wd, err := os.Getwd() if err != nil { return nil, err } cfg.Dir = wd } if cfg.Region == "" { cfg.Region = ResolveRegion("", os.Getenv("AWS_REGION")) } if cfg.Stderr == nil { cfg.Stderr = os.Stderr } b := &Builder{Config: cfg} confJSON, err := os.ReadFile(filepath.Join(cfg.Dir, cfg.App, "config.json")) if err != nil { return nil, fmt.Errorf("reading config.json: %w", err) } if err := json.Unmarshal(confJSON, &b.conf); err != nil { return nil, fmt.Errorf("unmarshaling config.json: %w", err) } switch b.conf.GOARCH() { case "amd64", "arm64": default: return nil, fmt.Errorf("config.json GOARCH %q must be amd64 or arm64", b.conf.GOARCH()) } b.res.App = cfg.App b.res.Arch = awsArch(b.conf.GOARCH()) return b, nil } // Result returns the current result. It is fully populated after a // successful Build; individual steps fill in their fields as they run, // and Result.Error records the error that ended a failed run. func (b *Builder) Result() Result { return b.res } // fail records err in the result and returns it, so callers that read // Result() after a failed step (e.g. to emit --json) see why it failed. func (b *Builder) fail(err error) error { b.res.Error = err.Error() return err } // logf logs progress via b.Logf, or log.Printf if unset. func (b *Builder) logf(format string, args ...any) { if b.Logf != nil { b.Logf(format, args...) return } log.Printf(format, args...) } // BuildImage runs monogok to produce a full disk image (.img) and // formats its ext4 /perm filesystem. It returns the image path and sets // Result.Image. This is the local artifact that BuildAMI publishes and // that flash-appliance tooling writes to disk. func (b *Builder) BuildImage(ctx context.Context) (string, error) { if err := b.buildImage(ctx, false); err != nil { return "", b.fail(fmt.Errorf("build image: %w", err)) } return b.res.Image, nil } // BuildGAF runs monogok to produce a gokrazy archive format file (.gaf), // the OTA update artifact. It returns the GAF path and sets Result.GAF. // A GAF is an update archive, not a full disk, so it cannot be turned // into an AMI. func (b *Builder) BuildGAF(ctx context.Context) (string, error) { if err := b.buildImage(ctx, true); err != nil { return "", b.fail(fmt.Errorf("build GAF: %w", err)) } return b.res.GAF, nil } // BuildAMI builds a full disk image and publishes it as an AWS AMI: // upload to S3, import an EBS snapshot, and register the image. It // returns the populated Result. Config.Bucket and Config.Region select // where the AMI is built. func (b *Builder) BuildAMI(ctx context.Context) (Result, error) { if _, err := b.BuildImage(ctx); err != nil { return b.res, err // BuildImage already wrapped+recorded it } if err := b.uploadToS3(ctx); err != nil { return b.res, b.fail(fmt.Errorf("copy to S3: %w", err)) } snapID, err := b.importSnapshot(ctx) if err != nil { return b.res, b.fail(fmt.Errorf("import snapshot: %w", err)) } b.logf("snap ID: %v", snapID) if err := b.registerAMI(ctx, snapID); err != nil { return b.res, b.fail(fmt.Errorf("register AMI: %w", err)) } b.logf("made AMI: %v", b.res.AMI) return b.res, nil } // buildImage runs monogok to produce the disk image (or GAF when gaf is // set) and, for a full image, formats the ext4 /perm filesystem. It sets // res.Image or res.GAF. func (b *Builder) buildImage(ctx context.Context, gaf bool) error { appDir := filepath.Join(b.Dir, b.App) if fi, err := os.Stat(appDir); err != nil || !fi.IsDir() { return fmt.Errorf("in wrong directory %v; no %q subdirectory found", b.Dir, b.App) } args := []string{"run", "github.com/bradfitz/monogok/cmd/monogok"} if gaf { args = append(args, "overwrite", "--gaf", filepath.Join(b.Dir, b.App+".gaf"), ) } else { args = append(args, "overwrite", "--full", filepath.Join(b.Dir, b.App+".img"), fmt.Sprintf("--target_storage_bytes=%d", imageSizeBytesFor(b.App)), ) } cmd := exec.CommandContext(ctx, "go", args...) cmd.Dir = appDir cmd.Stdout = b.Stderr cmd.Stderr = b.Stderr if err := cmd.Run(); err != nil { return err } if gaf { b.res.GAF = filepath.Join(b.Dir, b.App+".gaf") return nil } imgPath := filepath.Join(b.Dir, b.App+".img") f, err := os.OpenFile(imgPath, os.O_RDWR, 0) if err != nil { return fmt.Errorf("open %s: %w", imgPath, err) } defer f.Close() if err := mkfs.Perm(f, imageSizeBytesFor(b.App)); err != nil { return fmt.Errorf("formatting /perm in %s: %v", imgPath, err) } b.logf("Wrote ext4 /perm filesystem to %s.", imgPath) b.res.Image = imgPath return nil } // uploadToS3 uploads the built .img to s3:///. func (b *Builder) uploadToS3(ctx context.Context) error { cmd := b.awsCmd(ctx, "s3", "cp", b.App+".img", "s3://"+b.Bucket+"/") cmd.Dir = b.Dir cmd.Stdout = b.Stderr cmd.Stderr = b.Stderr return cmd.Run() } // importSnapshot starts an EC2 import-snapshot task from the uploaded // image and waits for it to complete, returning the EBS snapshot ID. It // sets res.Snapshot and res.Region. func (b *Builder) importSnapshot(ctx context.Context) (string, error) { taskID, err := b.startImportSnapshot(ctx) if err != nil { return "", err } snapID, err := b.waitForImportSnapshot(ctx, taskID) if err != nil { return "", fmt.Errorf("waitForImportSnapshot(%v): %w", taskID, err) } b.res.Snapshot = snapID b.res.Region = b.Region return snapID, nil } func (b *Builder) startImportSnapshot(ctx context.Context) (importTaskID string, err error) { out, err := b.awsCmd(ctx, "ec2", "import-snapshot", "--disk-container", "Url=s3://"+b.Bucket+"/"+b.App+".img").CombinedOutput() if err != nil { return "", fmt.Errorf("import snapshot: %v: %s", err, out) } var resp struct { ImportTaskID string `json:"ImportTaskId"` } /* { "ImportTaskId": "import-snap-0d2d72622b4359567", "SnapshotTaskDetail": { "DiskImageSize": 0.0, "Progress": "0", "Status": "active", "StatusMessage": "pending", "Url": "s3://tskrazy-import/tskrazy.img" }, "Tags": [] } */ if err := json.Unmarshal(out, &resp); err != nil { return "", fmt.Errorf("unmarshal response: %v: %s", err, out) } return resp.ImportTaskID, nil } /* % aws ec2 describe-import-snapshot-tasks --import-task-ids import-snap-0d2d72622b4359567 { "ImportSnapshotTasks": [ { "ImportTaskId": "import-snap-0d2d72622b4359567", "SnapshotTaskDetail": { "DiskImageSize": 1258299392.0, "Format": "RAW", "SnapshotId": "snap-053efd3539d787927", "Status": "completed", "Url": "s3://tskrazy-import/tskrazy.img", "UserBucket": { "S3Bucket": "tskrazy-import", "S3Key": "tskrazy.img" } }, "Tags": [] } ] } */ func (b *Builder) waitForImportSnapshot(ctx context.Context, importTaskID string) (snapID string, err error) { interactive := isTerminal(b.Stderr) var lastPhase string // last (status/message) reported in non-interactive mode for { out, err := b.awsCmd(ctx, "ec2", "describe-import-snapshot-tasks", "--import-task-ids", importTaskID).CombinedOutput() if err != nil { return "", fmt.Errorf("describe import snapshot tasks: %v: %s", err, out) } var resp struct { ImportSnapshotTasks []struct { SnapshotTaskDetail struct { SnapshotID string `json:"SnapshotId"` Status string `json:"Status"` StatusMessage string `json:"StatusMessage"` Progress string `json:"Progress"` } `json:"SnapshotTaskDetail"` } `json:"ImportSnapshotTasks"` } if err := json.Unmarshal(out, &resp); err != nil { return "", fmt.Errorf("unmarshal response: %v: %s", err, out) } var d struct { SnapshotID string `json:"SnapshotId"` Status string `json:"Status"` StatusMessage string `json:"StatusMessage"` Progress string `json:"Progress"` } if len(resp.ImportSnapshotTasks) > 0 { d = resp.ImportSnapshotTasks[0].SnapshotTaskDetail } if d.Status == "completed" { if interactive { fmt.Fprintln(b.Stderr) // move off the live progress line } return d.SnapshotID, nil } if interactive { // Repaint one line in place each poll. fmt.Fprintf(b.Stderr, "\r\x1b[K%s", importProgressLine(d.Status, d.StatusMessage, d.Progress)) } else if phase := d.Status + "/" + d.StatusMessage; phase != lastPhase { // Non-interactive (CI, pipe): one line per phase change only, // not a fresh line every poll. lastPhase = phase b.logf("%s", importProgressLine(d.Status, d.StatusMessage, d.Progress)) } select { case <-ctx.Done(): if interactive { fmt.Fprintln(b.Stderr) } return "", ctx.Err() case <-time.After(5 * time.Second): } } } // registerAMI registers an AMI from the given EBS snapshot. The AMI name // is derived from git via AMIName. It sets res.Name and res.AMI. func (b *Builder) registerAMI(ctx context.Context, ebsSnapID string) error { b.res.Name = AMIName(b.App, b.Dir) var arch, bootMode string switch b.conf.GOARCH() { case "arm64": // arm64 instances boot UEFI-only; "uefi-preferred" is rejected. arch, bootMode = "arm64", "uefi" case "amd64": arch, bootMode = "x86_64", "uefi-preferred" default: return fmt.Errorf("unknown arch %q", b.conf.GOARCH()) } out, err := b.awsCmd(ctx, "ec2", "register-image", "--name", b.res.Name, "--architecture", arch, // register-image defaults to paravirtual; arm64 rejects that // ("supports HVM AMIs only") and amd64 would produce an image that // won't boot on Nitro. Both need HVM. "--virtualization-type", "hvm", "--root-device-name", "/dev/sda1", "--ena-support", "--imds-support", "v2.0", "--boot-mode", bootMode, "--block-device-mappings", "DeviceName=/dev/sda1,Ebs={SnapshotId="+ebsSnapID+"}").CombinedOutput() if err != nil { return fmt.Errorf("register image: %v: %s", err, out) } /* On success: { "ImageId": "ami-052e1538166886ad2" } */ var resp struct { ImageID string `json:"ImageId"` } if err := json.Unmarshal(out, &resp); err != nil { return fmt.Errorf("unmarshal response: %v: %s", err, out) } if resp.ImageID == "" { return fmt.Errorf("empty image ID in response: %s", out) } b.res.AMI = resp.ImageID return nil } // awsCmd builds an aws command with the resolved --region prepended so // every call targets the same region deterministically. func (b *Builder) awsCmd(ctx context.Context, args ...string) *exec.Cmd { return exec.CommandContext(ctx, "aws", append([]string{"--region", b.Region}, args...)...) } // imageSizeBytesFor returns the disk image size to use for app. For Raspberry // Pi appliances the size is rounded up to the next power of two because // qemu-system-aarch64's raspi3b machine rejects SD card images whose size // isn't a power of two. func imageSizeBytesFor(app string) int64 { if !strings.HasPrefix(app, "tsapp-pi.") { return baseImageSizeBytes } n := int64(1) for n < baseImageSizeBytes { n <<= 1 } return n } // importProgressLine formats a one-line status for an in-progress EC2 // import-snapshot task from its Status, StatusMessage, and Progress (a // percentage string like "32") fields, any of which may be empty early on. func importProgressLine(status, statusMessage, progress string) string { msg := statusMessage if msg == "" { msg = status } if msg == "" { msg = "pending" } if progress == "" { return "importing snapshot: " + msg } return fmt.Sprintf("importing snapshot: %s%% (%s)", progress, msg) } // isTerminal reports whether w writes to an interactive terminal, so // callers can show a live-updating progress line instead of a stream of // separate log lines. func isTerminal(w io.Writer) bool { f, ok := w.(*os.File) return ok && isatty.IsTerminal(f.Fd()) } // awsArch maps a Go GOARCH to the AWS EC2 --architecture value. func awsArch(goarch string) string { switch goarch { case "arm64": return "arm64" case "amd64": return "x86_64" } return "" } // ResolveRegion picks the AWS region: an explicit flagVal wins, then // env ($AWS_REGION), then us-east-1 (where the Marketplace Catalog API // and its source AMI live). func ResolveRegion(flagVal, env string) string { if flagVal != "" { return flagVal } if env != "" { return env } return "us-east-1" } // AMIName returns a deterministic AMI name derived from git, running git // in dir: on a tagged commit it's - (releases); otherwise // -- for ad-hoc builds. If git is // unavailable it falls back to -. func AMIName(app, dir string) string { exact, _ := gitOutput(dir, "describe", "--exact-match", "--tags", "HEAD") describe, _ := gitOutput(dir, "describe", "--tags", "--always", "--dirty") return amiNameFrom(app, exact, describe, time.Now().Unix()) } // amiNameFrom is the pure decision behind AMIName, split out for testing. func amiNameFrom(app, exactTag, describe string, now int64) string { if exactTag != "" { return app + "-" + exactTag } if describe != "" { return fmt.Sprintf("%s-%s-%d", app, describe, now) } return fmt.Sprintf("%s-%d", app, now) } // gitOutput runs git with args in dir and returns trimmed stdout, or an // error (e.g. no git, not a repo, or the ref doesn't match). func gitOutput(dir string, args ...string) (string, error) { cmd := exec.Command("git", args...) cmd.Dir = dir out, err := cmd.Output() if err != nil { return "", err } return strings.TrimSpace(string(out)), nil }