// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // This program builds the Tailscale Appliance Gokrazy image. // // As of 2024-06-02 this is a exploratory work in progress and is // not intended for serious use. // // Tracking issue is https://github.com/tailscale/tailscale/issues/1866 package main import ( "encoding/json" "flag" "fmt" "log" "os" "os/exec" "path/filepath" "strings" "time" "tailscale.com/gokrazy/mkfs" ) var ( app = flag.String("app", "tsapp", "appliance name; one of the subdirectories of gokrazy/") bucket = flag.String("bucket", "tskrazy-import", "S3 bucket to upload disk image to while making AMI") build = flag.Bool("build", false, "if true, just build locally and stop, without uploading") gaf = flag.Bool("gaf", false, "if true, build a gokrazy archive format file instead of a full disk image") jsonOut = flag.Bool("json", false, "emit one machine-readable JSON result line to stdout") region = flag.String("region", "", "AWS region for import+register; default us-east-1 (honors $AWS_REGION)") ) // awsRegion is the resolved AWS region used for every aws invocation. Set once // in main from resolveRegion. var awsRegion string // result is the machine-readable output printed to stdout when --json is set. // Fields are populated as the run progresses; per docs/cli.md, existing fields // keep their meaning and consumers must tolerate new ones. type result struct { App string `json:"app"` Arch string `json:"arch"` Name string `json:"name,omitempty"` Image string `json:"image,omitempty"` GAF string `json:"gaf,omitempty"` Region string `json:"region,omitempty"` Snapshot string `json:"snapshot,omitempty"` AMI string `json:"ami,omitempty"` } var res result // baseImageSizeBytes is the size of the disk image we ask monogok to // produce (and that the AWS AMI import expects). It has to be large // enough to fit gokrazy's standard partition layout (see // github.com/bradfitz/monogok/disklayout): // // 4 MiB gap before the first partition // 100 MiB boot (FAT) // 500 MiB root A (squashfs; the partition OTA updates write into) // 500 MiB root B (squashfs) // ~96 MiB /perm (ext4; rest of the disk minus the secondary GPT) // // Bump this to give /perm more room (and to make the produced .img // file larger). The same value is passed to monogok via // --target_storage_bytes and to mkfs.Perm so the GPT and the ext4 // inside it agree on the disk's size. // // imageSizeBytesFor may round this up; callers should use that helper // instead of this constant. const baseImageSizeBytes = 1258299392 // imageSizeBytesFor returns the disk image size to use for app. For Raspberry // Pi appliances the size is rounded up to the next power of two because // qemu-system-aarch64's raspi3b machine rejects SD card images whose size // isn't a power of two. func imageSizeBytesFor(app string) int64 { if !strings.HasPrefix(app, "tsapp-pi.") { return baseImageSizeBytes } n := int64(1) for n < baseImageSizeBytes { n <<= 1 } return n } var conf gokrazyConfig // gokrazyConfig is the subset of gokrazy/internal/config.Struct // that we care about. type gokrazyConfig struct { // Environment is os.Environment pairs to use when // building userspace. // See https://gokrazy.org/userguide/instance-config/#environment Environment []string } func (c *gokrazyConfig) GOARCH() string { for _, e := range c.Environment { if v, ok := strings.CutPrefix(e, "GOARCH="); ok { return v } } return "" } func main() { flag.Parse() if *app == "" || strings.Contains(*app, "/") { log.Fatalf("--app must be non-empty name such as 'tsapp' or 'natlabapp'") } confJSON, err := os.ReadFile(filepath.Join(*app, "config.json")) if err != nil { log.Fatalf("reading config.json: %v", err) } if err := json.Unmarshal(confJSON, &conf); err != nil { log.Fatalf("unmarshaling config.json: %v", err) } switch conf.GOARCH() { case "amd64", "arm64": default: log.Fatalf("config.json GOARCH %q must be amd64 or arm64", conf.GOARCH()) } awsRegion = resolveRegion(*region, os.Getenv("AWS_REGION")) res.App = *app res.Arch = awsArch(conf.GOARCH()) if err := buildImage(); err != nil { log.Fatalf("build image: %v", err) } if *build || *gaf { log.Printf("built. stopping.") emitJSON() return } if err := copyToS3(); err != nil { log.Fatalf("copy to S3: %v", err) } importTask, err := startImportSnapshot() if err != nil { log.Fatalf("start import snapshot: %v", err) } snapID, err := waitForImportSnapshot(importTask) if err != nil { log.Fatalf("waitForImportSnapshot(%v): %v", importTask, err) } log.Printf("snap ID: %v", snapID) res.Snapshot = snapID res.Region = awsRegion res.Name = amiName(*app) ami, err := makeAMI(res.Name, snapID) if err != nil { log.Fatalf("makeAMI: %v", err) } log.Printf("made AMI: %v", ami) res.AMI = ami emitJSON() } // awsArch maps a Go GOARCH to the AWS EC2 --architecture value. func awsArch(goarch string) string { switch goarch { case "arm64": return "arm64" case "amd64": return "x86_64" } return "" } // resolveRegion picks the AWS region: an explicit --region wins, then // $AWS_REGION, then us-east-1 (where the Marketplace Catalog API and its // source AMI live). func resolveRegion(flagVal, env string) string { if flagVal != "" { return flagVal } if env != "" { return env } return "us-east-1" } // emitJSON writes the result as one JSON line to stdout when --json is set. // Everything else in this program goes to stderr, so stdout is a clean data // channel for scripts. func emitJSON() { if !*jsonOut { return } if err := json.NewEncoder(os.Stdout).Encode(&res); err != nil { log.Fatalf("encoding json result: %v", err) } } func buildImage() error { dir, err := os.Getwd() if err != nil { return err } if fi, err := os.Stat(filepath.Join(dir, *app)); err != nil || !fi.IsDir() { return fmt.Errorf("in wrong directory %v; no %q subdirectory found", dir, *app) } args := []string{"run", "github.com/bradfitz/monogok/cmd/monogok"} if *gaf { args = append(args, "overwrite", "--gaf", filepath.Join(dir, *app+".gaf"), ) } else { args = append(args, "overwrite", "--full", filepath.Join(dir, *app+".img"), fmt.Sprintf("--target_storage_bytes=%d", imageSizeBytesFor(*app)), ) } cmd := exec.Command("go", args...) cmd.Dir = filepath.Join(dir, *app) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return err } if *gaf { res.GAF = filepath.Join(dir, *app+".gaf") return nil } imgPath := filepath.Join(dir, *app+".img") f, err := os.OpenFile(imgPath, os.O_RDWR, 0) if err != nil { return fmt.Errorf("open %s: %w", imgPath, err) } defer f.Close() if err := mkfs.Perm(f, imageSizeBytesFor(*app)); err != nil { return fmt.Errorf("formatting /perm in %s: %v", imgPath, err) } log.Printf("Wrote ext4 /perm filesystem to %s.", imgPath) res.Image = imgPath return nil } // amiName returns a deterministic AMI name derived from git: on a tagged commit // it's - (releases); otherwise -- for // ad-hoc builds. If git is unavailable it falls back to -. func amiName(app string) string { exact, _ := gitOutput("describe", "--exact-match", "--tags", "HEAD") describe, _ := gitOutput("describe", "--tags", "--always", "--dirty") return amiNameFrom(app, exact, describe, time.Now().Unix()) } // amiNameFrom is the pure decision behind amiName, split out for testing. func amiNameFrom(app, exactTag, describe string, now int64) string { if exactTag != "" { return app + "-" + exactTag } if describe != "" { return fmt.Sprintf("%s-%s-%d", app, describe, now) } return fmt.Sprintf("%s-%d", app, now) } // gitOutput runs git with args and returns trimmed stdout, or an error (e.g. no // git, not a repo, or the ref doesn't match). func gitOutput(args ...string) (string, error) { out, err := exec.Command("git", args...).Output() if err != nil { return "", err } return strings.TrimSpace(string(out)), nil } // awsCmd builds an aws command with the resolved --region prepended so every // call targets the same region deterministically. func awsCmd(args ...string) *exec.Cmd { return exec.Command("aws", append([]string{"--region", awsRegion}, args...)...) } func copyToS3() error { cmd := awsCmd("s3", "cp", *app+".img", "s3://"+*bucket+"/") cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr return cmd.Run() } func startImportSnapshot() (importTaskID string, err error) { out, err := awsCmd("ec2", "import-snapshot", "--disk-container", "Url=s3://"+*bucket+"/"+*app+".img").CombinedOutput() if err != nil { return "", fmt.Errorf("import snapshot: %v: %s", err, out) } var resp struct { ImportTaskID string `json:"ImportTaskId"` } /* { "ImportTaskId": "import-snap-0d2d72622b4359567", "SnapshotTaskDetail": { "DiskImageSize": 0.0, "Progress": "0", "Status": "active", "StatusMessage": "pending", "Url": "s3://tskrazy-import/tskrazy.img" }, "Tags": [] } */ if err := json.Unmarshal(out, &resp); err != nil { return "", fmt.Errorf("unmarshal response: %v: %s", err, out) } return resp.ImportTaskID, nil } /* % aws ec2 describe-import-snapshot-tasks --import-task-ids import-snap-0d2d72622b4359567 { "ImportSnapshotTasks": [ { "ImportTaskId": "import-snap-0d2d72622b4359567", "SnapshotTaskDetail": { "DiskImageSize": 1258299392.0, "Format": "RAW", "SnapshotId": "snap-053efd3539d787927", "Status": "completed", "Url": "s3://tskrazy-import/tskrazy.img", "UserBucket": { "S3Bucket": "tskrazy-import", "S3Key": "tskrazy.img" } }, "Tags": [] } ] } */ func waitForImportSnapshot(importTaskID string) (snapID string, err error) { for { out, err := awsCmd("ec2", "describe-import-snapshot-tasks", "--import-task-ids", importTaskID).CombinedOutput() if err != nil { return "", fmt.Errorf("describe import snapshot tasks: %v: %s", err, out) } var resp struct { ImportSnapshotTasks []struct { SnapshotTaskDetail struct { SnapshotID string `json:"SnapshotId"` Status string `json:"Status"` } `json:"SnapshotTaskDetail"` } `json:"ImportSnapshotTasks"` } if err := json.Unmarshal(out, &resp); err != nil { return "", fmt.Errorf("unmarshal response: %v: %s", err, out) } if len(resp.ImportSnapshotTasks) > 0 { first := &resp.ImportSnapshotTasks[0] if first.SnapshotTaskDetail.Status == "completed" { return first.SnapshotTaskDetail.SnapshotID, nil } } log.Printf("Still waiting; got: %s", out) time.Sleep(5 * time.Second) // TODO(bradfitz): percentage bar? // Looks like: /* 2024/05/14 13:03:21 Still waiting; got: { "ImportSnapshotTasks": [ { "ImportTaskId": "import-snap-0232251d0fbcb33fd", "SnapshotTaskDetail": { "DiskImageSize": 1258299392.0, "Format": "RAW", "Progress": "32", "Status": "active", "StatusMessage": "validated", "Url": "s3://tskrazy-import/tskrazy.img", "UserBucket": { "S3Bucket": "tskrazy-import", "S3Key": "tskrazy.img" } }, "Tags": [] } ] }*/ } } func makeAMI(name, ebsSnapID string) (ami string, err error) { var arch, bootMode string switch conf.GOARCH() { case "arm64": // arm64 instances boot UEFI-only; "uefi-preferred" is rejected. arch, bootMode = "arm64", "uefi" case "amd64": arch, bootMode = "x86_64", "uefi-preferred" default: return "", fmt.Errorf("unknown arch %q", conf.GOARCH()) } out, err := awsCmd("ec2", "register-image", "--name", name, "--architecture", arch, // register-image defaults to paravirtual; arm64 rejects that // ("supports HVM AMIs only") and amd64 would produce an image that // won't boot on Nitro. Both need HVM. "--virtualization-type", "hvm", "--root-device-name", "/dev/sda1", "--ena-support", "--imds-support", "v2.0", "--boot-mode", bootMode, "--block-device-mappings", "DeviceName=/dev/sda1,Ebs={SnapshotId="+ebsSnapID+"}").CombinedOutput() if err != nil { return "", fmt.Errorf("register image: %v: %s", err, out) } /* On success: { "ImageId": "ami-052e1538166886ad2" } */ var resp struct { ImageID string `json:"ImageId"` } if err := json.Unmarshal(out, &resp); err != nil { return "", fmt.Errorf("unmarshal response: %v: %s", err, out) } if resp.ImageID == "" { return "", fmt.Errorf("empty image ID in response: %s", out) } return resp.ImageID, nil }