diff --git a/.gitattributes b/.gitattributes index 3eb528782..89734e2ba 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,60 @@ -go.mod filter=go-mod -*.go diff=golang +go.mod filter=go-mod eol=lf text +*.go diff=golang eol=lf text +*.adml eol=lf text +*.admx eol=lf text +*.bash eol=lf text +*.c eol=lf text +*.cgi eol=lf text +*.conf eol=lf text +*.css eol=lf text +*.csv eol=lf text +*.desktop eol=lf text +*.fish eol=lf text +*.gitattributes eol=lf text +*.gitignore eol=lf text +*.gitkeep eol=lf text +*.go eol=lf text +*.h eol=lf text +*.helmignore eol=lf text +*.htaccess eol=lf text +*.html eol=lf text +*.hujson eol=lf text +*.in eol=lf text +*.init eol=lf text +*.js eol=lf text +*.json eol=lf text +*.lock eol=lf text +*.lua eol=lf text +*.md eol=lf text +*.mod eol=lf text +*.nix eol=lf text +*.openrc eol=lf text +*.pbxproj eol=lf text +*.pem eol=lf text +*.plg eol=lf text +*.plist eol=lf text +*.rc eol=lf text +*.resolved eol=lf text +*.rev eol=lf text +*.rs eol=lf text +*.sc eol=lf text +*.service eol=lf text +*.sh eol=lf text +*.socket eol=lf text +*.stignore eol=lf text +*.sum eol=lf text +*.svg eol=lf text +*.swift eol=lf text +*.tmpl eol=lf text +*.toml eol=lf text +*.ts eol=lf text +*.tsx eol=lf text +*.txt eol=lf text +*.version eol=lf text +*.xcscheme eol=lf text +*.xcsettings eol=lf text +*.xib eol=lf text +*.xml eol=lf text +*.yaml eol=lf text +*.yml eol=lf text +*.zsh eol=lf text diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index abe6a2c3a..4f3290f58 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,7 @@ jobs: # Install a more recent Go that understands modern go.mod content. - name: Install Go - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # zizmor: ignore[cache-poisoning] v6.3.0 with: go-version-file: go.mod diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index 2b46aa9b0..956435f5f 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install govulncheck - run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest + run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@0782b76014f15f24e22a438f30f308df42899ba1 # 1.3.0 - name: Scan source code for known vulnerabilities run: PATH=$PWD/tool/:$PATH "$(./tool/go env GOPATH)/bin/govulncheck" -test ./... diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml index 6fc8913c4..757c1fd9e 100644 --- a/.github/workflows/installer.yml +++ b/.github/workflows/installer.yml @@ -37,8 +37,6 @@ jobs: - "elementary/docker:stable" - "elementary/docker:unstable" - "parrotsec/core:latest" - - "kalilinux/kali-rolling" - - "kalilinux/kali-dev" - "oraclelinux:9" - "oraclelinux:8" - "fedora:latest" @@ -61,6 +59,9 @@ jobs: - { image: "debian:stable-slim", deps: "curl" } - { image: "ubuntu:24.04", deps: "curl" } - { image: "fedora:latest", deps: "curl" } + # Kali doesn't have ca-certificates installed by default anymore + - { image: "kalilinux/kali-dev", "deps": "curl ca-certificates"} + - { image: "kalilinux/kali-rolling", "deps": "curl ca-certificates"} # Test TAILSCALE_VERSION pinning on a subset of distros. # Skip Alpine as community repos don't reliably keep old versions. - { image: "debian:stable-slim", deps: "curl", version: "1.80.0" } @@ -68,7 +69,7 @@ jobs: - { image: "fedora:latest", deps: "curl", version: "1.80.0" } runs-on: ubuntu-latest container: - image: ${{ matrix.image }} + image: ${{ matrix.image }} # zizmor: ignore[unpinned-images] options: --user root steps: - name: install dependencies (pacman) diff --git a/.github/workflows/natlab-integrationtest.yml b/.github/workflows/natlab-basic.yml similarity index 52% rename from .github/workflows/natlab-integrationtest.yml rename to .github/workflows/natlab-basic.yml index 162153cb2..1a19acfb8 100644 --- a/.github/workflows/natlab-integrationtest.yml +++ b/.github/workflows/natlab-basic.yml @@ -1,6 +1,7 @@ -# Run some natlab integration tests. +# Run a single natlab smoke test on every PR. The full natlab suite +# is opt-in and lives in .github/workflows/natlab-test.yml. # See https://github.com/tailscale/tailscale/issues/13038 -name: "natlab-integrationtest" +name: "natlab-basic" concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -17,17 +18,28 @@ on: branches: - "main" jobs: - natlab-integrationtest: + EasyEasy: runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm - name: Install qemu run: | sudo rm -f /var/lib/man-db/auto-update sudo apt-get -y update sudo apt-get -y remove man-db sudo apt-get install -y qemu-system-x86 qemu-utils + - name: Build VM image + # The test will build this if missing, but we do it explicitly + # to avoid cutting into the go test -timeout budget, and to + # fail earlier with a clearer error if the image build breaks. + run: | + make -C gokrazy natlab - name: Run natlab integration tests run: | - ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/integration/nat --run-vm-tests + ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/natlab/vmtest --run-vm-tests diff --git a/.github/workflows/natlab-test.yml b/.github/workflows/natlab-test.yml new file mode 100644 index 000000000..05dea6d0d --- /dev/null +++ b/.github/workflows/natlab-test.yml @@ -0,0 +1,182 @@ +# Run the full natlab/vmtest opt-in test suite. These tests boot QEMU VMs +# (gokrazy, Ubuntu, FreeBSD) and exercise vnet-driven networking scenarios. +# They are gated behind --run-vm-tests because they need KVM and are slow. +# +# This workflow runs: +# - on demand (workflow_dispatch) +# - on PRs that carry the "run-natlab-tests" label +# - on main, every 12 hours, via cron +# +# Layout: +# - "prepare" builds the gokrazy VM image, downloads the cloud images +# (Ubuntu, FreeBSD), and discovers every Test* function in the two +# opt-in packages. +# - "test" is a per-TestFoo matrix that depends on prepare. Each matrix +# job restores the shared caches and runs a single test. Adding a new +# TestFoo automatically gets its own job — no workflow edits needed. +# +# A separate workflow (.github/workflows/natlab-basic.yml) runs a single +# canary natlab test on every PR; this one runs the full suite. +name: "natlab-test" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + types: [labeled, synchronize, reopened] + schedule: + # Every 12 hours, off-the-hour to avoid GitHub's :00 cron-stampede window. + - cron: "23 3,15 * * *" + +jobs: + # prepare warms the per-workflow-run caches (gokrazy image, cloud VM + # images) and emits the dynamic matrix of test names. By doing the work + # once here, the matrix test jobs never race to rebuild or re-download + # the same artifacts on a cold cache. + prepare: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-natlab-tests')) + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + matrix: ${{ steps.list.outputs.matrix }} + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # The cloud VM image cache is keyed only on images.go (image URLs and + # SHAs), so it survives across workflow runs and is invalidated only + # when a new image source is added. + - name: Cache cloud VM images + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ~/.cache/tailscale/vmtest/images + key: natlab-vmimages-${{ hashFiles('tstest/natlab/vmtest/images.go') }} + + # The gokrazy VM image is keyed by github.sha. That means we rebuild + # it once per commit but matrix test jobs in the same run all share + # the result. Per-PR re-runs of the same sha (e.g. a rerun-failed) + # also get the cache. + - name: Cache gokrazy VM image + id: gokrazy-cache + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gokrazy/natlabapp.qcow2 + key: natlab-gokrazy-${{ github.sha }} + + # qemu-utils provides qemu-img, which the gokrazy Makefile uses to + # convert natlabapp.img to qcow2. Only install if we need it (cache + # miss); the test matrix jobs install qemu separately for the runtime. + - name: Install qemu-utils + if: steps.gokrazy-cache.outputs.cache-hit != 'true' + run: | + sudo rm -f /var/lib/man-db/auto-update + sudo apt-get -y update + sudo apt-get -y remove man-db + sudo apt-get install -y qemu-utils + + - name: Download cloud VM images + # natlabprep is idempotent: it checks the cache before downloading. + run: | + ./tool/go run ./tstest/natlab/vmtest/cmd/natlabprep + + - name: Build gokrazy VM image + if: steps.gokrazy-cache.outputs.cache-hit != 'true' + run: | + make -C gokrazy natlab + + - name: Discover tests + id: list + # Grep the test files directly rather than invoking `go test -list` + # so we don't pay the cost of compiling the test binaries here. The + # only test functions in these packages use the canonical + # `func TestFoo(t *testing.T)` signature. + # + # exclude is the set of tests that need special invocation + # (extra flags, a specific environment) and don't fit the + # single-test-per-matrix-job model. They stay runnable locally. + run: | + set -euo pipefail + exclude='^(TestGrid|TestVnetPerf.*)$' + tmp=$(mktemp) + for pkg_dir in tstest/natlab/vmtest tstest/integration/nat; do + pkg="./${pkg_dir}/" + for f in "${pkg_dir}"/*_test.go; do + [ -e "$f" ] || continue + { grep -hE '^func Test[A-Z][A-Za-z0-9_]*\(t \*testing\.T\)' "$f" || true; } \ + | sed -E 's/^func (Test[A-Za-z0-9_]+).*/\1/' \ + | { grep -vE "$exclude" || true; } \ + | while read -r t; do + jq -nc --arg pkg "$pkg" --arg test "$t" \ + '{pkg: $pkg, test: $test}' >> "$tmp" + done + done + done + matrix=$(jq -s -c . "$tmp") + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + echo "Discovered tests:" + jq . "$tmp" + + test: + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 20 + name: "${{ matrix.test }}" + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix) }} + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Install qemu + run: | + sudo rm -f /var/lib/man-db/auto-update + sudo apt-get -y update + sudo apt-get -y remove man-db + sudo apt-get install -y qemu-system-x86 qemu-utils + + # restore-only: prepare is the single writer of these caches, so + # matrix jobs don't write back. fail-on-cache-miss would be too + # strict for the gokrazy cache (e.g. a non-fatal cache eviction + # between prepare and us); we just rebuild on miss instead. + - name: Restore cloud VM images + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ~/.cache/tailscale/vmtest/images + key: natlab-vmimages-${{ hashFiles('tstest/natlab/vmtest/images.go') }} + + - name: Restore gokrazy VM image + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gokrazy/natlabapp.qcow2 + key: natlab-gokrazy-${{ github.sha }} + + # The gokrazy-based tests boot the kernel directly from + # vmlinuz that ships in the gokrazy/kernel.amd64 module. + # Tests look it up under GOMODCACHE via findKernelPath, so the + # module has to be present even though no Go source imports it + # in the test package itself. + - name: Download kernel.amd64 module + run: | + ./tool/go mod download github.com/gokrazy/kernel.amd64 + + - name: Run ${{ matrix.test }} + # Per-test timeout is well above the few-minute typical runtime + # but small enough that a stuck test fails fast instead of holding + # the runner for the job's 20-minute budget. + run: | + ./tool/go test -v -timeout=15m -count=1 ${{ matrix.pkg }} \ + -run='^${{ matrix.test }}$' --run-vm-tests diff --git a/.github/workflows/policybot-test.yml b/.github/workflows/policybot-test.yml new file mode 100644 index 000000000..c943ae0e0 --- /dev/null +++ b/.github/workflows/policybot-test.yml @@ -0,0 +1,45 @@ +name: policybot-test + +env: + HOME: ${{ github.workspace }} + GOMODCACHE: ${{ github.workspace }}/gomodcache + CMD_GO_USE_GIT_HASH: "true" + +on: + push: + branches: + - main + - "release-branch/*" + paths: + - .github/workflows/policybot-test.yml + - .policy.yml + - .policy-tests.yml + - go.mod + pull_request: + paths: + - .github/workflows/policybot-test.yml + - .policy.yml + - .policy-tests.yml + - go.mod + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + policybot-test: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + + # The version of github.com/tailscale/policybottest used here is + # pinned by go.mod via internal/tooldeps/tooldeps.go; bump it with + # "go get github.com/tailscale/policybottest@ && go mod tidy". + - name: Run policy tests + working-directory: src + run: ./tool/go run github.com/tailscale/policybottest -policy .policy.yml -tests .policy-tests.yml diff --git a/.github/workflows/request-dataplane-review.yml b/.github/workflows/request-dataplane-review.yml index 78bd8ff58..1367a6518 100644 --- a/.github/workflows/request-dataplane-review.yml +++ b/.github/workflows/request-dataplane-review.yml @@ -2,7 +2,7 @@ name: request-dataplane-review on: pull_request: - types: [ opened, synchronize, reopened, ready_for_review ] + types: [opened, synchronize, reopened, ready_for_review] paths: - ".github/workflows/request-dataplane-review.yml" - "**/*derp*" @@ -15,8 +15,6 @@ jobs: name: Request Dataplane Review runs-on: ubuntu-latest steps: - - name: Check out code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Get access token uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 id: generate-token @@ -24,6 +22,8 @@ jobs: # Get token for app: https://github.com/apps/change-visibility-bot app-id: ${{ secrets.VISIBILITY_BOT_APP_ID }} private-key: ${{ secrets.VISIBILITY_BOT_APP_PRIVATE_KEY }} + # Limit the token to only requesting reviewers on pull requests. + permission-pull-requests: write - name: Add reviewers env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} diff --git a/.github/workflows/request-k8s-review.yml b/.github/workflows/request-k8s-review.yml new file mode 100644 index 000000000..da335fb88 --- /dev/null +++ b/.github/workflows/request-k8s-review.yml @@ -0,0 +1,39 @@ +name: request-k8s-review + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - ".github/workflows/request-k8s-review.yml" + - "k8s-operator/**" + - "kube/**" + - "cmd/k8s-operator/**" + - "cmd/k8s-proxy/**" + - "cmd/k8s-nameserver/**" + - "cmd/containerboot/**" + - "cmd/sync-containers/**" + - "ipn/store/kubestore/**" + - "docs/k8s/**" + - "!**/depaware.txt" + +jobs: + request-k8s-review: + if: github.event.pull_request.draft == false + name: Request K8s Review + runs-on: ubuntu-latest + steps: + - name: Get access token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + id: generate-token + with: + # Get token for app: https://github.com/apps/change-visibility-bot + app-id: ${{ secrets.VISIBILITY_BOT_APP_ID }} + private-key: ${{ secrets.VISIBILITY_BOT_APP_PRIVATE_KEY }} + # Limit the token to only requesting reviewers on pull requests. + permission-pull-requests: write + - name: Add reviewers + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + url: ${{ github.event.pull_request.html_url }} + run: | + gh pr edit "$url" --add-reviewer tailscale/k8s-devs diff --git a/.github/workflows/ssh-integrationtest.yml b/.github/workflows/ssh-integrationtest.yml index afe2dd2f7..84432cd72 100644 --- a/.github/workflows/ssh-integrationtest.yml +++ b/.github/workflows/ssh-integrationtest.yml @@ -1,5 +1,5 @@ -# Run the ssh integration tests with `make sshintegrationtest`. -# These tests can also be running locally. +# Run the ssh integration tests in various Docker containers. +# These tests can also be run locally via `make sshintegrationtest`. name: "ssh-integrationtest" concurrency: @@ -15,9 +15,25 @@ on: jobs: ssh-integrationtest: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - base: "ubuntu:focal" + tag: "ssh-ubuntu-focal" + - base: "ubuntu:jammy" + tag: "ssh-ubuntu-jammy" + - base: "ubuntu:noble" + tag: "ssh-ubuntu-noble" + - base: "alpine:latest" + tag: "ssh-alpine-latest" steps: - name: Check out code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Run SSH integration tests + - name: Build test binaries run: | - make sshintegrationtest \ No newline at end of file + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled + - name: Run SSH integration tests (${{ matrix.base }}) + run: | + docker build --build-arg="BASE=${{ matrix.base }}" -t "${{ matrix.tag }}" ssh/tailssh/testcontainers diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38ebd1291..a5952f720 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,7 @@ jobs: run: go mod download - name: Cache Go modules if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # zizmor: ignore[cache-poisoning] v5.0.4 with: path: gomodcache # relative to workspace; see env note at top of file key: ${{ steps.hash.outputs.key }} @@ -183,7 +183,7 @@ jobs: TS_TEST_SHARD: ${{ matrix.shard }} - name: bench all working-directory: src - run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ $(for x in $(git grep -l "^func Benchmark" | xargs dirname | sort | uniq); do echo "./$x"; done) + run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run='^$' $(for x in $(git grep -l '^func Benchmark' | xargs dirname | sort | uniq); do echo "./$x"; done) env: GOARCH: ${{ matrix.goarch }} - name: check that no tracked files changed @@ -261,6 +261,7 @@ jobs: cigocached-host: ${{ vars.CIGOCACHED_AZURE_HOST }} - name: test + shell: bash if: matrix.key != 'win-bench' # skip on bench builder working-directory: src run: ./tool/go run ./cmd/testwrapper sharded:${{ matrix.shard }} @@ -268,9 +269,10 @@ jobs: NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI - name: bench all + shell: bash if: matrix.key == 'win-bench' working-directory: src - run: ./tool/go test ./... -bench=. -benchtime=1x -run="^$" + run: ./tool/go test -bench=. -benchtime=1x -run='^$' $(for x in $(git grep -l '^func Benchmark' | xargs dirname | sort | uniq); do echo "./$x"; done) env: NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI @@ -343,7 +345,7 @@ jobs: needs: gomod-cache runs-on: ubuntu-24.04 container: - image: golang:latest + image: golang:latest # zizmor: ignore[unpinned-images] options: --privileged steps: - name: checkout @@ -361,31 +363,7 @@ jobs: run: chown -R $(id -u):$(id -g) $PWD - name: privileged tests working-directory: src - run: ./tool/go test ./util/linuxfw ./derp/xdp - - vm: - needs: gomod-cache - runs-on: ["self-hosted", "linux", "vm"] - # VM tests run with some privileges, don't let them run on 3p PRs. - if: github.repository == 'tailscale/tailscale' - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: src - - name: Restore Go module cache - uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true - - name: Run VM tests - working-directory: src - run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2404 - env: - HOME: "/var/lib/ghrunner/home" - TMPDIR: "/tmp" - XDG_CACHE_HOME: "/var/lib/ghrunner/cache" + run: ./tool/go test $(./tool/go run ./tool/listpkgs --has-root-tests) cross: # cross-compile checks, build only. needs: gomod-cache @@ -642,6 +620,13 @@ jobs: run: | ./tool/go run ./cmd/tsconnect --fast-compression build ./tool/go run ./cmd/tsconnect --fast-compression build-pkg + - name: verify Google Chrome is available + run: | + which google-chrome + google-chrome --version + - name: tsconnect js/wasm headless-browser tests + working-directory: src + run: ./tool/go test ./tstest/integration/jswasmtest/ -v -timeout 180s --run-headless-browser-tests - name: Tidy cache working-directory: src shell: bash @@ -787,6 +772,14 @@ jobs: echo echo git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1) + - name: check that 'genreadme' is clean + working-directory: src + run: | + ./tool/go run ./misc/genreadme + git add -N . # ensure untracked files are noticed + echo + echo + git diff --name-only --exit-code || (echo "The files above need updating. Please run './tool/go run ./misc/genreadme'."; exit 1) make_tidy: runs-on: ubuntu-24.04 @@ -895,7 +888,6 @@ jobs: - test - windows - macos - - vm - cross - ios - wasm @@ -941,7 +933,6 @@ jobs: - test - windows - macos - - vm - cross - ios - wasm @@ -991,7 +982,6 @@ jobs: - test - windows - macos - - vm - wasm - fuzz - race-root-integration diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml index 1304fb222..f594fd6df 100644 --- a/.github/workflows/update-flake.yml +++ b/.github/workflows/update-flake.yml @@ -23,8 +23,8 @@ jobs: - name: Check out code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Run update-flakes - run: ./update-flake.sh + - name: Run updateflakes + run: ./tool/go run ./tool/updateflakes - name: Get access token uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 @@ -33,6 +33,9 @@ jobs: # Get token for app: https://github.com/apps/tailscale-code-updater app-id: ${{ secrets.CODE_UPDATER_APP_ID }} private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} + # Limit the token to only pushing a branch and opening a pull request. + permission-contents: write + permission-pull-requests: write - name: Send pull request uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 #v8.1.0 @@ -41,8 +44,8 @@ jobs: author: Flakes Updater committer: Flakes Updater branch: flakes - commit-message: "go.mod.sri: update SRI hash for go.mod changes" - title: "go.mod.sri: update SRI hash for go.mod changes" + commit-message: "flakehashes.json: update SRI hash for go.mod changes" + title: "flakehashes.json: update SRI hash for go.mod changes" body: Triggered by ${{ github.repository }}@${{ github.sha }} signoff: true delete-branch: true diff --git a/.github/workflows/update-webclient-prebuilt.yml b/.github/workflows/update-webclient-prebuilt.yml index 5bb0573a1..fd8876ecf 100644 --- a/.github/workflows/update-webclient-prebuilt.yml +++ b/.github/workflows/update-webclient-prebuilt.yml @@ -29,6 +29,9 @@ jobs: # Get token for app: https://github.com/apps/tailscale-code-updater app-id: ${{ secrets.CODE_UPDATER_APP_ID }} private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} + # Limit the token to only pushing a branch and opening a pull request. + permission-contents: write + permission-pull-requests: write - name: Send pull request id: pull-request diff --git a/.github/workflows/vet.yml b/.github/workflows/vet.yml index c03190e4f..d8f8e98a4 100644 --- a/.github/workflows/vet.yml +++ b/.github/workflows/vet.yml @@ -14,15 +14,17 @@ on: - main - "release-branch/*" paths: + - .github/workflows/vet.yml - "**.go" pull_request: paths: + - .github/workflows/vet.yml - "**.go" jobs: vet: - runs-on: [ self-hosted, linux ] - timeout-minutes: 5 + runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - name: Check out code diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 000000000..181d02bc2 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,32 @@ +name: security lint GitHub Actions with zizmor + +on: + push: + branches: ["main"] + paths: + - ".github/workflows/**" + pull_request: + branches: ["**"] + paths: + - ".github/workflows/**" + +permissions: {} + +jobs: + zizmor: + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 + with: + min-severity: high + advanced-security: false + annotations: true diff --git a/.gitignore b/.gitignore index 4bfabc80f..9d570d954 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,15 @@ # Binaries for programs and plugins *~ *.tmp -*.exe *.dll *.so *.dylib *.spk +*.exe +# tool/go.exe is built specially and committed. +!/tool/go.exe + cmd/tailscale/tailscale cmd/tailscaled/tailscaled ssh/tailssh/testcontainers/tailscaled @@ -55,3 +58,5 @@ client/web/build/assets # Ignore syncthing state directory. /.stfolder +fbstatus +gafpush diff --git a/.policy-tests.yml b/.policy-tests.yml new file mode 100644 index 000000000..2ee550cde --- /dev/null +++ b/.policy-tests.yml @@ -0,0 +1,258 @@ +# Tests for testdata/tailscale.com.policy.yml. +# +# Run from the parent directory with: +# go run . -policy testdata/tailscale.com.policy.yml \ +# -tests testdata/tailscale.com.policy-test.yml + +teams: + # Members are picked solely so tests below can refer to them. + # `alice` and `bob` are control-protocol-owners *and* dev. `carol`, + # `dave`, and `eve` are dev only. `outsider` is in nothing. + tailscale/control-protocol-owners: + - alice + - bob + tailscale/dev: + - alice + - bob + - carol + - dave + - eve + +tests: + # ------------------------------------------------------------------ + # Baseline: every PR (touching tailcfg/ or not) requires a +1 from a + # tailscale/dev member. + # ------------------------------------------------------------------ + + - name: non-tailcfg PR with no review is pending + pull_request: + author: carol + changed_files: + - README.md + - cmd/tailscale/main.go + expect: + status: pending + rules: + "tailcfg changes approved by control-protocol-owners": skipped + "tailcfg changes overridden by another tailscale/dev": skipped + "any tailscale/dev review": pending + + - name: non-tailcfg PR with dev review is approved + pull_request: + author: carol + changed_files: + - README.md + reviews: + - user: dave + state: approved + expect: + status: approved + rules: + "any tailscale/dev review": approved + + - name: non-tailcfg PR with thumbs-up comment from dev is approved + pull_request: + author: carol + changed_files: + - README.md + comments: + - user: dave + body: ":+1:" + expect: + status: approved + rules: + "any tailscale/dev review": approved + + - name: non-tailcfg PR with non-dev review is pending + pull_request: + author: carol + changed_files: + - README.md + reviews: + - user: outsider + state: approved + expect: + status: pending + rules: + "any tailscale/dev review": pending + + - name: PR with no files at all is pending without a dev review + pull_request: + author: carol + changed_files: [] + expect: + status: pending + rules: + "any tailscale/dev review": pending + + # ------------------------------------------------------------------ + # tailcfg/ requires BOTH a control-protocol-owners review (or an + # override comment) AND a dev +1. Owners are in tailscale/dev too, + # so their single review satisfies both gates. + # ------------------------------------------------------------------ + + - name: tailcfg change without anything is pending + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + expect: + status: pending + rules: + "tailcfg changes approved by control-protocol-owners": pending + "tailcfg changes overridden by another tailscale/dev": pending + "any tailscale/dev review": pending + + - name: a single owner review approves both the owner rule and the dev rule + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + reviews: + - user: alice + state: approved + expect: + status: approved + rules: + "tailcfg changes approved by control-protocol-owners": approved + "any tailscale/dev review": approved + + - name: owner cannot self-approve their own tailcfg PR + pull_request: + author: alice + changed_files: + - tailcfg/tailcfg.go + reviews: + - user: alice + state: approved + expect: + status: pending + + - name: non-owner dev review alone leaves the tailcfg gate pending + pull_request: + author: alice + changed_files: + - tailcfg/tailcfg.go + reviews: + - user: carol + state: approved + expect: + status: pending + rules: + "tailcfg changes approved by control-protocol-owners": pending + "tailcfg changes overridden by another tailscale/dev": pending + "any tailscale/dev review": approved + + # ------------------------------------------------------------------ + # policybot-override flow on a tailcfg/ PR. + # ------------------------------------------------------------------ + + # An override comment by itself only satisfies the override rule. + # The baseline dev-review rule still needs a separate +1. + - name: override comment alone leaves baseline dev review pending + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + comments: + - user: dave + body: "policybot-override: emergency rollback" + expect: + status: pending + rules: + "tailcfg changes overridden by another tailscale/dev": approved + "any tailscale/dev review": pending + + - name: override comment plus a separate dev review approves + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + comments: + - user: dave + body: "policybot-override: emergency rollback" + reviews: + - user: eve + state: approved + expect: + status: approved + + # One dev doing BOTH (post the override AND a regular +1) works too. + - name: same dev posts both the override and a +1 review + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + comments: + - user: dave + body: "policybot-override: shipping a typo fix" + reviews: + - user: dave + state: approved + expect: + status: approved + + - name: override comment from the author does not approve the override + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + comments: + - user: carol + body: "policybot-override: please let me ship this" + expect: + status: pending + rules: + "tailcfg changes overridden by another tailscale/dev": pending + + - name: override comment from a non-dev does not approve the override + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + comments: + - user: outsider + body: "policybot-override: I am not on the dev team" + expect: + rules: + "tailcfg changes overridden by another tailscale/dev": pending + + - name: empty-reason policybot-override does not approve + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + comments: + - user: dave + body: "policybot-override:" + expect: + rules: + "tailcfg changes overridden by another tailscale/dev": pending + + # Defaults regression guards: a normal review approval from a dev + # member must NOT silently approve the override rule. Same for a + # plain :+1: comment. Without the explicit `methods: github_review: + # false` / `comments: []` on the override rule, both would. + - name: a regular dev review does not silently satisfy the override rule + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + reviews: + - user: dave + state: approved + expect: + rules: + "tailcfg changes overridden by another tailscale/dev": pending + + - name: a thumbs-up comment from a dev does not silently satisfy the override rule + pull_request: + author: carol + changed_files: + - tailcfg/tailcfg.go + comments: + - user: dave + body: ":+1:" + expect: + rules: + "tailcfg changes overridden by another tailscale/dev": pending diff --git a/.policy.yml b/.policy.yml new file mode 100644 index 000000000..cd91ccf75 --- /dev/null +++ b/.policy.yml @@ -0,0 +1,84 @@ +# Approval policy for this repository, enforced by policy-bot +# (https://github.com/palantir/policy-bot) running at +# https://policybot.corp.ts.net. +# +# This file replaces the role GitHub's CODEOWNERS played: when a pull +# request touches a path covered by a rule below, policy-bot posts a +# status check that blocks merging until the required reviewers approve. +# +# Policy and rule syntax reference: +# https://github.com/palantir/policy-bot/blob/develop/README.md +# Example policy files (team-approval, disapproval, remote, etc.): +# https://github.com/palantir/policy-bot/tree/develop/config/policy-examples +# +# Do not add to this policy without wide discussion. +# See https://github.com/tailscale/corp/issues/13972. + +policy: + approval: + # tailcfg/ has an extra gate: either a control-protocol-owners + # review or an explicit policybot-override: comment from a dev. + # The block is skipped on PRs that don't touch tailcfg/. + - or: + - tailcfg changes approved by control-protocol-owners + - tailcfg changes overridden by another tailscale/dev + # And every PR, regardless of files, needs a +1 from a dev. + # This rule also keeps the policy from collapsing into "all rules + # skipped" (which policy-bot treats as a failure) on non-tailcfg + # PRs. + - any tailscale/dev review + +approval_rules: + - name: tailcfg changes approved by control-protocol-owners + if: + changed_files: + paths: + - "^tailcfg/" + requires: + count: 1 + teams: + - "tailscale/control-protocol-owners" + + - name: tailcfg changes overridden by another tailscale/dev + description: | + Any member of @tailscale/dev (other than the PR author) can + override the control-protocol-owners requirement by leaving a + comment of the form + + policybot-override: + + on the pull request. The reason can be anything but should + explain why the override is appropriate; it stays in the PR + conversation as a record. The override comment also counts as + that developer's approval. + if: + changed_files: + paths: + - "^tailcfg/" + requires: + count: 1 + teams: + - "tailscale/dev" + options: + methods: + # Explicitly turn off the defaults (github_review: true, + # comments: [":+1:", "👍"]) so the ONLY way to satisfy this + # rule is a "policybot-override:" comment. Otherwise a normal + # review approval or thumbs-up from any tailscale/dev member + # would silently pass the rule. + github_review: false + comments: [] + comment_patterns: + - '^policybot-override: \S.*' + + - name: any tailscale/dev review + description: | + Every PR needs at least one approval from a member of + @tailscale/dev. policy-bot's default approval methods count + a GitHub review approval, a ":+1:" comment, or a "👍" + comment as approval. The PR author cannot approve their + own PR. + requires: + count: 1 + teams: + - "tailscale/dev" diff --git a/CODEOWNERS b/CODEOWNERS index af9b0d9f9..9da519f43 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,7 @@ -/tailcfg/ @tailscale/control-protocol-owners +# This repository does NOT use GitHub's CODEOWNERS for review enforcement. +# Approval policies live in .policy.yml at the repository root and are +# enforced by policy-bot (https://github.com/palantir/policy-bot). +# +# To change required reviewers for a path, edit .policy.yml. +# +# See https://github.com/tailscale/corp/issues/13972. diff --git a/Makefile b/Makefile index b78ef0469..bad81f6e7 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ vet: ## Run go vet tidy: ## Run go mod tidy and update nix flake hashes ./tool/go mod tidy - ./update-flake.sh + ./tool/go run ./tool/updateflakes lint: ## Run golangci-lint ./tool/go run github.com/golangci/golangci-lint/cmd/golangci-lint run @@ -137,15 +137,66 @@ publishdevproxy: check-image-repo ## Build and publish k8s-proxy image to locati sshintegrationtest: ## Run the SSH integration tests in various Docker containers @GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ - echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \ - echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \ - echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \ - echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers + echo "Testing on ubuntu:focal, ubuntu:jammy, ubuntu:noble, alpine:latest (in parallel)" && \ + docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers & \ + docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers & \ + docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers & \ + docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers & \ + wait .PHONY: generate generate: ## Generate code ./tool/go generate ./... +.PHONY: tsapp-build-and-flash-pi +tsapp-build-and-flash-pi: ## Build a tsapp-pi.arm64 GAF from HEAD and flash a local SD card (macOS auto-detects the disk; pass DISK=/dev/sdX on Linux) + cd gokrazy && ../tool/go run build.go --gaf --app=tsapp-pi.arm64 + ./tool/go run --exec=sudo ./cmd/tailscale configure flash-appliance \ + --variant=pi-arm64 \ + --gaf=gokrazy/tsapp-pi.arm64.gaf \ + $(if $(DISK),--disk=$(DISK)) \ + $(if $(wildcard $(HOME)/.ssh/id_ed25519.pub),--add-ssh-authorized-keys=$(HOME)/.ssh/id_ed25519.pub) + +.PHONY: tsapp-qemu-pi +tsapp-qemu-pi: ## Build tsapp-pi.arm64 and boot it under qemu-system-aarch64 with a framebuffer GUI window and working network (requires mtools, dtc, qemu-efi-aarch64) + cd gokrazy && ../tool/go run build.go --build --app=tsapp-pi.arm64 + # Extract the kernel from the FAT boot partition for direct -kernel boot. + rm -f gokrazy/tsapp-pi.arm64.vmlinuz + mcopy -i gokrazy/tsapp-pi.arm64.img@@4194304 ::vmlinuz gokrazy/tsapp-pi.arm64.vmlinuz + # Use the "virt" machine (not raspi3b) because it provides working + # PCI e1000 networking and, with UEFI firmware, an EFI framebuffer + # via the ramfb device. The raspi3b machine's USB NIC emulation is + # too broken for DHCP and its SoC watchdog reboots the guest. + # + # Find the UEFI firmware. Common paths: + # Debian/Ubuntu: /usr/share/qemu-efi-aarch64/QEMU_EFI.fd + # Homebrew: /opt/homebrew/share/qemu/edk2-aarch64-code.fd + # Fedora: /usr/share/edk2/aarch64/QEMU_EFI.fd + QEMU_EFI=$$(for f in \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd \ + /opt/homebrew/share/qemu/edk2-aarch64-code.fd \ + /usr/share/edk2/aarch64/QEMU_EFI.fd \ + $$(dirname $$(which qemu-system-aarch64))/../share/qemu/edk2-aarch64-code.fd; do \ + [ -f "$$f" ] && echo "$$f" && break; \ + done) && \ + [ -n "$$QEMU_EFI" ] || { echo "error: cannot find QEMU EFI firmware (install qemu-efi-aarch64)"; exit 1; } && \ + qemu-system-aarch64 \ + -M virt -cpu cortex-a53 -m 1G \ + -bios "$$QEMU_EFI" \ + -device ramfb \ + -device e1000,netdev=net0 -netdev user,id=net0 \ + -kernel gokrazy/tsapp-pi.arm64.vmlinuz \ + -append "console=ttyAMA0,115200 nowatchdog gokrazy.log_to_serial=1 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init rootwait" \ + -drive file=gokrazy/tsapp-pi.arm64.img,format=raw,if=none,id=disk0 \ + -device virtio-blk-device,drive=disk0 \ + -serial mon:stdio + +.PHONY: tsapp-push-pi +tsapp-push-pi: ## Build a tsapp-pi.arm64 GAF from HEAD and push it to a running Pi over the network (pass PI=) + @[ -n "$(PI)" ] || { echo "usage: make tsapp-push-pi PI="; exit 1; } + cd gokrazy && ../tool/go run build.go --gaf --app=tsapp-pi.arm64 + ./tool/go run ./gokrazy/gafpush --gaf=gokrazy/tsapp-pi.arm64.gaf --pi=$(PI) + .PHONY: pin-github-actions pin-github-actions: ./tool/go tool github.com/stacklok/frizbee actions .github/workflows diff --git a/VERSION.txt b/VERSION.txt index acbb747ac..e402df2dd 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.97.0 +1.103.0 diff --git a/appc/appconnector_test.go b/appc/appconnector_test.go index d14ef68fc..c58aa8041 100644 --- a/appc/appconnector_test.go +++ b/appc/appconnector_test.go @@ -736,6 +736,7 @@ func TestRateLogger(t *testing.T) { } func TestRouteStoreMetrics(t *testing.T) { + clientmetric.ResetForTest(t) metricStoreRoutes(1, 1) metricStoreRoutes(1, 1) // the 1 buckets value should be 2 metricStoreRoutes(5, 5) // the 5 buckets value should be 1 diff --git a/appc/conn25.go b/appc/conn25.go index fd1748fa6..e4c999a7a 100644 --- a/appc/conn25.go +++ b/appc/conn25.go @@ -5,18 +5,20 @@ package appc import ( "cmp" + "fmt" "slices" + "strings" "tailscale.com/ipn/ipnext" "tailscale.com/tailcfg" "tailscale.com/types/appctype" - "tailscale.com/util/mak" + "tailscale.com/types/dnstype" "tailscale.com/util/set" ) const AppConnectorsExperimentalAttrName = "tailscale.com/app-connectors-experimental" -func isEligibleConnector(peer tailcfg.NodeView) bool { +func isPeerEligibleConnector(peer tailcfg.NodeView) bool { if !peer.Valid() || !peer.Hostinfo().Valid() { return false } @@ -39,7 +41,7 @@ func sortByPreference(ns []tailcfg.NodeView) { func PickConnector(nb ipnext.NodeBackend, app appctype.Conn25Attr) []tailcfg.NodeView { appTagsSet := set.SetOf(app.Connectors) matches := nb.AppendMatchingPeers(nil, func(n tailcfg.NodeView) bool { - if !isEligibleConnector(n) { + if !isPeerEligibleConnector(n) { return false } for _, t := range n.Tags().All() { @@ -53,50 +55,32 @@ func PickConnector(nb ipnext.NodeBackend, app appctype.Conn25Attr) []tailcfg.Nod return matches } -// PickSplitDNSPeers looks at the netmap peers capabilities and finds which peers -// want to be connectors for which domains. -func PickSplitDNSPeers(hasCap func(c tailcfg.NodeCapability) bool, self tailcfg.NodeView, peers map[tailcfg.NodeID]tailcfg.NodeView) map[string][]tailcfg.NodeView { - var m map[string][]tailcfg.NodeView +// DNSAddrScheme is the custom URI scheme used for conn25-managed split DNS +// entries to determine the destination at query time rather than configuration +// time. +const DNSAddrScheme = "tailscale-app" + +func AppDNSRoutes(hasCap func(c tailcfg.NodeCapability) bool, self tailcfg.NodeView) map[string][]*dnstype.Resolver { if !hasCap(AppConnectorsExperimentalAttrName) { - return m + return nil } apps, err := tailcfg.UnmarshalNodeCapViewJSON[appctype.AppConnectorAttr](self.CapMap(), AppConnectorsExperimentalAttrName) if err != nil { - return m + return nil } - tagToDomain := make(map[string][]string) + appNamesByDomain := map[string]string{} for _, app := range apps { - for _, tag := range app.Connectors { - tagToDomain[tag] = append(tagToDomain[tag], app.Domains...) + for _, domain := range app.Domains { + domain, _ = strings.CutPrefix(domain, "*.") + domain = strings.ToLower(domain) + // in the case of multiple apps specifying the same domain (which is misconfiguration + // that should be validated at point of input) last write wins. + appNamesByDomain[domain] = app.Name } } - // NodeIDs are Comparable, and we have a map of NodeID to NodeView anyway, so - // use a Set of NodeIDs to deduplicate, and populate into a []NodeView later. - var work map[string]set.Set[tailcfg.NodeID] - for _, peer := range peers { - if !isEligibleConnector(peer) { - continue - } - for _, t := range peer.Tags().All() { - domains := tagToDomain[t] - for _, domain := range domains { - if work[domain] == nil { - mak.Set(&work, domain, set.Set[tailcfg.NodeID]{}) - } - work[domain].Add(peer.ID()) - } - } - } - - // Populate m. Make a []tailcfg.NodeView from []tailcfg.NodeID using the peers map. - // And sort it to our preference. - for domain, ids := range work { - nodes := make([]tailcfg.NodeView, 0, ids.Len()) - for id := range ids { - nodes = append(nodes, peers[id]) - } - sortByPreference(nodes) - mak.Set(&m, domain, nodes) + m := make(map[string][]*dnstype.Resolver, len(appNamesByDomain)) + for domain, appName := range appNamesByDomain { + m[domain] = []*dnstype.Resolver{{Addr: fmt.Sprintf("%s:%s", DNSAddrScheme, appName), UseWithExitNode: true}} } return m } diff --git a/appc/conn25_test.go b/appc/conn25_test.go index fc14caf36..7194a67fe 100644 --- a/appc/conn25_test.go +++ b/appc/conn25_test.go @@ -5,17 +5,18 @@ package appc import ( "encoding/json" - "reflect" + "fmt" "testing" "github.com/google/go-cmp/cmp" "tailscale.com/ipn/ipnext" "tailscale.com/tailcfg" "tailscale.com/types/appctype" + "tailscale.com/types/dnstype" "tailscale.com/types/opt" ) -func TestPickSplitDNSPeers(t *testing.T) { +func TestAppDNSRoutes(t *testing.T) { getBytesForAttr := func(name string, domains []string, tags []string) []byte { attr := appctype.AppConnectorAttr{ Name: name, @@ -32,83 +33,105 @@ func TestPickSplitDNSPeers(t *testing.T) { appTwoBytes := getBytesForAttr("app2", []string{"a.example.com"}, []string{"tag:two"}) appThreeBytes := getBytesForAttr("app3", []string{"woo.b.example.com", "hoo.b.example.com"}, []string{"tag:three1", "tag:three2"}) appFourBytes := getBytesForAttr("app4", []string{"woo.b.example.com", "c.example.com"}, []string{"tag:four1", "tag:four2"}) + appFiveBytes := getBytesForAttr("app5", []string{"*.example.com", "example.com"}, []string{"tag:one"}) + appSixBytes := getBytesForAttr("app6", []string{"*.Example.com", "EXAMPLE.com", "EXAMPLE.COM"}, []string{"tag:one"}) - makeNodeView := func(id tailcfg.NodeID, name string, tags []string) tailcfg.NodeView { - return (&tailcfg.Node{ - ID: id, - Name: name, - Tags: tags, - Hostinfo: (&tailcfg.Hostinfo{AppConnector: opt.NewBool(true)}).View(), - }).View() + resolver := func(appName string) []*dnstype.Resolver { + return []*dnstype.Resolver{{Addr: fmt.Sprintf("%s:%s", DNSAddrScheme, appName), UseWithExitNode: true}} } - nvp1 := makeNodeView(1, "p1", []string{"tag:one"}) - nvp2 := makeNodeView(2, "p2", []string{"tag:four1", "tag:four2"}) - nvp3 := makeNodeView(3, "p3", []string{"tag:two", "tag:three1"}) - nvp4 := makeNodeView(4, "p4", []string{"tag:two", "tag:three2", "tag:four2"}) for _, tt := range []struct { name string - want map[string][]tailcfg.NodeView - peers []tailcfg.NodeView + hasCap bool config []tailcfg.RawMessage + want map[string][]*dnstype.Resolver }{ { - name: "empty", + name: "no-capability", // hasCap false should return nil regardless of config. + hasCap: false, }, { - name: "bad-config", // bad config should return a nil map rather than error. + name: "no-apps", // hasCap true but no configured apps returns an empty map. + hasCap: true, + want: map[string][]*dnstype.Resolver{}, + }, + { + name: "bad-config", // bad config should return nil rather than error. + hasCap: true, config: []tailcfg.RawMessage{tailcfg.RawMessage(`hey`)}, }, { - name: "no-peers", + name: "single-app", + hasCap: true, config: []tailcfg.RawMessage{tailcfg.RawMessage(appOneBytes)}, - }, - { - name: "peers-that-are-not-connectors", - config: []tailcfg.RawMessage{tailcfg.RawMessage(appOneBytes)}, - peers: []tailcfg.NodeView{ - (&tailcfg.Node{ - ID: 5, - Name: "p5", - Tags: []string{"tag:one"}, - }).View(), - (&tailcfg.Node{ - ID: 6, - Name: "p6", - Tags: []string{"tag:one"}, - }).View(), + want: map[string][]*dnstype.Resolver{ + "example.com": resolver("app1"), }, }, { - name: "peers-that-dont-match-tags", - config: []tailcfg.RawMessage{tailcfg.RawMessage(appOneBytes)}, - peers: []tailcfg.NodeView{ - makeNodeView(5, "p5", []string{"tag:seven"}), - makeNodeView(6, "p6", nil), + name: "single-app-multi-domain", + hasCap: true, + config: []tailcfg.RawMessage{tailcfg.RawMessage(appThreeBytes)}, + want: map[string][]*dnstype.Resolver{ + "woo.b.example.com": resolver("app3"), + "hoo.b.example.com": resolver("app3"), }, }, { - name: "matching-tagged-connector-peers", + name: "multi-app-no-overlap", + hasCap: true, config: []tailcfg.RawMessage{ tailcfg.RawMessage(appOneBytes), tailcfg.RawMessage(appTwoBytes), - tailcfg.RawMessage(appThreeBytes), - tailcfg.RawMessage(appFourBytes), }, - peers: []tailcfg.NodeView{ - nvp1, - nvp2, - nvp3, - nvp4, - makeNodeView(5, "p5", nil), + want: map[string][]*dnstype.Resolver{ + "example.com": resolver("app1"), + "a.example.com": resolver("app2"), }, - want: map[string][]tailcfg.NodeView{ - // p5 has no matching tags and so doesn't appear - "example.com": {nvp1}, - "a.example.com": {nvp3, nvp4}, - "woo.b.example.com": {nvp2, nvp3, nvp4}, - "hoo.b.example.com": {nvp3, nvp4}, - "c.example.com": {nvp2, nvp4}, + }, + { + name: "domain-collision-last-write-wins", + hasCap: true, + config: []tailcfg.RawMessage{ + tailcfg.RawMessage(appThreeBytes), // app3: woo.b.example.com, hoo.b.example.com + tailcfg.RawMessage(appFourBytes), // app4: woo.b.example.com, c.example.com + }, + want: map[string][]*dnstype.Resolver{ + // app4 overwrites app3 for the shared domain + "woo.b.example.com": resolver("app4"), + "hoo.b.example.com": resolver("app3"), + "c.example.com": resolver("app4"), + }, + }, + { + name: "wildcards-are-stripped-and-deduped", + hasCap: true, + config: []tailcfg.RawMessage{tailcfg.RawMessage(appFiveBytes)}, + want: map[string][]*dnstype.Resolver{ + // *.example.com and example.com should both normalize to example.com. + "example.com": resolver("app5"), + }, + }, + { + name: "domains-are-normalized-and-deduped", + hasCap: true, + config: []tailcfg.RawMessage{tailcfg.RawMessage(appSixBytes)}, + want: map[string][]*dnstype.Resolver{ + // *.Example.com, EXAMPLE.com, EXAMPLE.COM should all normalize to example.com. + "example.com": resolver("app6"), + }, + }, + { + name: "sub-domains-and-top-domains-do-not-collide", + hasCap: true, + config: []tailcfg.RawMessage{ + tailcfg.RawMessage(appTwoBytes), + tailcfg.RawMessage(appFiveBytes), + }, + want: map[string][]*dnstype.Resolver{ + // *.example.com normalizes to example.com; a.example.com remains distinct. + "a.example.com": resolver("app2"), + "example.com": resolver("app5"), }, }, } { @@ -120,15 +143,11 @@ func TestPickSplitDNSPeers(t *testing.T) { } } selfView := selfNode.View() - peers := map[tailcfg.NodeID]tailcfg.NodeView{} - for _, p := range tt.peers { - peers[p.ID()] = p - } - got := PickSplitDNSPeers(func(_ tailcfg.NodeCapability) bool { - return true - }, selfView, peers) - if !reflect.DeepEqual(got, tt.want) { - t.Fatalf("got %v, want %v", got, tt.want) + got := AppDNSRoutes(func(_ tailcfg.NodeCapability) bool { + return tt.hasCap + }, selfView) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatalf("AppDNSRoutes (-want, +got):\n%s", diff) } }) } diff --git a/build_dist.sh b/build_dist.sh index c05644711..a32fe419e 100755 --- a/build_dist.sh +++ b/build_dist.sh @@ -51,6 +51,13 @@ while [ "$#" -gt 1 ]; do ldflags="$ldflags -w -s" tags="${tags:+$tags,},$(GOOS= GOARCH= $go run ./cmd/featuretags --min)" ;; + --strip) + # --min overrides your flags, when you're using custom tags and want to + # additionally strip symbols to help reduce the size, this is the easiest + # way to do it. + shift + ldflags="$ldflags -w -s" + ;; --box) if [ ! -z "${TAGS:-}" ]; then echo "set either --box or \$TAGS, but not both" diff --git a/cache_key_test.go b/cache_key_test.go new file mode 100644 index 000000000..43de02e13 --- /dev/null +++ b/cache_key_test.go @@ -0,0 +1,57 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package tailscaleroot + +import ( + "os" + "os/exec" + "strings" + "testing" + + "tailscale.com/util/cibuild" +) + +// TestTsgoRevInCacheKey verifies that the Tailscale Go toolchain's git +// revision (from go.toolchain.rev) is blended into Go build cache keys. +// Without this, bumping the toolchain to a new commit that doesn't change +// the Go version number would silently reuse stale cached build artifacts. +// +// See https://github.com/tailscale/tailscale/issues/36589. +func TestTsgoRevInCacheKey(t *testing.T) { + goRoot := goEnv(t, "GOROOT") + isTsgo := strings.Contains(goRoot, "/.cache/tsgo/") + if !cibuild.OnTailscaleCI() && !isTsgo { + t.Skip("skipping; not in Tailscale CI and not using the Tailscale Go toolchain") + } + + rev := strings.TrimSpace(GoToolchainRev) + if rev == "" { + t.Fatal("go.toolchain.rev is empty") + } + + // Build the small stdlib "errors" package with GODEBUG=gocachehash=1, + // which causes cmd/go to log its cache key computations to stderr. + cmd := exec.Command("go", "build", "errors") + cmd.Env = append(os.Environ(), "GODEBUG=gocachehash=1") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("go build errors failed: %v\n%s", err, out) + } + + // The cache key output should contain the toolchain rev alongside the + // Go version, e.g.: + // HASH[moduleIndex]: "go1.26.2 dfe2a5fd8ee2e68b08ce5ff259269f50ecadf2f4" + if !strings.Contains(string(out), rev) { + t.Errorf("go.toolchain.rev %q not found in GODEBUG=gocachehash=1 output:\n%s", rev, out) + } +} + +func goEnv(t *testing.T, key string) string { + t.Helper() + out, err := exec.Command("go", "env", key).Output() + if err != nil { + t.Fatalf("go env %s: %v", key, err) + } + return strings.TrimSpace(string(out)) +} diff --git a/client/local/cert.go b/client/local/cert.go index 701bfe026..0ce56b74f 100644 --- a/client/local/cert.go +++ b/client/local/cert.go @@ -10,13 +10,55 @@ import ( "crypto/tls" "errors" "fmt" + "net/http" "net/url" + "strconv" "strings" "time" "go4.org/mem" ) +// rateLimitedError is returned from cert-fetching methods when the +// upstream ACME CA reported a rate limit. Callers should unpack it via +// [RateLimitRetryAfter]. +type rateLimitedError struct { + retryAfter time.Duration + underlying error +} + +func (e rateLimitedError) Error() string { return e.underlying.Error() } +func (e rateLimitedError) Unwrap() error { return e.underlying } + +// RateLimitRetryAfter reports whether err was a rate-limit failure from +// the upstream ACME CA and, if so, returns the CA's suggested wait +// (zero if none was provided). +func RateLimitRetryAfter(err error) (retryAfter time.Duration, ok bool) { + var rl rateLimitedError + if errors.As(err, &rl) { + return rl.retryAfter, true + } + return 0, false +} + +// retryAfterFromHeader parses a Retry-After header, matching the +// delta-seconds + HTTP-date pattern in tempfork/acme/http.go. +func retryAfterFromHeader(h http.Header) time.Duration { + v := h.Get("Retry-After") + if i, err := strconv.Atoi(v); err == nil { + return time.Duration(i) * time.Second + } + t, err := http.ParseTime(v) + if err != nil { + return 0 + } + d := time.Until(t) + if d < 0 { + return 0 + } + return d +} + // SetDNS adds a DNS TXT record for the given domain name, containing // the provided TXT value. The intended use case is answering // LetsEncrypt/ACME dns-01 challenges. @@ -43,6 +85,8 @@ func (lc *Client) SetDNS(ctx context.Context, name, value string) error { // // It returns a cached certificate from disk if it's still valid. // +// Rate-limit failures can be identified via [RateLimitRetryAfter]. +// // Deprecated: use [Client.CertPair]. func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { return defaultClient.CertPair(ctx, domain) @@ -52,6 +96,8 @@ func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err e // // It returns a cached certificate from disk if it's still valid. // +// Rate-limit failures can be identified via [RateLimitRetryAfter]. +// // API maturity: this is considered a stable API. func (lc *Client) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { return lc.CertPairWithValidity(ctx, domain, 0) @@ -65,10 +111,18 @@ func (lc *Client) CertPair(ctx context.Context, domain string) (certPEM, keyPEM // least the given duration, if permitted by the CA. If the certificate is // valid, but for less than minValidity, it will be synchronously renewed. // +// Rate-limit failures can be identified via [RateLimitRetryAfter]. +// // API maturity: this is considered a stable API. func (lc *Client) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) { res, err := lc.send(ctx, "GET", fmt.Sprintf("/localapi/v0/cert/%s?type=pair&min_validity=%s", domain, minValidity), 200, nil) if err != nil { + if hse, ok := errors.AsType[httpStatusError](err); ok && hse.HTTPStatus == http.StatusTooManyRequests { + return nil, nil, rateLimitedError{ + retryAfter: retryAfterFromHeader(hse.Header), + underlying: err, + } + } return nil, nil, err } // with ?type=pair, the response PEM is first the one private diff --git a/client/local/debugportmapper.go b/client/local/debugportmapper.go index 1cbb3ee0a..869462e64 100644 --- a/client/local/debugportmapper.go +++ b/client/local/debugportmapper.go @@ -50,6 +50,9 @@ type DebugPortmapOpts struct { // process. // // opts can be nil; if so, default values will be used. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts) (io.ReadCloser, error) { vals := make(url.Values) if opts == nil { diff --git a/client/local/local.go b/client/local/local.go index e72589306..889fafd65 100644 --- a/client/local/local.go +++ b/client/local/local.go @@ -2,6 +2,12 @@ // SPDX-License-Identifier: BSD-3-Clause // Package local contains a Go client for the Tailscale LocalAPI. +// +// The APIs in this package vary in maturity: some methods are considered +// stable APIs and are documented as such, while others are not necessarily +// stable and are subject to change between releases. Methods without an +// explicit "API maturity" note in their documentation should be assumed +// to be unstable. package local import ( @@ -135,6 +141,9 @@ func (lc *Client) defaultDialer(ctx context.Context, network, addr string) (net. // authenticating to the local Tailscale daemon vary by platform. // // DoLocalRequest may mutate the request to add Authorization headers. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DoLocalRequest(req *http.Request) (*http.Response, error) { req.Header.Set("Tailscale-Cap", strconv.Itoa(int(tailcfg.CurrentCapabilityVersion))) lc.tsClientOnce.Do(func() { @@ -280,7 +289,7 @@ func (lc *Client) sendWithHeaders( } if res.StatusCode != wantStatus { err = fmt.Errorf("%v: %s", res.Status, bytes.TrimSpace(slurp)) - return nil, nil, httpStatusError{bestError(err, slurp), res.StatusCode} + return nil, nil, httpStatusError{bestError(err, slurp), res.StatusCode, res.Header} } return slurp, res.Header, nil } @@ -288,6 +297,7 @@ func (lc *Client) sendWithHeaders( type httpStatusError struct { error HTTPStatus int + Header http.Header } func (lc *Client) get200(ctx context.Context, path string) ([]byte, error) { @@ -316,6 +326,8 @@ func decodeJSON[T any](b []byte) (ret T, err error) { // For connections proxied by tailscaled, this looks up the owner of the given // address as TCP first, falling back to UDP; if you want to only check a // specific address family, use WhoIsProto. +// +// API maturity: this is considered a stable API. func (lc *Client) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) { body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr)) if err != nil { @@ -327,6 +339,39 @@ func (lc *Client) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsR return decodeJSON[*apitype.WhoIsResponse](body) } +// WhoIsForService is like [Client.WhoIs] but scopes the returned CapMap to +// capabilities that apply to the named VIP service. This enables per-service +// capability resolution on hosts that advertise multiple VIP services. +// +// API maturity: this is considered a stable API. +func (lc *Client) WhoIsForService(ctx context.Context, remoteAddr string, svcName tailcfg.ServiceName) (*apitype.WhoIsResponse, error) { + body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr)+"&svc_name="+url.QueryEscape(string(svcName))) + if err != nil { + if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound { + return nil, ErrPeerNotFound + } + return nil, err + } + return decodeJSON[*apitype.WhoIsResponse](body) +} + +// WhoIsForIP is like [Client.WhoIs] but scopes the returned CapMap to +// capabilities that apply to the given destination IP. The IP may be a +// VIP service address, the node's own tailnet address, or any other +// routable IP the node handles. +// +// API maturity: this is considered a stable API. +func (lc *Client) WhoIsForIP(ctx context.Context, remoteAddr string, dst netip.Addr) (*apitype.WhoIsResponse, error) { + body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr)+"&dst_ip="+url.QueryEscape(dst.String())) + if err != nil { + if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound { + return nil, ErrPeerNotFound + } + return nil, err + } + return decodeJSON[*apitype.WhoIsResponse](body) +} + // ErrPeerNotFound is returned by [Client.WhoIs], [Client.WhoIsNodeKey] and // [Client.WhoIsProto] when a peer is not found. var ErrPeerNotFound = errors.New("peer not found") @@ -334,6 +379,8 @@ var ErrPeerNotFound = errors.New("peer not found") // WhoIsNodeKey returns the owner of the given wireguard public key. // // If not found, the error is ErrPeerNotFound. +// +// API maturity: this is considered a stable API. func (lc *Client) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*apitype.WhoIsResponse, error) { body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(key.String())) if err != nil { @@ -349,6 +396,8 @@ func (lc *Client) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*apityp // IP:port, for the given protocol (tcp or udp). // // If not found, the error is [ErrPeerNotFound]. +// +// API maturity: this is considered a stable API. func (lc *Client) WhoIsProto(ctx context.Context, proto, remoteAddr string) (*apitype.WhoIsResponse, error) { body, err := lc.get200(ctx, "/localapi/v0/whois?proto="+url.QueryEscape(proto)+"&addr="+url.QueryEscape(remoteAddr)) if err != nil { @@ -425,6 +474,9 @@ func (lc *Client) SetGauge(ctx context.Context, name string, value int) error { // TailDaemonLogs returns a stream the Tailscale daemon's logs as they arrive. // Close the context to stop the stream. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/logtap", nil) if err != nil { @@ -441,12 +493,18 @@ func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { } // EventBusGraph returns a graph of active publishers and subscribers in the eventbus -// as a [eventbus.DebugTopics] +// as a [eventbus.DebugTopics]. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) EventBusGraph(ctx context.Context) ([]byte, error) { return lc.get200(ctx, "/localapi/v0/debug-bus-graph") } // EventBusQueues returns a JSON snapshot of event bus queue depths per client. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) EventBusQueues(ctx context.Context) ([]byte, error) { return lc.get200(ctx, "/localapi/v0/debug-bus-queues") } @@ -455,6 +513,9 @@ func (lc *Client) EventBusQueues(ctx context.Context) ([]byte, error) { // Each pair is a valid event and a nil error, or a zero event a non-nil error. // In case of error, the iterator ends after the pair reporting the error. // Iteration stops if ctx ends. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) StreamBusEvents(ctx context.Context) iter.Seq2[eventbus.DebugEvent, error] { return func(yield func(eventbus.DebugEvent, error) bool) { req, err := http.NewRequestWithContext(ctx, "GET", @@ -523,6 +584,8 @@ type BugReportOpts struct { // // The opts type specifies options to pass to the Tailscale daemon when // generating this bug report. +// +// API maturity: this is considered a stable API. func (lc *Client) BugReportWithOpts(ctx context.Context, opts BugReportOpts) (string, error) { qparams := make(url.Values) if opts.Note != "" { @@ -568,12 +631,17 @@ func (lc *Client) BugReportWithOpts(ctx context.Context, opts BugReportOpts) (st // // This is the same as calling [Client.BugReportWithOpts] and only specifying the Note // field. +// +// API maturity: this is considered a stable API. func (lc *Client) BugReport(ctx context.Context, note string) (string, error) { return lc.BugReportWithOpts(ctx, BugReportOpts{Note: note}) } // DebugAction invokes a debug action, such as "rebind" or "restun". -// These are development tools and subject to change or removal over time. +// These are development tools. +// +// API maturity: this method is not considered a stable API and is +// subject to change or removal between releases. func (lc *Client) DebugAction(ctx context.Context, action string) error { body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil) if err != nil { @@ -584,7 +652,10 @@ func (lc *Client) DebugAction(ctx context.Context, action string) error { // DebugActionBody invokes a debug action with a body parameter, such as // "debug-force-prefer-derp". -// These are development tools and subject to change or removal over time. +// These are development tools. +// +// API maturity: this method is not considered a stable API and is +// subject to change or removal between releases. func (lc *Client) DebugActionBody(ctx context.Context, action string, rbody io.Reader) error { body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, rbody) if err != nil { @@ -594,7 +665,10 @@ func (lc *Client) DebugActionBody(ctx context.Context, action string, rbody io.R } // DebugResultJSON invokes a debug action and returns its result as something JSON-able. -// These are development tools and subject to change or removal over time. +// These are development tools. +// +// API maturity: this method is not considered a stable API and is +// subject to change or removal between releases. func (lc *Client) DebugResultJSON(ctx context.Context, action string) (any, error) { body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil) if err != nil { @@ -607,6 +681,27 @@ func (lc *Client) DebugResultJSON(ctx context.Context, action string) (any, erro return x, nil } +// GetDebugResultJSON invokes a debug action and decodes the JSON response +// into a value of type T. It avoids the marshal/unmarshal roundtrip that +// callers of [Client.DebugResultJSON] otherwise need to do to get a typed +// value. +// +// These are development tools. +// +// API maturity: this function is not considered a stable API and is +// subject to change or removal between releases. +func GetDebugResultJSON[T any](ctx context.Context, lc *Client, action string) (T, error) { + var v T + body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil) + if err != nil { + return v, fmt.Errorf("error %w: %s", err, body) + } + if err := json.Unmarshal(body, &v); err != nil { + return v, err + } + return v, nil +} + // QueryOptionalFeatures queries the optional features supported by the Tailscale daemon. func (lc *Client) QueryOptionalFeatures(ctx context.Context) (*apitype.OptionalFeatures, error) { body, err := lc.send(ctx, "POST", "/localapi/v0/debug-optional-features", 200, nil) @@ -636,6 +731,9 @@ func (lc *Client) SetDevStoreKeyValue(ctx context.Context, key, value string) er // SetComponentDebugLogging sets component's debug logging enabled for // the provided duration. If the duration is in the past, the debug logging // is disabled. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) SetComponentDebugLogging(ctx context.Context, component string, d time.Duration) error { if !buildfeatures.HasDebug { return feature.ErrUnavailable @@ -664,6 +762,8 @@ func Status(ctx context.Context) (*ipnstate.Status, error) { } // Status returns the Tailscale daemon's status. +// +// API maturity: this is considered a stable API. func (lc *Client) Status(ctx context.Context) (*ipnstate.Status, error) { return lc.status(ctx, "") } @@ -674,6 +774,8 @@ func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) { } // StatusWithoutPeers returns the Tailscale daemon's status, without the peer info. +// +// API maturity: this is considered a stable API. func (lc *Client) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) { return lc.status(ctx, "?peers=false") } @@ -778,6 +880,9 @@ func (lc *Client) PushFile(ctx context.Context, target tailcfg.StableNodeID, siz // CheckIPForwarding asks the local Tailscale daemon whether it looks like the // machine is properly configured to forward IP packets as a subnet router // or exit node. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) CheckIPForwarding(ctx context.Context) error { if !buildfeatures.HasAdvertiseRoutes { return nil @@ -801,6 +906,9 @@ func (lc *Client) CheckIPForwarding(ctx context.Context) error { // CheckUDPGROForwarding asks the local Tailscale daemon whether it looks like // the machine is optimally configured to forward UDP packets as a subnet router // or exit node. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) CheckUDPGROForwarding(ctx context.Context) error { body, err := lc.get200(ctx, "/localapi/v0/check-udp-gro-forwarding") if err != nil { @@ -850,6 +958,9 @@ func (lc *Client) CheckPrefs(ctx context.Context, p *ipn.Prefs) error { return err } +// GetPrefs returns the [ipn.Prefs] of the current Tailscale profile. +// +// API maturity: this is considered a stable API. func (lc *Client) GetPrefs(ctx context.Context) (*ipn.Prefs, error) { body, err := lc.get200(ctx, "/localapi/v0/prefs") if err != nil { @@ -867,6 +978,8 @@ func (lc *Client) GetPrefs(ctx context.Context) (*ipn.Prefs, error) { // or a policy restriction. An optional reason or justification for the request can be // provided as a context value using [apitype.RequestReasonKey]. If permitted by policy, // access may be granted, and the reason will be logged for auditing purposes. +// +// API maturity: this is considered a stable API. func (lc *Client) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) { body, err := lc.send(ctx, "PATCH", "/localapi/v0/prefs", http.StatusOK, jsonBody(mp)) if err != nil { @@ -877,6 +990,9 @@ func (lc *Client) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Pref // GetDNSOSConfig returns the system DNS configuration for the current device. // That is, it returns the DNS configuration that the system would use if Tailscale weren't being used. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) { if !buildfeatures.HasDNS { return nil, feature.ErrUnavailable @@ -910,7 +1026,26 @@ func (lc *Client) QueryDNS(ctx context.Context, name string, queryType string) ( return res.Bytes, res.Resolvers, nil } -// StartLoginInteractive starts an interactive login. +// StartLoginInteractive starts an interactive login, requesting a new +// auth URL from the control plane if a login flow is not already in +// progress. If one is, the existing auth URL is re-sent. +// +// The auth URL is not returned by this method; it is delivered +// asynchronously to IPN bus watchers (see [Client.WatchIPNBus]) as an +// [ipn.Notify] with a non-empty BrowseToURL field. StartLoginInteractive +// returns as soon as the login has been requested; it does not wait for +// the login to complete. +// +// Calling StartLoginInteractive does not itself change the node's +// desired run state, but successfully completing the login does: the +// node's WantRunning pref is set to true, so a stopped node +// ("tailscale down") starts once the login finishes. If the login is +// completed as a different user or node identity than the current +// profile's, the node switches to an existing profile matching the new +// identity if one exists, or else updates the current profile to the +// new identity. +// +// API maturity: this is considered a stable API. func (lc *Client) StartLoginInteractive(ctx context.Context) error { _, err := lc.send(ctx, "POST", "/localapi/v0/login-interactive", http.StatusNoContent, nil) return err @@ -935,6 +1070,8 @@ func (lc *Client) Logout(ctx context.Context) error { // tailscaled), a FQDN, or an IP address. // // The ctx is only used for the duration of the call, not the lifetime of the [net.Conn]. +// +// API maturity: this is considered a stable API. func (lc *Client) DialTCP(ctx context.Context, host string, port uint16) (net.Conn, error) { return lc.UserDial(ctx, "tcp", host, port) } @@ -946,6 +1083,8 @@ func (lc *Client) DialTCP(ctx context.Context, host string, port uint16) (net.Co // // The ctx is only used for the duration of the call, not the lifetime of the // [net.Conn]. +// +// API maturity: this is considered a stable API. func (lc *Client) UserDial(ctx context.Context, network, host string, port uint16) (net.Conn, error) { connCh := make(chan net.Conn, 1) trace := httptrace.ClientTrace{ @@ -972,6 +1111,19 @@ func (lc *Client) UserDial(ctx context.Context, network, host string, port uint1 if res.StatusCode != http.StatusSwitchingProtocols { body, _ := io.ReadAll(res.Body) res.Body.Close() + if res.StatusCode == http.StatusOK && res.Header.Get("Dial-Self") == "true" { + // Server told us to dial the address ourselves rather than + // proxying through the daemon. This happens for non-Tailscale + // addresses where the daemon shouldn't dial as root on the + // client's behalf. The server provides the resolved address + // to avoid a TOCTOU race with DNS re-resolution. + addr := res.Header.Get("Dial-Addr") + if addr == "" { + return nil, errors.New("server returned Dial-Self without Dial-Addr") + } + var d net.Dialer + return d.DialContext(ctx, network, addr) + } return nil, fmt.Errorf("unexpected HTTP response: %s, %s", res.Status, body) } // From here on, the underlying net.Conn is ours to use, but there @@ -997,6 +1149,10 @@ func (lc *Client) UserDial(ctx context.Context, network, host string, port uint1 // CurrentDERPMap returns the current DERPMap that is being used by the local tailscaled. // It is intended to be used with netcheck to see availability of DERPs. +// +// API maturity: this is considered a stable API, though the returned +// [tailcfg.DERPMap] type is subject to minor changes over time; its +// general shape is stable. func (lc *Client) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) { var derpMap tailcfg.DERPMap res, err := lc.send(ctx, "GET", "/localapi/v0/derpmap", 200, nil) @@ -1009,6 +1165,66 @@ func (lc *Client) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) return &derpMap, nil } +// CertDomains returns the list of domains for which the local tailscaled can +// fetch TLS certificates, equivalent to the DNS.CertDomains field of the +// current netmap. The returned list is sorted in ascending order, and is +// empty if no netmap has been received yet. +// +// API maturity: this is considered a stable API. +func (lc *Client) CertDomains(ctx context.Context) ([]string, error) { + body, err := lc.get200(ctx, "/localapi/v0/cert-domains") + if err != nil { + return nil, err + } + return decodeJSON[[]string](body) +} + +// DNSConfig returns the [tailcfg.DNSConfig] from the current netmap. +// It returns an error if no netmap has been received yet. +// It is intended for callers that need fields like ExtraRecords or CertDomains +// without pulling the rest of the netmap. +func (lc *Client) DNSConfig(ctx context.Context) (*tailcfg.DNSConfig, error) { + body, err := lc.get200(ctx, "/localapi/v0/dns-config") + if err != nil { + return nil, err + } + return decodeJSON[*tailcfg.DNSConfig](body) +} + +// PeerByID returns a peer's current full [tailcfg.Node] looked up by its +// [tailcfg.NodeID]. It returns an error if no peer with that NodeID is in the +// current netmap. +// +// It is intended for callers that observed a peer-mutation signal (e.g. +// [ipn.Notify.PeerChangedPatch] or [ipn.Notify.PeersChanged]) and want the +// latest state of the affected node without having to apply the patch +// themselves. +func (lc *Client) PeerByID(ctx context.Context, id tailcfg.NodeID) (*tailcfg.Node, error) { + body, err := lc.get200(ctx, "/localapi/v0/peer-by-id?id="+strconv.FormatInt(int64(id), 10)) + if err != nil { + return nil, err + } + return decodeJSON[*tailcfg.Node](body) +} + +// UserProfile returns the current [tailcfg.UserProfile] for the given +// [tailcfg.UserID]. It returns an error if no user with that UserID is in the +// current netmap. +// +// It is the LocalAPI fallback for IPN-bus consumers that see a UserID +// referenced by a peer Node and want to resolve it to a UserProfile. Sessions +// opted in to [ipn.NotifyPeerChanges] / [ipn.NotifyPeerPatches] also receive +// UserProfiles automatically via [ipn.Notify.UserProfiles]. +// +// API maturity: this is considered a stable API. +func (lc *Client) UserProfile(ctx context.Context, id tailcfg.UserID) (*tailcfg.UserProfile, error) { + body, err := lc.get200(ctx, "/localapi/v0/user-profile?id="+strconv.FormatInt(int64(id), 10)) + if err != nil { + return nil, err + } + return decodeJSON[*tailcfg.UserProfile](body) +} + // PingOpts contains options for the ping request. // // The zero value is valid, which means to use defaults. @@ -1045,6 +1261,8 @@ func (lc *Client) Ping(ctx context.Context, ip netip.Addr, pingtype tailcfg.Ping // DisconnectControl shuts down all connections to control, thus making control consider this node inactive. This can be // run on HA subnet router or app connector replicas before shutting them down to ensure peers get told to switch over // to another replica whilst there is still some grace period for the existing connections to terminate. +// +// API maturity: this is considered a stable API. func (lc *Client) DisconnectControl(ctx context.Context) error { _, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/disconnect-control", 200, nil, nil) if err != nil { @@ -1140,13 +1358,18 @@ func (lc *Client) ReloadConfig(ctx context.Context) (ok bool, err error) { // SwitchToEmptyProfile creates and switches to a new unnamed profile. The new // profile is not assigned an ID until it is persisted after a successful login. -// In order to login to the new profile, the user must call LoginInteractive. +// In order to login to the new profile, the user must call +// [Client.StartLoginInteractive]. +// +// API maturity: this is considered a stable API. func (lc *Client) SwitchToEmptyProfile(ctx context.Context) error { _, err := lc.send(ctx, "PUT", "/localapi/v0/profiles/", http.StatusCreated, nil) return err } // SwitchProfile switches to the given profile. +// +// API maturity: this is considered a stable API. func (lc *Client) SwitchProfile(ctx context.Context, profile ipn.ProfileID) error { _, err := lc.send(ctx, "POST", "/localapi/v0/profiles/"+url.PathEscape(string(profile)), 204, nil) return err @@ -1181,6 +1404,11 @@ func (lc *Client) QueryFeature(ctx context.Context, feature string) (*tailcfg.Qu return decodeJSON[*tailcfg.QueryFeatureResponse](body) } +// DebugDERPRegion reports diagnostic information about the DERP region with +// the given ID or code. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DebugDERPRegion(ctx context.Context, regionIDOrCode string) (*ipnstate.DebugDERPRegionReport, error) { v := url.Values{"region": {regionIDOrCode}} body, err := lc.send(ctx, "POST", "/localapi/v0/debug-derp-region?"+v.Encode(), 200, nil) @@ -1191,6 +1419,9 @@ func (lc *Client) DebugDERPRegion(ctx context.Context, regionIDOrCode string) (* } // DebugPacketFilterRules returns the packet filter rules for the current device. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.FilterRule, error) { body, err := lc.send(ctx, "POST", "/localapi/v0/debug-packet-filter-rules", 200, nil) if err != nil { @@ -1202,6 +1433,9 @@ func (lc *Client) DebugPacketFilterRules(ctx context.Context) ([]tailcfg.FilterR // DebugSetExpireIn marks the current node key to expire in d. // // This is meant primarily for debug and testing. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DebugSetExpireIn(ctx context.Context, d time.Duration) error { v := url.Values{"expiry": {fmt.Sprint(time.Now().Add(d).Unix())}} _, err := lc.send(ctx, "POST", "/localapi/v0/set-expiry-sooner?"+v.Encode(), 200, nil) @@ -1210,6 +1444,9 @@ func (lc *Client) DebugSetExpireIn(ctx context.Context, d time.Duration) error { // DebugPeerRelaySessions returns debug information about the current peer // relay sessions running through this node. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DebugPeerRelaySessions(ctx context.Context) (*status.ServerStatus, error) { body, err := lc.send(ctx, "GET", "/localapi/v0/debug-peer-relay-sessions", 200, nil) if err != nil { @@ -1222,6 +1459,9 @@ func (lc *Client) DebugPeerRelaySessions(ctx context.Context) (*status.ServerSta // // The provided context does not determine the lifetime of the // returned [io.ReadCloser]. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) StreamDebugCapture(ctx context.Context) (io.ReadCloser, error) { req, err := http.NewRequestWithContext(ctx, "POST", "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-capture", nil) if err != nil { @@ -1248,9 +1488,16 @@ func (lc *Client) StreamDebugCapture(ctx context.Context) (io.ReadCloser, error) // resources. // // A default set of ipn.Notify messages are returned but the set can be modified by mask. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*IPNBusWatcher, error) { + m, err := mask.MarshalText() + if err != nil { + return nil, err + } req, err := http.NewRequestWithContext(ctx, "GET", - "http://"+apitype.LocalAPIHost+"/localapi/v0/watch-ipn-bus?mask="+fmt.Sprint(mask), + "http://"+apitype.LocalAPIHost+"/localapi/v0/watch-ipn-bus?mask="+string(m), nil) if err != nil { return nil, err @@ -1274,6 +1521,8 @@ func (lc *Client) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*IP // CheckUpdate returns a [*tailcfg.ClientVersion] indicating whether or not an update is available // to be installed via the LocalAPI. In case the LocalAPI can't install updates, it returns a // ClientVersion that says that we are up to date. +// +// API maturity: this is considered a stable API. func (lc *Client) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion, error) { body, err := lc.get200(ctx, "/localapi/v0/update/check") if err != nil { @@ -1290,6 +1539,8 @@ func (lc *Client) CheckUpdate(ctx context.Context) (*tailcfg.ClientVersion, erro // To turn it on, there must have been a previously used exit node. // The most previously used one is reused. // This is a convenience method for GUIs. To select an actual one, update the prefs. +// +// API maturity: this is considered a stable API. func (lc *Client) SetUseExitNode(ctx context.Context, on bool) error { _, err := lc.send(ctx, "POST", "/localapi/v0/set-use-exit-node-enabled?enabled="+strconv.FormatBool(on), http.StatusOK, nil) return err @@ -1298,6 +1549,9 @@ func (lc *Client) SetUseExitNode(ctx context.Context, on bool) error { // DriveSetServerAddr instructs Taildrive to use the server at addr to access // the filesystem. This is used on platforms like Windows and MacOS to let // Taildrive know to use the file server running in the GUI app. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DriveSetServerAddr(ctx context.Context, addr string) error { _, err := lc.send(ctx, "PUT", "/localapi/v0/drive/fileserver-address", http.StatusCreated, strings.NewReader(addr)) return err @@ -1306,6 +1560,9 @@ func (lc *Client) DriveSetServerAddr(ctx context.Context, addr string) error { // DriveShareSet adds or updates the given share in the list of shares that // Taildrive will serve to remote nodes. If a share with the same name already // exists, the existing share is replaced/updated. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DriveShareSet(ctx context.Context, share *drive.Share) error { _, err := lc.send(ctx, "PUT", "/localapi/v0/drive/shares", http.StatusCreated, jsonBody(share)) return err @@ -1313,6 +1570,9 @@ func (lc *Client) DriveShareSet(ctx context.Context, share *drive.Share) error { // DriveShareRemove removes the share with the given name from the list of // shares that Taildrive will serve to remote nodes. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DriveShareRemove(ctx context.Context, name string) error { _, err := lc.send( ctx, @@ -1324,6 +1584,9 @@ func (lc *Client) DriveShareRemove(ctx context.Context, name string) error { } // DriveShareRename renames the share from old to new name. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DriveShareRename(ctx context.Context, oldName, newName string) error { _, err := lc.send( ctx, @@ -1336,6 +1599,9 @@ func (lc *Client) DriveShareRename(ctx context.Context, oldName, newName string) // DriveShareList returns the list of shares that drive is currently serving // to remote nodes. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) DriveShareList(ctx context.Context) ([]*drive.Share, error) { result, err := lc.get200(ctx, "/localapi/v0/drive/shares") if err != nil { @@ -1392,8 +1658,25 @@ func (lc *Client) SuggestExitNode(ctx context.Context) (apitype.ExitNodeSuggesti return decodeJSON[apitype.ExitNodeSuggestionResponse](body) } +// SuggestExitNodeWithProbe requests an exit node suggestion based on an immediate routecheck probe, +// waits for the probe to finish, and returns the exit node's details. +func (lc *Client) SuggestExitNodeWithProbe(ctx context.Context) (apitype.ExitNodeSuggestionResponse, error) { + if !buildfeatures.HasRouteCheck { + return apitype.ExitNodeSuggestionResponse{}, feature.ErrUnavailable + } + v := url.Values{"probe": {"true"}} + body, err := lc.send(ctx, "POST", "/localapi/v0/suggest-exit-node?"+v.Encode(), 200, nil) + if err != nil { + return apitype.ExitNodeSuggestionResponse{}, err + } + return decodeJSON[apitype.ExitNodeSuggestionResponse](body) +} + // CheckSOMarkInUse reports whether the socket mark option is in use. This will only // be true if tailscale is running on Linux and tailscaled uses SO_MARK. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) CheckSOMarkInUse(ctx context.Context) (bool, error) { body, err := lc.get200(ctx, "/localapi/v0/check-so-mark-in-use") if err != nil { @@ -1410,11 +1693,19 @@ func (lc *Client) CheckSOMarkInUse(ctx context.Context) (bool, error) { } // ShutdownTailscaled requests a graceful shutdown of tailscaled. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) ShutdownTailscaled(ctx context.Context) error { _, err := lc.send(ctx, "POST", "/localapi/v0/shutdown", 200, nil) return err } +// GetAppConnectorRouteInfo returns the current [appctype.RouteInfo] for this +// node's app connector. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) GetAppConnectorRouteInfo(ctx context.Context) (appctype.RouteInfo, error) { body, err := lc.get200(ctx, "/localapi/v0/appc-route-info") if err != nil { @@ -1422,3 +1713,13 @@ func (lc *Client) GetAppConnectorRouteInfo(ctx context.Context) (appctype.RouteI } return decodeJSON[appctype.RouteInfo](body) } + +// GetServices returns the Services visible to this node, +// including their names, IP addresses, and ports, keyed by service name. +func (lc *Client) GetServices(ctx context.Context) (map[tailcfg.ServiceName]tailcfg.ServiceDetails, error) { + body, err := lc.get200(ctx, "/localapi/v0/services") + if err != nil { + return nil, err + } + return decodeJSON[map[tailcfg.ServiceName]tailcfg.ServiceDetails](body) +} diff --git a/client/local/local_test.go b/client/local/local_test.go index a5377fbd6..58a87b224 100644 --- a/client/local/local_test.go +++ b/client/local/local_test.go @@ -61,6 +61,57 @@ func TestWhoIsPeerNotFound(t *testing.T) { } } +func TestUserDialSelf(t *testing.T) { + // Start a real TCP listener that the client should dial directly + // when the server tells it to dial-self. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + c.Write([]byte("hello")) + c.Close() + } + }() + targetAddr := ln.Addr().(*net.TCPAddr) + + // Mock LocalAPI server that returns Dial-Self response. + nw := nettest.GetNetwork(t) + ts := nettest.NewHTTPServer(nw, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Dial-Self", "true") + w.Header().Set("Dial-Addr", targetAddr.String()) + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + lc := &Client{ + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + return nw.Dial(ctx, network, ts.Listener.Addr().String()) + }, + } + + conn, err := lc.UserDial(context.Background(), "tcp", targetAddr.IP.String(), uint16(targetAddr.Port)) + if err != nil { + t.Fatalf("UserDial: %v", err) + } + defer conn.Close() + + buf := make([]byte, 5) + n, err := conn.Read(buf) + if err != nil { + t.Fatalf("Read: %v", err) + } + if got := string(buf[:n]); got != "hello" { + t.Errorf("got %q, want %q", got, "hello") + } +} + func TestDeps(t *testing.T) { deptest.DepChecker{ BadDeps: map[string]string{ diff --git a/client/local/routecheck.go b/client/local/routecheck.go new file mode 100644 index 000000000..bf64842f7 --- /dev/null +++ b/client/local/routecheck.go @@ -0,0 +1,43 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_routecheck + +package local + +import ( + "context" + "errors" + "fmt" + "net/http" + + "tailscale.com/net/routecheck" +) + +// ErrReportPending is returned by [Client.RouteCheck] and [Client.RouteCheckProbe] +// when the report is pending. +var ErrRouteCheckReportUnavailable = errors.New("report pending") + +// RouteCheckProbe performs a routecheck probe and waits for its report. +func (lc *Client) RouteCheckProbe(ctx context.Context) (*routecheck.Report, error) { + body, err := lc.send(ctx, "POST", "/localapi/v0/routecheck?probe=true", http.StatusOK, nil) + if err != nil { + if hs, ok := errors.AsType[httpStatusError](err); ok && hs.HTTPStatus == http.StatusNoContent { + return nil, ErrRouteCheckReportUnavailable + } + return nil, fmt.Errorf("error %w: %s", err, body) + } + return decodeJSON[*routecheck.Report](body) +} + +// RouteCheck requests the report compiled by the latest routecheck probe. +func (lc *Client) RouteCheck(ctx context.Context) (*routecheck.Report, error) { + body, err := lc.send(ctx, "POST", "/localapi/v0/routecheck", http.StatusOK, nil) + if err != nil { + if hs, ok := errors.AsType[httpStatusError](err); ok && hs.HTTPStatus == http.StatusNoContent { + return nil, ErrRouteCheckReportUnavailable + } + return nil, fmt.Errorf("error %w: %s", err, body) + } + return decodeJSON[*routecheck.Report](body) +} diff --git a/client/local/serve.go b/client/local/serve.go index 7f9a16a03..c6e96ec07 100644 --- a/client/local/serve.go +++ b/client/local/serve.go @@ -17,6 +17,9 @@ import ( // GetServeConfig return the current serve config. // // If the serve config is empty, it returns (nil, nil). +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) GetServeConfig(ctx context.Context) (*ipn.ServeConfig, error) { body, h, err := lc.sendWithHeaders(ctx, "GET", "/localapi/v0/serve-config", 200, nil, nil) if err != nil { diff --git a/client/local/serviceclientprefs.go b/client/local/serviceclientprefs.go new file mode 100644 index 000000000..44dc40243 --- /dev/null +++ b/client/local/serviceclientprefs.go @@ -0,0 +1,37 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_serviceclientprefs + +package local + +import ( + "context" + "net/http" + + "tailscale.com/client/tailscale/apitype" + "tailscale.com/feature/serviceclientprefs/serviceclient" +) + +// GetServiceClientPrefs returns all of the current profile's [serviceclient.Prefs]. +// +// API maturity: this method is not considered a stable API and is subject to change between releases. +func (lc *Client) GetServiceClientPrefs(ctx context.Context) (serviceclient.Prefs, error) { + body, err := lc.get200(ctx, "/localapi/v0/prefs/service-clients") + if err != nil { + return nil, err + } + return decodeJSON[serviceclient.Prefs](body) +} + +// SetServiceClientPref merges the non-empty fields from an [apitype.ServiceClientPrefRequest] into the +// saved service client prefs for the current profile and returns the full updated set. +// +// API maturity: this method is not considered a stable API and is subject to change between releases. +func (lc *Client) SetServiceClientPref(ctx context.Context, req apitype.ServiceClientPrefRequest) (serviceclient.Prefs, error) { + body, err := lc.send(ctx, "POST", "/localapi/v0/prefs/service-clients", http.StatusOK, jsonBody(req)) + if err != nil { + return nil, err + } + return decodeJSON[serviceclient.Prefs](body) +} diff --git a/client/local/syspolicy.go b/client/local/syspolicy.go index 49708fa15..4753811bb 100644 --- a/client/local/syspolicy.go +++ b/client/local/syspolicy.go @@ -13,6 +13,9 @@ import ( ) // GetEffectivePolicy returns the effective policy for the specified scope. +// +// API maturity: this method is not considered a stable API and is +// subject to change between releases. func (lc *Client) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) { scopeID, err := scope.MarshalText() if err != nil { diff --git a/client/local/tailnetlock.go b/client/local/tailnetlock.go index 5af90eb16..8445d9bd0 100644 --- a/client/local/tailnetlock.go +++ b/client/local/tailnetlock.go @@ -18,17 +18,22 @@ import ( "tailscale.com/types/tkatype" ) -// NetworkLockStatus fetches information about the tailnet key authority, if one is configured. -func (lc *Client) NetworkLockStatus(ctx context.Context) (*ipnstate.NetworkLockStatus, error) { +// TailnetLockStatus fetches information about the tailnet key authority, if one is configured. +func (lc *Client) TailnetLockStatus(ctx context.Context) (*ipnstate.TailnetLockStatus, error) { body, err := lc.send(ctx, "GET", "/localapi/v0/tka/status", 200, nil) if err != nil { return nil, fmt.Errorf("error: %w", err) } - return decodeJSON[*ipnstate.NetworkLockStatus](body) + return decodeJSON[*ipnstate.TailnetLockStatus](body) } -// NetworkLockInit initializes the tailnet key authority. -func (lc *Client) NetworkLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.NetworkLockStatus, error) { +// Deprecated: use [Client.TailnetLockStatus] instead. +func (lc *Client) NetworkLockStatus(ctx context.Context) (*ipnstate.TailnetLockStatus, error) { + return lc.TailnetLockStatus(ctx) +} + +// TailnetLockInit initializes the tailnet key authority. +func (lc *Client) TailnetLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.TailnetLockStatus, error) { var b bytes.Buffer type initRequest struct { Keys []tka.Key @@ -44,12 +49,17 @@ func (lc *Client) NetworkLockInit(ctx context.Context, keys []tka.Key, disableme if err != nil { return nil, fmt.Errorf("error: %w", err) } - return decodeJSON[*ipnstate.NetworkLockStatus](body) + return decodeJSON[*ipnstate.TailnetLockStatus](body) } -// NetworkLockWrapPreauthKey wraps a pre-auth key with information to +// Deprecated: use [Client.TailnetLockInit] instead. +func (lc *Client) NetworkLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.TailnetLockStatus, error) { + return lc.TailnetLockInit(ctx, keys, disablementValues, supportDisablement) +} + +// TailnetLockWrapPreauthKey wraps a pre-auth key with information to // enable unattended bringup in the locked tailnet. -func (lc *Client) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) { +func (lc *Client) TailnetLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) { encodedPrivate, err := tkaKey.MarshalText() if err != nil { return "", err @@ -71,8 +81,13 @@ func (lc *Client) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey stri return string(body), nil } -// NetworkLockModify adds and/or removes key(s) to the tailnet key authority. -func (lc *Client) NetworkLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error { +// Deprecated: use [Client.TailnetLockWrapPreauthKey] instead. +func (lc *Client) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) { + return lc.TailnetLockWrapPreauthKey(ctx, preauthKey, tkaKey) +} + +// TailnetLockModify adds and/or removes key(s) to the tailnet key authority. +func (lc *Client) TailnetLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error { var b bytes.Buffer type modifyRequest struct { AddKeys []tka.Key @@ -89,9 +104,14 @@ func (lc *Client) NetworkLockModify(ctx context.Context, addKeys, removeKeys []t return nil } -// NetworkLockSign signs the specified node-key and transmits that signature to the control plane. +// Deprecated: use [Client.TailnetLockModify] instead. +func (lc *Client) NetworkLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error { + return lc.TailnetLockModify(ctx, addKeys, removeKeys) +} + +// TailnetLockSign signs the specified node-key and transmits that signature to the control plane. // rotationPublic, if specified, must be an ed25519 public key. -func (lc *Client) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error { +func (lc *Client) TailnetLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error { var b bytes.Buffer type signRequest struct { NodeKey key.NodePublic @@ -108,8 +128,13 @@ func (lc *Client) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, r return nil } -// NetworkLockAffectedSigs returns all signatures signed by the specified keyID. -func (lc *Client) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) { +// Deprecated: use [Client.TailnetLockSign] instead. +func (lc *Client) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error { + return lc.TailnetLockSign(ctx, nodeKey, rotationPublic) +} + +// TailnetLockAffectedSigs returns all signatures signed by the specified keyID. +func (lc *Client) TailnetLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) { body, err := lc.send(ctx, "POST", "/localapi/v0/tka/affected-sigs", 200, bytes.NewReader(keyID)) if err != nil { return nil, fmt.Errorf("error: %w", err) @@ -117,19 +142,29 @@ func (lc *Client) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.Key return decodeJSON[[]tkatype.MarshaledSignature](body) } -// NetworkLockLog returns up to maxEntries number of changes to network-lock state. -func (lc *Client) NetworkLockLog(ctx context.Context, maxEntries int) ([]ipnstate.NetworkLockUpdate, error) { +// Deprecated: use [Client.TailnetLockAffectedSigs] instead. +func (lc *Client) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) { + return lc.TailnetLockAffectedSigs(ctx, keyID) +} + +// TailnetLockLog returns up to maxEntries number of changes to tailnet-lock state. +func (lc *Client) TailnetLockLog(ctx context.Context, maxEntries int) ([]ipnstate.TailnetLockUpdate, error) { v := url.Values{} v.Set("limit", fmt.Sprint(maxEntries)) body, err := lc.send(ctx, "GET", "/localapi/v0/tka/log?"+v.Encode(), 200, nil) if err != nil { return nil, fmt.Errorf("error %w: %s", err, body) } - return decodeJSON[[]ipnstate.NetworkLockUpdate](body) + return decodeJSON[[]ipnstate.TailnetLockUpdate](body) } -// NetworkLockForceLocalDisable forcibly shuts down network lock on this node. -func (lc *Client) NetworkLockForceLocalDisable(ctx context.Context) error { +// Deprecated: use [Client.TailnetLockLog] instead. +func (lc *Client) NetworkLockLog(ctx context.Context, maxEntries int) ([]ipnstate.TailnetLockUpdate, error) { + return lc.TailnetLockLog(ctx, maxEntries) +} + +// TailnetLockForceLocalDisable forcibly shuts down tailnet lock on this node. +func (lc *Client) TailnetLockForceLocalDisable(ctx context.Context) error { // This endpoint expects an empty JSON stanza as the payload. var b bytes.Buffer if err := json.NewEncoder(&b).Encode(struct{}{}); err != nil { @@ -142,9 +177,14 @@ func (lc *Client) NetworkLockForceLocalDisable(ctx context.Context) error { return nil } -// NetworkLockVerifySigningDeeplink verifies the network lock deeplink contained +// Deprecated: use [Client.TailnetLockForceLocalDisable] instead. +func (lc *Client) NetworkLockForceLocalDisable(ctx context.Context) error { + return lc.TailnetLockForceLocalDisable(ctx) +} + +// TailnetLockVerifySigningDeeplink verifies the tailnet lock deeplink contained // in url and returns information extracted from it. -func (lc *Client) NetworkLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) { +func (lc *Client) TailnetLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) { vr := struct { URL string }{url} @@ -157,8 +197,13 @@ func (lc *Client) NetworkLockVerifySigningDeeplink(ctx context.Context, url stri return decodeJSON[*tka.DeeplinkValidationResult](body) } -// NetworkLockGenRecoveryAUM generates an AUM for recovering from a tailnet-lock key compromise. -func (lc *Client) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) { +// Deprecated: use [Client.TailnetLockVerifySigningDeeplink] instead. +func (lc *Client) NetworkLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) { + return lc.TailnetLockVerifySigningDeeplink(ctx, url) +} + +// TailnetLockGenRecoveryAUM generates an AUM for recovering from a tailnet-lock key compromise. +func (lc *Client) TailnetLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) { vr := struct { Keys []tkatype.KeyID ForkFrom string @@ -172,8 +217,13 @@ func (lc *Client) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tk return body, nil } -// NetworkLockCosignRecoveryAUM co-signs a recovery AUM using the node's tailnet lock key. -func (lc *Client) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) { +// Deprecated: use [Client.TailnetLockGenRecoveryAUM] instead. +func (lc *Client) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) { + return lc.TailnetLockGenRecoveryAUM(ctx, removeKeys, forkFrom) +} + +// TailnetLockCosignRecoveryAUM co-signs a recovery AUM using the node's tailnet lock key. +func (lc *Client) TailnetLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) { r := bytes.NewReader(aum.Serialize()) body, err := lc.send(ctx, "POST", "/localapi/v0/tka/cosign-recovery-aum", 200, r) if err != nil { @@ -183,8 +233,13 @@ func (lc *Client) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) return body, nil } -// NetworkLockSubmitRecoveryAUM submits a recovery AUM to the control plane. -func (lc *Client) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error { +// Deprecated: use [Client.TailnetLockCosignRecoveryAUM] instead. +func (lc *Client) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) { + return lc.TailnetLockCosignRecoveryAUM(ctx, aum) +} + +// TailnetLockSubmitRecoveryAUM submits a recovery AUM to the control plane. +func (lc *Client) TailnetLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error { r := bytes.NewReader(aum.Serialize()) _, err := lc.send(ctx, "POST", "/localapi/v0/tka/submit-recovery-aum", 200, r) if err != nil { @@ -193,10 +248,20 @@ func (lc *Client) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) return nil } -// NetworkLockDisable shuts down network-lock across the tailnet. -func (lc *Client) NetworkLockDisable(ctx context.Context, secret []byte) error { +// Deprecated: use [Client.TailnetLockSubmitRecoveryAUM] instead. +func (lc *Client) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error { + return lc.TailnetLockSubmitRecoveryAUM(ctx, aum) +} + +// TailnetLockDisable shuts down tailnet-lock across the tailnet. +func (lc *Client) TailnetLockDisable(ctx context.Context, secret []byte) error { if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil { return fmt.Errorf("error: %w", err) } return nil } + +// Deprecated: use [Client.TailnetLockDisable] instead. +func (lc *Client) NetworkLockDisable(ctx context.Context, secret []byte) error { + return lc.TailnetLockDisable(ctx, secret) +} diff --git a/client/systray/logo.go b/client/systray/logo.go index a0f8bf7d0..334cd7917 100644 --- a/client/systray/logo.go +++ b/client/systray/logo.go @@ -11,6 +11,7 @@ import ( "image" "image/color" "image/png" + "log" "runtime" "sync" "time" @@ -204,12 +205,49 @@ var ( ) var ( - bg = color.NRGBA{0, 0, 0, 255} - fg = color.NRGBA{255, 255, 255, 255} - gray = color.NRGBA{255, 255, 255, 102} - red = color.NRGBA{229, 111, 74, 255} + black = color.NRGBA{0, 0, 0, 255} + white = color.NRGBA{255, 255, 255, 255} + darkGray = color.NRGBA{102, 102, 102, 255} + lightGray = color.NRGBA{153, 153, 153, 255} + red = color.NRGBA{229, 111, 74, 255} + transparent = color.NRGBA{} + + // default values to dark theme + bg = black + fg = white + gray = darkGray ) +// SetTheme sets the color theme of the systray icon. +// +// Supported themes are: +// - dark - white and gray dots over black background +// - dark:nobg - white and grey dots over transparent background +// - light - black and gray dots over white background +// - light:nobg - black and grey dots over transparent background +func SetTheme(theme string) { + switch theme { + case "dark": + bg = black + fg = white + gray = darkGray + case "dark:nobg": + bg = transparent + fg = white + gray = darkGray + case "light": + bg = white + fg = black + gray = lightGray + case "light:nobg": + bg = transparent + fg = black + gray = lightGray + default: + log.Printf("unknown theme: %q", theme) + } +} + // render returns a PNG image of the logo. func (logo tsLogo) render() *bytes.Buffer { const borderUnits = 1 diff --git a/client/systray/startup-creator.go b/client/systray/startup-creator.go index 369190012..02a018099 100644 --- a/client/systray/startup-creator.go +++ b/client/systray/startup-creator.go @@ -3,7 +3,6 @@ //go:build cgo || !darwin -// Package systray provides a minimal Tailscale systray application. package systray import ( diff --git a/client/systray/systray.go b/client/systray/systray.go index 65c1bec20..eb550fde4 100644 --- a/client/systray/systray.go +++ b/client/systray/systray.go @@ -69,6 +69,11 @@ func (menu *Menu) Run(client *local.Client) { go menu.lc.SetGauge(menu.bgCtx, "systray_running", 1) defer menu.lc.SetGauge(menu.bgCtx, "systray_running", 0) + // set initial title, which is used by the systray package as the ID of the StatusNotifierItem. + // This value will get overwritten later as the client status changes. + // This must be called before systray.Run. + systray.SetTitle("tailscale") + systray.Run(menu.onReady, menu.onExit) } @@ -172,10 +177,6 @@ See https://tailscale.com/kb/1597/linux-systray for more information.`) } setAppIcon(disconnected) - // set initial title, which is used by the systray package as the ID of the StatusNotifierItem. - // This value will get overwritten later as the client status changes. - systray.SetTitle("tailscale") - menu.rebuild() menu.mu.Lock() @@ -292,21 +293,23 @@ func (menu *Menu) rebuild() { accounts := systray.AddMenuItem(account, "") setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL) time.Sleep(newMenuDelay) - for _, profile := range menu.allProfiles { - title := profileTitle(profile) - var item *systray.MenuItem - if profile.ID == menu.curProfile.ID { - item = accounts.AddSubMenuItemCheckbox(title, "", true) - } else { - item = accounts.AddSubMenuItem(title, "") - } - setRemoteIcon(item, profile.UserProfile.ProfilePicURL) - onClick(ctx, item, func(ctx context.Context) { - select { - case <-ctx.Done(): - case menu.accountsCh <- profile.ID: + if len(menu.allProfiles) > 1 { + for _, profile := range menu.allProfiles { + title := profileTitle(profile) + var item *systray.MenuItem + if profile.ID == menu.curProfile.ID { + item = accounts.AddSubMenuItemCheckbox(title, "", true) + } else { + item = accounts.AddSubMenuItem(title, "") } - }) + setRemoteIcon(item, profile.UserProfile.ProfilePicURL) + onClick(ctx, item, func(ctx context.Context) { + select { + case <-ctx.Done(): + case menu.accountsCh <- profile.ID: + } + }) + } } } @@ -352,16 +355,27 @@ func (menu *Menu) rebuild() { // profileTitle returns the title string for a profile menu item. func profileTitle(profile ipn.LoginProfile) string { - title := profile.Name + tailnet := "" if profile.NetworkProfile.DomainName != "" { - if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { - // windows and mac don't support multi-line menu - title += " (" + profile.NetworkProfile.DisplayNameOrDefault() + ")" - } else { - title += "\n" + profile.NetworkProfile.DisplayNameOrDefault() - } + tailnet = profile.NetworkProfile.DisplayNameOrDefault() } - return title + // windows and mac don't support multi-line menu items. + multiline := runtime.GOOS != "windows" && runtime.GOOS != "darwin" + + return formatProfileTitle(profile.Name, tailnet, multiline) +} + +// formatProfileTitle builds a profile menu label from a login name and an +// optional tailnet name. The tailnet portion is omitted when it matches the +// login name, so single-user tailnets don't show the same string twice. +func formatProfileTitle(name, tailnet string, multiline bool) string { + if tailnet == "" || strings.EqualFold(name, tailnet) { + return name + } + if multiline { + return name + "\n" + tailnet + } + return name + " (" + tailnet + ")" } var ( @@ -621,11 +635,9 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { title += strings.Split(sugg.Name, ".")[0] } menu.exitNodes.AddSeparator() - rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", false) + active := recommendedIsActive(status, sugg.ID, sugg.Location.CountryCode(), sugg.Location.City()) + rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", active) setExitNodeOnClick(rm, sugg.ID) - if status.ExitNodeStatus != nil && sugg.ID == status.ExitNodeStatus.ID { - rm.Check() - } } } @@ -647,13 +659,11 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { if !ps.Online { name += " (offline)" } - sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", false) + active := status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID + sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", active) if !ps.Online { sm.Disable() } - if status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID { - sm.Check() - } setExitNodeOnClick(sm, ps.ID) } } @@ -743,6 +753,30 @@ func (mc *mvCountry) sortedCities() []*mvCity { return cities } +// recommendedIsActive reports whether the suggested exit node corresponds to +// the currently active exit node in status. +func recommendedIsActive(status *ipnstate.Status, suggID tailcfg.StableNodeID, suggCountry, suggCity string) bool { + if status == nil || status.ExitNodeStatus == nil || status.ExitNodeStatus.ID.IsZero() { + return false + } + if suggID == status.ExitNodeStatus.ID { + return true + } + if suggCountry == "" || suggCity == "" { + return false + } + for _, p := range status.Peer { + if p.ID != status.ExitNodeStatus.ID { + continue + } + if loc := p.Location; loc != nil && loc.CountryCode == suggCountry && loc.City == suggCity { + return true + } + return false + } + return false +} + // countryFlag takes a 2-character ASCII string and returns the corresponding emoji flag. // It returns the empty string on error. func countryFlag(code string) string { diff --git a/client/systray/systray_test.go b/client/systray/systray_test.go new file mode 100644 index 000000000..6bb2bfee3 --- /dev/null +++ b/client/systray/systray_test.go @@ -0,0 +1,147 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build cgo || !darwin + +package systray + +import ( + "testing" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/key" +) + +func TestProfileTitleMultiline(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + login string + tailnet string + multiline bool + want string + }{ + {"no_tailnet", "alice@example.com", "", true, "alice@example.com"}, + {"dup_exact", "example.com", "example.com", true, "example.com"}, + {"dup_casefold", "Example.com", "example.com", false, "Example.com"}, + {"distinct_multiline", "alice@example.com", "example.com", true, "alice@example.com\nexample.com"}, + {"distinct_singleline", "alice@example.com", "example.com", false, "alice@example.com (example.com)"}, + {"empty", "", "", true, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := formatProfileTitle(tt.login, tt.tailnet, tt.multiline); got != tt.want { + t.Errorf("profileTitleMultiline; got %v, want %v", got, tt.want) + } + }) + } +} + +func TestRecommendedIsActive(t *testing.T) { + t.Parallel() + + const ( + activeID = tailcfg.StableNodeID("active") + suggID = tailcfg.StableNodeID("suggestion") + ) + usNYC := &tailcfg.Location{CountryCode: "US", City: "New York"} + usCHI := &tailcfg.Location{CountryCode: "US", City: "Chicago"} + seSTO := &tailcfg.Location{CountryCode: "SE", City: "Stockholm"} + + statusWith := func(activePeer *ipnstate.PeerStatus) *ipnstate.Status { + s := &ipnstate.Status{ + ExitNodeStatus: &ipnstate.ExitNodeStatus{ID: activeID}, + } + if activePeer != nil { + s.Peer = map[key.NodePublic]*ipnstate.PeerStatus{{}: activePeer} + } + return s + } + + tests := []struct { + name string + status *ipnstate.Status + suggID tailcfg.StableNodeID + suggCountry string + suggCity string + isActive bool + }{ + { + name: "nil_status", + status: nil, + suggID: suggID, + }, + { + name: "no_exit_node", + status: &ipnstate.Status{}, + suggID: suggID, + }, + { + name: "exit_node_id_is_zero", + status: &ipnstate.Status{ExitNodeStatus: &ipnstate.ExitNodeStatus{}}, + suggID: suggID, + }, + { + name: "exact_id_match_short-circuits", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: usCHI}), + suggID: activeID, + suggCountry: "US", + suggCity: "New York", + isActive: true, + }, + { + name: "id_mismatch_but_same_city", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: usNYC}), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + isActive: true, + }, + { + name: "different_city", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: usCHI}), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + }, + { + name: "different_country", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: seSTO}), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + }, + { + name: "id_mismatch_suggestion_has_no_location", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: usNYC}), + suggID: suggID, + }, + { + name: "id_mismatch_active_peer_has_no_location", + status: statusWith(&ipnstate.PeerStatus{ID: activeID}), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + }, + { + name: "active_peer_not_in_status", + status: statusWith(nil), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + isExitNodeActive := recommendedIsActive(tt.status, tt.suggID, tt.suggCountry, tt.suggCity) + if isExitNodeActive != tt.isActive { + t.Errorf("recommendedIsActive; got %v, want %v", isExitNodeActive, tt.isActive) + } + }) + } +} diff --git a/client/tailscale/apitype/apitype.go b/client/tailscale/apitype/apitype.go index d7d1440be..12293762b 100644 --- a/client/tailscale/apitype/apitype.go +++ b/client/tailscale/apitype/apitype.go @@ -76,7 +76,7 @@ type ReloadConfigResponse struct { type ExitNodeSuggestionResponse struct { ID tailcfg.StableNodeID Name string - Location tailcfg.LocationView `json:",omitempty"` + Location tailcfg.LocationView `json:",omitzero"` } // DNSOSConfig mimics dns.OSConfig without forcing us to import the entire dns package @@ -104,3 +104,20 @@ type OptionalFeatures struct { // are not guaranteed to be present.) Features map[string]bool } + +// ServiceClientPrefRequest is the body POSTed to the LocalAPI endpoint /localapi/v0/prefs/service-clients. +// Empty values for Client, Username, and DatabaseName mean "don't change this value". +type ServiceClientPrefRequest struct { + // Key is the identifier for the service client pref. Required. Format is ":" + // where serviceName is a [tailcfg.ServiceName], e.g. "svc:my-db:5432". + Key string + + // Client is the name of the client that the user picked in the service launch. Optional. + Client string `json:",omitzero"` + + // Username is the username that the user entered in the service launch. Optional. + Username string `json:",omitzero"` + + // DatabaseName is the database name that the user entered in the service launch. Optional. + DatabaseName string `json:",omitzero"` +} diff --git a/client/tailscale/keys.go b/client/tailscale/keys.go index 6edbae034..483d12390 100644 --- a/client/tailscale/keys.go +++ b/client/tailscale/keys.go @@ -22,7 +22,7 @@ type Key struct { // KeyCapabilities are the capabilities of a Key. type KeyCapabilities struct { - Devices KeyDeviceCapabilities `json:"devices,omitempty"` + Devices KeyDeviceCapabilities `json:"devices"` } // KeyDeviceCapabilities are the device-related capabilities of a Key. diff --git a/client/web/auth.go b/client/web/auth.go index 916f24782..1281695a9 100644 --- a/client/web/auth.go +++ b/client/web/auth.go @@ -199,7 +199,8 @@ func (s *Server) controlSupportsCheckMode(ctx context.Context) bool { if err != nil { return true } - return strings.HasSuffix(controlURL.Host, ".tailscale.com") + return strings.HasSuffix(controlURL.Host, ".tailscale.com") || + controlURL.Host == "control.tailscale" // for natlab tests } // awaitUserAuth blocks until the given session auth has been completed diff --git a/client/web/src/components/exit-node-selector.tsx b/client/web/src/components/exit-node-selector.tsx index a564ebbfc..75b2551df 100644 --- a/client/web/src/components/exit-node-selector.tsx +++ b/client/web/src/components/exit-node-selector.tsx @@ -61,7 +61,7 @@ export default function ExitNodeSelector({ none, // not using exit nodes advertising, // advertising as exit node using, // using another exit node - offline, // selected exit node node is offline + offline, // selected exit node is offline ] = useMemo( () => [ selected.ID === noExitNode.ID, diff --git a/client/web/web.go b/client/web/web.go index 3e5fa4b54..95259ef1a 100644 --- a/client/web/web.go +++ b/client/web/web.go @@ -35,8 +35,10 @@ import ( "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" "tailscale.com/tailcfg" + "tailscale.com/tsweb" "tailscale.com/types/logger" "tailscale.com/types/views" + "tailscale.com/util/ctxkey" "tailscale.com/util/httpm" "tailscale.com/util/syspolicy/policyclient" "tailscale.com/version" @@ -527,45 +529,40 @@ func (s *Server) serveLoginAPI(w http.ResponseWriter, r *http.Request) { } } -type apiHandler[data any] struct { - s *Server - w http.ResponseWriter - r *http.Request - - // permissionCheck allows for defining whether a requesting peer's - // capabilities grant them access to make the given data update. - // If permissionCheck reports false, the request fails as unauthorized. - permissionCheck func(data data, peer peerCapabilities) bool -} - -// newHandler constructs a new api handler which restricts the given request -// to the specified permission check. If the permission check fails for -// the peer associated with the request, an unauthorized error is returned -// to the client. -func newHandler[data any](s *Server, w http.ResponseWriter, r *http.Request, permissionCheck func(data data, peer peerCapabilities) bool) *apiHandler[data] { - return &apiHandler[data]{ - s: s, - w: w, - r: r, - permissionCheck: permissionCheck, +// handleJSON manages decoding the request's body JSON as data and passing it +// on to the provided handler function. +func handleJSON[data any](h func(ctx context.Context, data data) error) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var body data + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err := h(r.Context(), body); err != nil { + if httpErr, ok := errors.AsType[tsweb.HTTPError](err); ok { + tsweb.WriteHTTPError(w, r, httpErr) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + w.WriteHeader(http.StatusOK) } } -// alwaysAllowed can be passed as the permissionCheck argument to newHandler -// for requests that are always allowed to complete regardless of a peer's -// capabilities. -func alwaysAllowed[data any](_ data, _ peerCapabilities) bool { return true } +var contextKeyPeer = ctxkey.New("peer-capabilities", peerCapabilities{}) -func (a *apiHandler[data]) getPeer() (peerCapabilities, error) { +func (s *Server) setPeer(r *http.Request) (*http.Request, error) { // TODO(tailscale/corp#16695,sonia): We also call StatusWithoutPeers and // WhoIs when originally checking for a session from authorizeRequest. // Would be nice if we could pipe those through to here so we don't end // up having to re-call them to grab the peer capabilities. - status, err := a.s.lc.StatusWithoutPeers(a.r.Context()) + status, err := s.lc.StatusWithoutPeers(r.Context()) if err != nil { return nil, err } - whois, err := a.s.lc.WhoIs(a.r.Context(), a.r.RemoteAddr) + whois, err := s.lc.WhoIs(r.Context(), r.RemoteAddr) if err != nil { return nil, err } @@ -573,56 +570,11 @@ func (a *apiHandler[data]) getPeer() (peerCapabilities, error) { if err != nil { return nil, err } - return peer, nil + return r.WithContext(contextKeyPeer.WithValue(r.Context(), peer)), nil } -type noBodyData any // empty type, for use from serveAPI for endpoints with empty body - -// handle runs the given handler if the source peer satisfies the -// constraints for running this request. -// -// handle is expected for use when `data` type is empty, or set to -// `noBodyData` in practice. For requests that expect JSON body data -// to be attached, use handleJSON instead. -func (a *apiHandler[data]) handle(h http.HandlerFunc) { - peer, err := a.getPeer() - if err != nil { - http.Error(a.w, err.Error(), http.StatusInternalServerError) - return - } - var body data // not used - if !a.permissionCheck(body, peer) { - http.Error(a.w, "not allowed", http.StatusUnauthorized) - return - } - h(a.w, a.r) -} - -// handleJSON manages decoding the request's body JSON and passing -// it on to the provided function if the source peer satisfies the -// constraints for running this request. -func (a *apiHandler[data]) handleJSON(h func(ctx context.Context, data data) error) { - defer a.r.Body.Close() - var body data - if err := json.NewDecoder(a.r.Body).Decode(&body); err != nil { - http.Error(a.w, err.Error(), http.StatusInternalServerError) - return - } - peer, err := a.getPeer() - if err != nil { - http.Error(a.w, err.Error(), http.StatusInternalServerError) - return - } - if !a.permissionCheck(body, peer) { - http.Error(a.w, "not allowed", http.StatusUnauthorized) - return - } - - if err := h(a.r.Context(), body); err != nil { - http.Error(a.w, err.Error(), http.StatusInternalServerError) - return - } - a.w.WriteHeader(http.StatusOK) +func (s *Server) getPeer(ctx context.Context) peerCapabilities { + return contextKeyPeer.Value(ctx) } // serveAPI serves requests for the web client api. @@ -637,67 +589,44 @@ func (s *Server) serveAPI(w http.ResponseWriter, r *http.Request) { } } + var err error + r, err = s.setPeer(r) + if err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + path := strings.TrimPrefix(r.URL.Path, "/api") switch { case path == "/data" && r.Method == httpm.GET: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.serveGetNodeData) + s.serveGetNodeData(w, r) return case path == "/exit-nodes" && r.Method == httpm.GET: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.serveGetExitNodes) + s.serveGetExitNodes(w, r) return case path == "/routes" && r.Method == httpm.POST: - peerAllowed := func(d postRoutesRequest, p peerCapabilities) bool { - if d.SetExitNode && !p.canEdit(capFeatureExitNodes) { - return false - } else if d.SetRoutes && !p.canEdit(capFeatureSubnets) { - return false - } - return true - } - newHandler[postRoutesRequest](s, w, r, peerAllowed). - handleJSON(s.servePostRoutes) + handleJSON[postRoutesRequest](s.servePostRoutes)(w, r) return case path == "/device-details-click" && r.Method == httpm.POST: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.serveDeviceDetailsClick) + s.serveDeviceDetailsClick(w, r) return case path == "/local/v0/logout" && r.Method == httpm.POST: - peerAllowed := func(_ noBodyData, peer peerCapabilities) bool { - return peer.canEdit(capFeatureAccount) - } - newHandler[noBodyData](s, w, r, peerAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return case path == "/local/v0/prefs" && r.Method == httpm.PATCH: - peerAllowed := func(data maskedPrefs, peer peerCapabilities) bool { - if data.RunSSHSet && !peer.canEdit(capFeatureSSH) { - return false - } - return true - } - newHandler[maskedPrefs](s, w, r, peerAllowed). - handleJSON(s.serveUpdatePrefs) + handleJSON[maskedPrefs](s.serveUpdatePrefs)(w, r) return case path == "/local/v0/update/check" && r.Method == httpm.GET: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return case path == "/local/v0/update/check" && r.Method == httpm.POST: - peerAllowed := func(_ noBodyData, peer peerCapabilities) bool { - return peer.canEdit(capFeatureAccount) - } - newHandler[noBodyData](s, w, r, peerAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return case path == "/local/v0/update/progress" && r.Method == httpm.POST: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return case path == "/local/v0/upload-client-metrics" && r.Method == httpm.POST: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return } http.Error(w, "invalid endpoint", http.StatusNotFound) @@ -1122,6 +1051,11 @@ type maskedPrefs struct { } func (s *Server) serveUpdatePrefs(ctx context.Context, prefs maskedPrefs) error { + peer := s.getPeer(ctx) + if prefs.RunSSHSet && !peer.canEdit(capFeatureSSH) { + return tsweb.Error(http.StatusUnauthorized, "RunSSHSet not allowed", nil) + } + _, err := s.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ RunSSHSet: prefs.RunSSHSet, Prefs: ipn.Prefs{ @@ -1140,6 +1074,17 @@ type postRoutesRequest struct { } func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) error { + if !data.SetExitNode && !data.SetRoutes { + return tsweb.Error(http.StatusBadRequest, "must specify SetExitNode or SetRoutes", nil) + } + peer := s.getPeer(ctx) + if data.SetExitNode && !peer.canEdit(capFeatureExitNodes) { + return tsweb.Error(http.StatusUnauthorized, "SetExitNode not allowed", nil) + } + if data.SetRoutes && !peer.canEdit(capFeatureSubnets) { + return tsweb.Error(http.StatusUnauthorized, "SetRoutes not allowed", nil) + } + prefs, err := s.lc.GetPrefs(ctx) if err != nil { return err @@ -1153,13 +1098,14 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er } currNonExitRoutes = append(currNonExitRoutes, r.String()) } - // Set non-edited fields to their current values. - if data.SetExitNode { - data.AdvertiseRoutes = currNonExitRoutes - } else if data.SetRoutes { + // For each group of fields not being set, preserve the current prefs. + if !data.SetExitNode { data.AdvertiseExitNode = currAdvertisingExitNode data.UseExitNode = prefs.ExitNodeID } + if !data.SetRoutes { + data.AdvertiseRoutes = currNonExitRoutes + } // Calculate routes. routesStr := strings.Join(data.AdvertiseRoutes, ",") @@ -1336,6 +1282,19 @@ func (s *Server) proxyRequestToLocalAPI(w http.ResponseWriter, r *http.Request) return } + switch path { + case "/v0/logout": + if !s.getPeer(r.Context()).canEdit(capFeatureAccount) { + http.Error(w, "not allowed", http.StatusUnauthorized) + return + } + case "/v0/update/check": + if r.Method == httpm.POST && !s.getPeer(r.Context()).canEdit(capFeatureAccount) { + http.Error(w, "not allowed", http.StatusUnauthorized) + return + } + } + localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi" + path req, err := http.NewRequestWithContext(r.Context(), r.Method, localAPIURL, r.Body) if err != nil { diff --git a/client/web/web_test.go b/client/web/web_test.go index 032cd5222..51b6a8ac5 100644 --- a/client/web/web_test.go +++ b/client/web/web_test.go @@ -191,7 +191,7 @@ func TestServeAPI(t *testing.T) { reqBody: "{\"setExitNode\":true}", tests: []requestTest{{ remoteIP: remoteIPWithNoCapabilities, - wantResponse: "not allowed", + wantResponse: "SetExitNode not allowed", wantStatus: http.StatusUnauthorized, }, { remoteIP: remoteIPWithAllCapabilities, @@ -204,7 +204,7 @@ func TestServeAPI(t *testing.T) { reqContentType: "application/json", tests: []requestTest{{ remoteIP: remoteIPWithNoCapabilities, - wantResponse: "not allowed", + wantResponse: "RunSSHSet not allowed", wantStatus: http.StatusUnauthorized, }, { remoteIP: remoteIPWithAllCapabilities, @@ -1604,3 +1604,149 @@ func TestCSRFProtect(t *testing.T) { }) } } + +func TestServePostRoutes(t *testing.T) { + existingExitNodeID := tailcfg.StableNodeID("existing-exit-node") + existingRoute := netip.MustParsePrefix("192.168.1.0/24") + + existingPrefs := &ipn.Prefs{ + ExitNodeID: existingExitNodeID, + AdvertiseRoutes: []netip.Prefix{existingRoute}, + } + + tests := []struct { + name string + data postRoutesRequest + peerCaps peerCapabilities + wantErr bool + wantEditPrefs bool // whether EditPrefs (PATCH /prefs) should be called + wantExitNodeID tailcfg.StableNodeID + wantRoutes []netip.Prefix + }{ + { + name: "empty-request", + data: postRoutesRequest{}, + peerCaps: peerCapabilities{capFeatureExitNodes: true, capFeatureSubnets: true}, + wantErr: true, + wantEditPrefs: false, + }, + { + name: "SetExitNode-only", + data: postRoutesRequest{ + SetExitNode: true, + UseExitNode: "new-exit-node", + }, + peerCaps: peerCapabilities{capFeatureExitNodes: true, capFeatureSubnets: true}, + wantEditPrefs: true, + wantExitNodeID: "new-exit-node", + wantRoutes: []netip.Prefix{existingRoute}, + }, + { + name: "SetExitNode-not-allowed", + data: postRoutesRequest{ + SetExitNode: true, + UseExitNode: "new-exit-node", + }, + peerCaps: peerCapabilities{capFeatureSubnets: true}, + wantErr: true, + }, + { + name: "SetRoutes-only", + data: postRoutesRequest{ + SetRoutes: true, + AdvertiseRoutes: []string{"10.0.0.0/8"}, + }, + peerCaps: peerCapabilities{capFeatureExitNodes: true, capFeatureSubnets: true}, + wantEditPrefs: true, + wantExitNodeID: existingExitNodeID, + wantRoutes: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, + }, + { + name: "SetRoutes-not-allowed", + data: postRoutesRequest{ + SetRoutes: true, + AdvertiseRoutes: []string{"10.0.0.0/8"}, + }, + peerCaps: peerCapabilities{capFeatureExitNodes: true}, + wantErr: true, + }, + { + name: "SetExitNode-and-SetRoutes", + data: postRoutesRequest{ + SetExitNode: true, + SetRoutes: true, + UseExitNode: "new-exit-node", + AdvertiseRoutes: []string{"10.0.0.0/8"}, + }, + peerCaps: peerCapabilities{capFeatureExitNodes: true, capFeatureSubnets: true}, + wantEditPrefs: true, + wantExitNodeID: "new-exit-node", + wantRoutes: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotPrefs *ipn.MaskedPrefs + + lal := memnet.Listen("local-tailscaled.sock:80") + defer lal.Close() + + localapi := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/localapi/v0/prefs" { + t.Errorf("unexpected localapi call to %q", r.URL.Path) + http.Error(w, "unexpected localapi call", http.StatusInternalServerError) + return + } + switch r.Method { + case httpm.GET: + writeJSON(w, existingPrefs) + case httpm.PATCH: + var mp ipn.MaskedPrefs + if err := json.NewDecoder(r.Body).Decode(&mp); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + gotPrefs = &mp + writeJSON(w, gotPrefs.Prefs) + default: + t.Errorf("unexpected method %q on /prefs", r.Method) + http.Error(w, "unexpected method", http.StatusMethodNotAllowed) + } + })} + defer localapi.Close() + go localapi.Serve(lal) + + s := &Server{ + mode: ManageServerMode, + lc: &local.Client{Dial: lal.Dial}, + } + + ctx := contextKeyPeer.WithValue(t.Context(), tt.peerCaps) + err := s.servePostRoutes(ctx, tt.data) + + if tt.wantErr { + if err == nil { + t.Error("wanted error, got nil") + } + if gotPrefs != nil { + t.Error("EditPrefs should not have been called on error") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if gotPrefs == nil { + t.Fatal("expected EditPrefs to be called") + } + if diff := cmp.Diff(tt.wantExitNodeID, gotPrefs.ExitNodeID); diff != "" { + t.Errorf("ExitNodeID mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tt.wantRoutes, gotPrefs.AdvertiseRoutes, cmp.Comparer(func(a, b netip.Prefix) bool { return a.Compare(b) == 0 })); diff != "" { + t.Errorf("AdvertiseRoutes mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/clientupdate/clientupdate.go b/clientupdate/clientupdate.go index 6d034b342..20d2e7b0a 100644 --- a/clientupdate/clientupdate.go +++ b/clientupdate/clientupdate.go @@ -11,6 +11,7 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "encoding/json" "errors" "fmt" @@ -37,6 +38,25 @@ import ( "tailscale.com/version/distro" ) +// GokrazyUpdateArgs contains arguments for updating a Gokrazy appliance from a +// GAF fetched from a URL. +type GokrazyUpdateArgs struct { + // URL is the GAF download URL. + URL string + + // AllowUnsigned permits installing a GAF without signature verification. + // It is intended for tests that serve a GAF from a fileserver that does + // not publish distsign.pub. + AllowUnsigned bool + + // Logf is optional; nil discards log messages. + Logf logger.Logf +} + +// GokrazyUpdateFromURL updates a Gokrazy appliance from a GAF fetched from a +// URL, if Gokrazy update support is linked into the binary. +var GokrazyUpdateFromURL feature.Hook[func(context.Context, GokrazyUpdateArgs) error] + const ( StableTrack = "stable" UnstableTrack = "unstable" @@ -197,6 +217,17 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) { // release cadence with Synology Package Center and use their // auto-update mechanism. return up.updateSynology, false + case distro.Gokrazy: + // Only the official Tailscale appliance image (built with the + // ts_appliance build tag, which causes hostinfo to report + // Package="tsapp") is auto-updatable. A user running a custom + // Gokrazy build that happens to include tailscaled must not be + // updated with our stock GAFs. TS_FORCE_ALLOW_TSAPP_UPDATE is an + // escape hatch for callers who know what they're doing. + if hi.Package != "tsapp" && !envknob.Bool("TS_FORCE_ALLOW_TSAPP_UPDATE") { + return nil, false + } + return up.updateGokrazy, true case distro.Debian: // includes Ubuntu return up.updateDebLike, true case distro.Arch: @@ -330,7 +361,7 @@ func (up *Updater) updateSynology() error { if err != nil { return err } - latest, err := latestPackages(up.Track) + latest, err := LatestPackages(up.Track) if err != nil { return err } @@ -864,6 +895,56 @@ func (up *Updater) updateFreeBSD() (err error) { return nil } +// updateGokrazy fetches the latest signed GAF for this gokrazy device variant +// (vm-amd64, vm-arm64, or pi-arm64) from up.PkgsAddr and applies it via the +// local gokrazy init update API. +func (up *Updater) updateGokrazy() error { + if !GokrazyUpdateFromURL.IsSet() { + return errors.New("gokrazy update support is not linked into this binary") + } + variant, err := gokrazyDeviceVariant() + if err != nil { + return err + } + latest, err := LatestPackages(up.Track) + if err != nil { + return err + } + gafName, ok := latest.GAFs[variant] + if !ok { + return fmt.Errorf("no GAF for device %q on %q track", variant, up.Track) + } + if latest.GAFsVersion == "" { + return fmt.Errorf("no GAF version on %q track", up.Track) + } + if !up.confirm(latest.GAFsVersion) { + return nil + } + gafURL := fmt.Sprintf("%s/%s/%s", strings.TrimRight(up.PkgsAddr, "/"), up.Track, gafName) + up.Logf("Updating to %s (%s)", latest.GAFsVersion, gafURL) + return GokrazyUpdateFromURL.Get()(context.Background(), GokrazyUpdateArgs{ + URL: gafURL, + Logf: up.Logf, + }) +} + +// gokrazyDeviceVariant returns the GAFs JSON key for the current gokrazy +// device, e.g. "vm-amd64", "vm-arm64", or "pi-arm64". On arm64, it reads the +// device-tree model to tell a Raspberry Pi apart from a VM. +func gokrazyDeviceVariant() (string, error) { + switch runtime.GOARCH { + case "amd64": + return "vm-amd64", nil + case "arm64": + b, _ := os.ReadFile("/sys/firmware/devicetree/base/model") + if strings.HasPrefix(strings.Trim(string(b), "\x00\r\n\t "), "Raspberry Pi") { + return "pi-arm64", nil + } + return "vm-arm64", nil + } + return "", fmt.Errorf("unsupported gokrazy GOARCH %q", runtime.GOARCH) +} + func (up *Updater) updateLinuxBinary() error { // Root is needed to overwrite binaries and restart systemd unit. if err := requireRoot(); err != nil { @@ -1224,7 +1305,7 @@ func LatestTailscaleVersion(track string) (string, error) { track = CurrentTrack } - latest, err := latestPackages(track) + latest, err := LatestPackages(track) if err != nil { return "", err } @@ -1236,8 +1317,11 @@ func LatestTailscaleVersion(track string) (string, error) { ver = latest.MacZipsVersion case "linux": ver = latest.TarballsVersion - if distro.Get() == distro.Synology { + switch distro.Get() { + case distro.Synology: ver = latest.SPKsVersion + case distro.Gokrazy: + ver = latest.GAFsVersion } } @@ -1247,7 +1331,8 @@ func LatestTailscaleVersion(track string) (string, error) { return ver, nil } -type trackPackages struct { +// TrackPackages is the JSON shape served at //?mode=json. +type TrackPackages struct { Version string Tarballs map[string]string TarballsVersion string @@ -1255,6 +1340,8 @@ type trackPackages struct { ExesVersion string MSIs map[string]string MSIsVersion string + GAFs map[string]string + GAFsVersion string MacZips map[string]string MacZipsVersion string SPKs map[string]map[string]string @@ -1263,14 +1350,16 @@ type trackPackages struct { var tailscaleHTTPEndpoint = "https://pkgs.tailscale.com" -func latestPackages(track string) (*trackPackages, error) { +// LatestPackages fetches the package manifest served at +// //?mode=json for the current runtime.GOOS. +func LatestPackages(track string) (*TrackPackages, error) { url := fmt.Sprintf("%s/%s/?mode=json&os=%s", tailscaleHTTPEndpoint, track, runtime.GOOS) res, err := http.Get(url) if err != nil { return nil, fmt.Errorf("fetching latest tailscale version: %w", err) } defer res.Body.Close() - var latest trackPackages + var latest TrackPackages if err := json.NewDecoder(res.Body).Decode(&latest); err != nil { return nil, fmt.Errorf("decoding JSON: %v: %w", res.Status, err) } diff --git a/clientupdate/clientupdate_gokrazy.go b/clientupdate/clientupdate_gokrazy.go new file mode 100644 index 000000000..1ffd5477f --- /dev/null +++ b/clientupdate/clientupdate_gokrazy.go @@ -0,0 +1,239 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package clientupdate + +import ( + "archive/zip" + "context" + "fmt" + "hash/crc32" + "io" + "net" + "net/http" + "os" + "strings" + "time" + + "tailscale.com/clientupdate/distsign" + "tailscale.com/types/logger" + "tailscale.com/util/progresstracking" +) + +const ( + gokrazyUpdateSocket = "/run/gokrazy-http.sock" + gokrazyUpdateBaseURL = "http://gokrazy-local-unixsock" +) + +// GokrazyUpdateFromURL downloads a Gokrazy archive format file from args.URL, +// installs its partitions using the local gokrazy init update API, switches to +// the new root partition, and asks gokrazy to reboot. +// +// The local gokrazy API is reached over gokrazyUpdateSocket. The +// gokrazyUpdateBaseURL host is only a net/http URL sentinel; it is not resolved +// with DNS. +func init() { + GokrazyUpdateFromURL.Set(gokrazyUpdateFromURL) +} + +func gokrazyUpdateFromURL(ctx context.Context, args GokrazyUpdateArgs) error { + logf := args.Logf + if logf == nil { + logf = logger.Discard + } + + tmp, err := os.CreateTemp("", "tailscale-gokrazy-*.gaf") + if err != nil { + return err + } + tmpName := tmp.Name() + tmp.Close() + defer os.Remove(tmpName) + + logf("downloading %s", args.URL) + if args.AllowUnsigned { + if err := downloadUnverified(ctx, logf, args.URL, tmpName); err != nil { + return err + } + } else { + if err := distsign.DownloadVerified(ctx, logf, args.URL, tmpName); err != nil { + return err + } + } + + zr, err := zip.OpenReader(tmpName) + if err != nil { + return err + } + defer zr.Close() + + logf("download complete") + + gokClient := gokrazyHTTPClient() + for _, part := range []struct { + name string + path string + }{ + {"root.img", "/update/root"}, + {"boot.img", "/update/boot"}, + {"mbr.img", "/update/mbr"}, + } { + logf("writing %s...", part.name) + if err := putGokrazyGAFMember(ctx, gokClient, zr.File, part.name, part.path); err != nil { + return err + } + logf("wrote %s", part.name) + } + if err := postGokrazy(ctx, gokClient, "/update/switch"); err != nil { + return err + } + logf("switched boot target") + if err := postGokrazy(ctx, gokClient, "/reboot?async=true&kexec_merge_cmdline=true"); err != nil { + return err + } + logf("reboot requested") + return nil +} + +// downloadUnverified saves the GAF at srcURL to dstPath without verifying +// a signature. It is used only when args.AllowUnsigned is set, for tests +// that serve the GAF from a fileserver that does not publish distsign.pub +// and for the gafpush "sftp the GAF onto the appliance and update from a +// local path" flow, which uses a "file://" URL. +func downloadUnverified(ctx context.Context, logf logger.Logf, srcURL, dstPath string) error { + if after, ok := strings.CutPrefix(srcURL, "file://"); ok { + return copyLocalFile(after, dstPath, logf) + } + req, err := http.NewRequestWithContext(ctx, "GET", srcURL, nil) + if err != nil { + return err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("download GAF: %s", res.Status) + } + f, err := os.Create(dstPath) + if err != nil { + return err + } + total := res.ContentLength + pw := progresstracking.NewWriter(io.Discard, total, time.Second, func(done int64) { + if total > 0 { + logf("downloading: %d / %d MB (%.0f%%)", done>>20, total>>20, float64(done)/float64(total)*100) + } + }) + if _, err := io.Copy(f, io.TeeReader(res.Body, pw)); err != nil { + f.Close() + return err + } + return f.Close() +} + +// copyLocalFile copies the GAF at src to dst. Used by the "file://" branch +// of downloadUnverified. The source file is left in place; callers that +// staged it (e.g. gafpush) clean up after the update completes. +func copyLocalFile(src, dst string, logf logger.Logf) error { + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + df, err := os.Create(dst) + if err != nil { + return err + } + fi, err := sf.Stat() + if err != nil { + df.Close() + return err + } + total := fi.Size() + logf("copying local GAF %s (%d MB)", src, total>>20) + pw := progresstracking.NewWriter(io.Discard, total, time.Second, func(done int64) { + if total > 0 { + logf("copying: %d / %d MB (%.0f%%)", done>>20, total>>20, float64(done)/float64(total)*100) + } + }) + if _, err := io.Copy(df, io.TeeReader(sf, pw)); err != nil { + df.Close() + return err + } + return df.Close() +} + +func gokrazyHTTPClient() *http.Client { + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", gokrazyUpdateSocket) + } + return &http.Client{ + Transport: tr, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +func putGokrazyGAFMember(ctx context.Context, hc *http.Client, files []*zip.File, name, path string) error { + var zf *zip.File + for _, f := range files { + if f.Name == name { + zf = f + break + } + } + if zf == nil { + return fmt.Errorf("GAF is missing %s", name) + } + rc, err := zf.Open() + if err != nil { + return err + } + defer rc.Close() + + h := crc32.NewIEEE() + body := io.TeeReader(rc, h) + req, err := http.NewRequestWithContext(ctx, "PUT", gokrazyUpdateBaseURL+path, body) + if err != nil { + return err + } + req.ContentLength = int64(zf.UncompressedSize64) + req.Header.Set("X-Gokrazy-Update-Hash", "crc32") + res, err := hc.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + resBody, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + if res.StatusCode != http.StatusOK { + return fmt.Errorf("PUT %s: %s: %s", path, res.Status, strings.TrimSpace(string(resBody))) + } + if got, want := strings.TrimSpace(string(resBody)), fmt.Sprintf("%08x", h.Sum32()); got != want { + return fmt.Errorf("PUT %s: gokrazy checksum = %q; want %q", path, got, want) + } + return nil +} + +func postGokrazy(ctx context.Context, hc *http.Client, path string) error { + req, err := http.NewRequestWithContext(ctx, "POST", gokrazyUpdateBaseURL+path, nil) + if err != nil { + return err + } + res, err := hc.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + return fmt.Errorf("POST %s: %s: %s", path, res.Status, strings.TrimSpace(string(body))) + } + return nil +} diff --git a/clientupdate/clientupdate_test.go b/clientupdate/clientupdate_test.go index 8095151c8..d2e7601be 100644 --- a/clientupdate/clientupdate_test.go +++ b/clientupdate/clientupdate_test.go @@ -373,7 +373,7 @@ func TestCheckOutdatedAlpineRepo(t *testing.T) { testServ := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, _ *http.Request) { - version := trackPackages{ + version := TrackPackages{ MSIsVersion: tt.latestHTTPVersion, MacZipsVersion: tt.latestHTTPVersion, TarballsVersion: tt.latestHTTPVersion, diff --git a/clientupdate/clientupdate_windows.go b/clientupdate/clientupdate_windows.go index 70a3c5091..50b77c38b 100644 --- a/clientupdate/clientupdate_windows.go +++ b/clientupdate/clientupdate_windows.go @@ -38,12 +38,12 @@ const ( updaterPrefix = "tailscale-updater" ) -func makeSelfCopy() (origPathExe, tmpPathExe string, err error) { - selfExe, err := os.Executable() +func makeCmdTailscaleCopy() (origPathExe, tmpPathExe string, err error) { + srcExe, err := findCmdTailscale() if err != nil { return "", "", err } - f, err := os.Open(selfExe) + f, err := os.Open(srcExe) if err != nil { return "", "", err } @@ -59,7 +59,25 @@ func makeSelfCopy() (origPathExe, tmpPathExe string, err error) { f2.Close() return "", "", err } - return selfExe, f2.Name(), f2.Close() + return srcExe, f2.Name(), f2.Close() +} + +// findCmdTailscale returns the path to the binary that should be copied for the update +// re-execution. The copy is re-executed with "update" as a subcommand, so it must be +// a binary that handles "update" (ie tailscale.exe, not tailscaled.exe) +func findCmdTailscale() (string, error) { + selfExe, err := os.Executable() + if err != nil { + return "", err + } + if strings.EqualFold(filepath.Base(selfExe), "tailscale.exe") { + return selfExe, nil + } + ts := filepath.Join(filepath.Dir(selfExe), "tailscale.exe") + if _, err := os.Stat(ts); err != nil { + return "", fmt.Errorf("cannot find tailscale.exe alongside %s: %w", selfExe, err) + } + return ts, nil } func markTempFileWindows(name string) error { @@ -159,14 +177,14 @@ you can run the command prompt as Administrator one of these ways: up.Logf("making tailscale.exe copy to switch to...") up.cleanupOldDownloads(filepath.Join(os.TempDir(), updaterPrefix+"-*.exe")) - _, selfCopy, err := makeSelfCopy() + _, cmdTailscaleCopy, err := makeCmdTailscaleCopy() if err != nil { return err } - defer os.Remove(selfCopy) + defer os.Remove(cmdTailscaleCopy) up.Logf("running tailscale.exe copy for final install...") - cmd := exec.Command(selfCopy, "update") + cmd := exec.Command(cmdTailscaleCopy, "update") cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winVersionEnv+"="+ver) cmd.Stdout = up.Stderr cmd.Stderr = up.Stderr diff --git a/clientupdate/distsign/distsign.go b/clientupdate/distsign/distsign.go index c804b855c..99ab193f8 100644 --- a/clientupdate/distsign/distsign.go +++ b/clientupdate/distsign/distsign.go @@ -56,9 +56,11 @@ import ( "github.com/hdevalence/ed25519consensus" "golang.org/x/crypto/blake2s" "tailscale.com/feature" + "tailscale.com/net/netutil" "tailscale.com/types/logger" "tailscale.com/util/httpm" "tailscale.com/util/must" + "tailscale.com/util/progresstracking" ) const ( @@ -329,7 +331,7 @@ func fetch(url string, limit int64) ([]byte, error) { // download writes the response body of url into a local file at dst, up to // limit bytes. On success, the returned value is a BLAKE2s hash of the file. func (c *Client) download(ctx context.Context, url, dst string, limit int64) ([]byte, int64, error) { - tr := http.DefaultTransport.(*http.Transport).Clone() + tr := netutil.NewDefaultTransport() tr.Proxy = feature.HookProxyFromEnvironment.GetOrNil() defer tr.CloseIdleConnections() hc := &http.Client{ @@ -372,7 +374,10 @@ func (c *Client) download(ctx context.Context, url, dst string, limit int64) ([] return nil, 0, err } defer of.Close() - pw := &progressWriter{total: res.ContentLength, logf: c.logf} + total := res.ContentLength + pw := progresstracking.NewWriter(io.Discard, total, 2*time.Second, func(done int64) { + c.logf("Downloaded %v/%v (%.1f%%)", done, total, float64(done)/float64(total)*100) + }) h := NewPackageHash() n, err := io.Copy(io.MultiWriter(of, h, pw), io.LimitReader(dlRes.Body, limit)) if err != nil { @@ -387,31 +392,10 @@ func (c *Client) download(ctx context.Context, url, dst string, limit int64) ([] if err := of.Close(); err != nil { return nil, n, err } - pw.print() return h.Sum(nil), h.Len(), nil } -type progressWriter struct { - done int64 - total int64 - lastPrint time.Time - logf logger.Logf -} - -func (pw *progressWriter) Write(p []byte) (n int, err error) { - pw.done += int64(len(p)) - if time.Since(pw.lastPrint) > 2*time.Second { - pw.print() - } - return len(p), nil -} - -func (pw *progressWriter) print() { - pw.lastPrint = time.Now() - pw.logf("Downloaded %v/%v (%.1f%%)", pw.done, pw.total, float64(pw.done)/float64(pw.total)*100) -} - func parsePrivateKey(data []byte, typeTag string) (ed25519.PrivateKey, error) { b, rest := pem.Decode(data) if b == nil { diff --git a/clientupdate/distsign/url.go b/clientupdate/distsign/url.go new file mode 100644 index 000000000..1571cc73a --- /dev/null +++ b/clientupdate/distsign/url.go @@ -0,0 +1,42 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package distsign + +import ( + "context" + "fmt" + "net/url" + "strings" + + "tailscale.com/types/logger" +) + +// DownloadVerified is a convenience wrapper around [Client.Download] +// for callers that have a full URL (e.g. +// https://pkgs.tailscale.com/unstable/foo.gaf) rather than a base URL +// plus path. It splits srcURL into a base ("scheme://host") and a path, +// constructs a [Client] for the base, and downloads with signature +// verification to dstPath. +func DownloadVerified(ctx context.Context, logf logger.Logf, srcURL, dstPath string) error { + if logf == nil { + logf = logger.Discard + } + u, err := url.Parse(srcURL) + if err != nil { + return fmt.Errorf("parsing URL %q: %w", srcURL, err) + } + if u.Scheme == "" || u.Host == "" { + return fmt.Errorf("URL %q is missing scheme or host", srcURL) + } + base := &url.URL{Scheme: u.Scheme, User: u.User, Host: u.Host} + path := strings.TrimPrefix(u.Path, "/") + if path == "" { + return fmt.Errorf("URL %q has no path component", srcURL) + } + c, err := NewClient(logf, base.String()) + if err != nil { + return err + } + return c.Download(ctx, path, dstPath) +} diff --git a/cmd/cigocacher/cigocacher.go b/cmd/cigocacher/cigocacher.go index 74ed08367..5da783561 100644 --- a/cmd/cigocacher/cigocacher.go +++ b/cmd/cigocacher/cigocacher.go @@ -143,19 +143,32 @@ func main() { log.Printf("Using cigocached at %s", *srvURL) } c.remote = &cachers.HTTPClient{ - BaseURL: *srvURL, - Disk: c.disk, - HTTPClient: httpClient(srvHost, *srvHostDial), - AccessToken: *token, - Verbose: *verbose, - BestEffortHTTP: true, + BaseURL: *srvURL, + Disk: c.disk, + HTTPClient: httpClient(srvHost, *srvHostDial), + AccessToken: *token, + Verbose: *verbose, + BestEffortHTTP: true, + AsyncPutTimeout: asyncPutTimeout, + AsyncPutMaxConcurrent: 10, } } var p *cacheproc.Process p = &cacheproc.Process{ Close: func() error { + if c.remote != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if !c.remote.Shutdown(ctx) { + log.Printf("cigocacher: timed out waiting for background PUTs to drain") + } + // Always surface dropped PUTs. + if timedOut, canceled := c.remote.PutsTimedOut.Load(), c.remote.PutsCanceled.Load(); timedOut+canceled > 0 { + log.Printf("cigocacher: %d background PUTs timed out, %d canceled", timedOut, canceled) + } + } if c.verbose { - log.Printf("gocacheprog: closing; %d gets (%d hits, %d misses, %d errors); %d puts (%d errors)", + log.Printf("cigocacher: closing; %d gets (%d hits, %d misses, %d errors); %d puts (%d errors)", p.Gets.Load(), p.GetHits.Load(), p.GetMisses.Load(), p.GetErrors.Load(), p.Puts.Load(), p.PutErrors.Load()) } return c.close() @@ -338,3 +351,23 @@ func fetchStats(cl *http.Client, baseURL, accessToken string) (string, error) { } return string(b), nil } + +const ( + // minPutTimeout is the floor we clamp to for small objects where the time is + // dominated by fixed overheads like connection establishment, waiting for a + // busy server to service the request etc. + minPutTimeout = 5 * time.Second + // maxPutTimeout is the ceiling we clamp to for large objects. + maxPutTimeout = 30 * time.Second + // minAverageBandwidth is the minimum average bandwidth (2MiB/s) we require + // for PUTs to complete within the timeout in its linear scaling region. + minAverageBandwidth = 2 * 1 << 20 / float64(time.Second) +) + +// asyncPutTimeout returns a size-dependent timeout for async PUTs to the remote +// gocached server. It returns 5s for size <= 10MiB, 30s for size >= 60MiB and +// scales linearly in between. +func asyncPutTimeout(size int64) time.Duration { + timeout := time.Duration(float64(size) / minAverageBandwidth) + return min(max(minPutTimeout, timeout), maxPutTimeout) +} diff --git a/cmd/cigocacher/cigocacher_test.go b/cmd/cigocacher/cigocacher_test.go new file mode 100644 index 000000000..a58514ba7 --- /dev/null +++ b/cmd/cigocacher/cigocacher_test.go @@ -0,0 +1,25 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "testing" + "time" +) + +func TestAsyncPutTimeout(t *testing.T) { + for size, expected := range map[int64]time.Duration{ + 0: 5 * time.Second, + 10: 5 * time.Second, + 10 * 1 << 20: 5 * time.Second, + 20 * 1 << 20: 10 * time.Second, + 40 * 1 << 20: 20 * time.Second, + 60 * 1 << 20: 30 * time.Second, + 10 * 1 << 30: 30 * time.Second, + } { + if actual := asyncPutTimeout(size); actual != expected { + t.Errorf("for size %d, expected %v, but got %v", size, expected, actual) + } + } +} diff --git a/cmd/cloner/cloner.go b/cmd/cloner/cloner.go index ab4a7b22f..2894c862d 100644 --- a/cmd/cloner/cloner.go +++ b/cmd/cloner/cloner.go @@ -143,25 +143,9 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) { writef("if src.%s != nil {", fname) writef("dst.%s = make([]%s, len(src.%s))", fname, n, fname) writef("for i := range dst.%s {", fname) - if ptr, isPtr := ft.Elem().(*types.Pointer); isPtr { - writef("if src.%s[i] == nil { dst.%s[i] = nil } else {", fname, fname) - if codegen.ContainsPointers(ptr.Elem()) { - if _, isIface := ptr.Elem().Underlying().(*types.Interface); isIface { - writef("\tdst.%s[i] = new((*src.%s[i]).Clone())", fname, fname) - } else { - writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname) - } - } else { - writef("\tdst.%s[i] = new(*src.%s[i])", fname, fname) - } - writef("}") - } else if ft.Elem().String() == "encoding/json.RawMessage" { - writef("\tdst.%s[i] = append(src.%s[i][:0:0], src.%s[i]...)", fname, fname, fname) - } else if _, isIface := ft.Elem().Underlying().(*types.Interface); isIface { - writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname) - } else { - writef("\tdst.%s[i] = *src.%s[i].Clone()", fname, fname) - } + writeSliceElemClone(writef, ft.Elem(), + fmt.Sprintf("src.%s[i]", fname), + fmt.Sprintf("dst.%s[i]", fname)) writef("}") writef("}") } else { @@ -185,15 +169,32 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) { writef("}") case *types.Map: elem := ft.Elem() - if sliceType, isSlice := elem.(*types.Slice); isSlice { + if sliceType, isSlice := elem.Underlying().(*types.Slice); isSlice { n := it.QualifiedName(sliceType.Elem()) writef("if dst.%s != nil {", fname) writef("\tdst.%s = map[%s]%s{}", fname, it.QualifiedName(ft.Key()), it.QualifiedName(elem)) - writef("\tfor k := range src.%s {", fname) - // use zero-length slice instead of nil to ensure - // the key is always copied. - writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname) - writef("\t}") + if codegen.ContainsPointers(sliceType.Elem()) { + writef("\tfor k, sv := range src.%s {", fname) + writef("\t\tif sv == nil {") + writef("\t\t\tdst.%s[k] = nil", fname) + writef("\t\t\tcontinue") + writef("\t\t}") + writef("\t\tdst.%s[k] = make([]%s, len(sv))", fname, n) + writef("\t\tfor i := range sv {") + innerWritef := func(format string, args ...any) { + writef("\t\t"+format, args...) + } + writeSliceElemClone(innerWritef, sliceType.Elem(), + "sv[i]", fmt.Sprintf("dst.%s[k][i]", fname)) + writef("\t\t}") + writef("\t}") + } else { + writef("\tfor k := range src.%s {", fname) + // use zero-length slice instead of nil to ensure + // the key is always copied. + writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname) + writef("\t}") + } writef("}") } else if codegen.IsViewType(elem) || !codegen.ContainsPointers(elem) { // If the map values are view types (which are @@ -242,6 +243,31 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) { buf.Write(codegen.AssertStructUnchanged(t, name, typeParams, "Clone", it)) } +// writeSliceElemClone generates code to deep-clone a single slice element +// from srcExpr to dstExpr. It handles pointer, json.RawMessage, interface, +// and named struct element types. +func writeSliceElemClone(writef func(string, ...any), elemType types.Type, srcExpr, dstExpr string) { + if ptr, isPtr := elemType.(*types.Pointer); isPtr { + writef("if %s == nil { %s = nil } else {", srcExpr, dstExpr) + if codegen.ContainsPointers(ptr.Elem()) { + if _, isIface := ptr.Elem().Underlying().(*types.Interface); isIface { + writef("\t%s = new((*%s).Clone())", dstExpr, srcExpr) + } else { + writef("\t%s = %s.Clone()", dstExpr, srcExpr) + } + } else { + writef("\t%s = new(*%s)", dstExpr, srcExpr) + } + writef("}") + } else if elemType.String() == "encoding/json.RawMessage" { + writef("%s = append(%s[:0:0], %s...)", dstExpr, srcExpr, srcExpr) + } else if _, isIface := elemType.Underlying().(*types.Interface); isIface { + writef("%s = %s.Clone()", dstExpr, srcExpr) + } else { + writef("%s = *%s.Clone()", dstExpr, srcExpr) + } +} + // hasBasicUnderlying reports true when typ.Underlying() is a slice or a map. func hasBasicUnderlying(typ types.Type) bool { switch typ.Underlying().(type) { diff --git a/cmd/cloner/cloner_test.go b/cmd/cloner/cloner_test.go index c0a946480..04670e1f5 100644 --- a/cmd/cloner/cloner_test.go +++ b/cmd/cloner/cloner_test.go @@ -7,6 +7,7 @@ import ( "reflect" "testing" + "github.com/google/go-cmp/cmp" "tailscale.com/cmd/cloner/clonerex" ) @@ -182,6 +183,46 @@ func TestNamedMapContainer(t *testing.T) { } } +func TestMapSlicePointerContainer(t *testing.T) { + num := 42 + orig := &clonerex.MapSlicePointerContainer{ + Routes: map[string][]*clonerex.SliceContainer{ + "route1": { + {Slice: []*int{&num}}, + {Slice: []*int{&num, &num}}, + }, + "route2": { + {Slice: []*int{&num}}, + }, + }, + } + + cloned := orig.Clone() + if !reflect.DeepEqual(orig, cloned) { + t.Errorf("Clone() = %v, want %v", cloned, orig) + } + + // Mutate cloned.Routes pointer values + *cloned.Routes["route1"][0].Slice[0] = 999 + if *orig.Routes["route1"][0].Slice[0] == 999 { + t.Errorf("Clone() aliased memory in Routes: original was modified") + } +} + +func TestMapSlicePointerContainerNilValue(t *testing.T) { + num := 7 + orig := &clonerex.MapSlicePointerContainer{ + Routes: map[string][]*clonerex.SliceContainer{ + "nil-value": nil, + "non-nil": {{Slice: []*int{&num}}}, + }, + } + cloned := orig.Clone() + if diff := cmp.Diff(orig.Routes, cloned.Routes); diff != "" { + t.Errorf("Clone() Routes mismatch (-orig +cloned):\n%s", diff) + } +} + func TestDeeplyNestedMap(t *testing.T) { num := 123 orig := &clonerex.DeeplyNestedMap{ @@ -242,3 +283,13 @@ func TestDeeplyNestedMap(t *testing.T) { t.Errorf("Clone() aliased FourLevels map: new nested key appeared in original") } } + +func TestMapWithNamedSliceValues(t *testing.T) { + orig := &clonerex.MapWithNamedSliceValues{ + M: map[string]clonerex.NamedSlice{"k": {"foo", "bar"}}, + } + cloned := orig.Clone() + if diff := cmp.Diff(orig, cloned); diff != "" { + t.Errorf("Clone() mismatch (-orig +cloned):\n%s", diff) + } +} diff --git a/cmd/cloner/clonerex/clonerex.go b/cmd/cloner/clonerex/clonerex.go index d17dbefc5..2f4c7a9e1 100644 --- a/cmd/cloner/clonerex/clonerex.go +++ b/cmd/cloner/clonerex/clonerex.go @@ -1,11 +1,13 @@ // Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -//go:generate go run tailscale.com/cmd/cloner -clonefunc=true -type SliceContainer,InterfaceContainer,MapWithPointers,DeeplyNestedMap,NamedMapContainer +//go:generate go run tailscale.com/cmd/cloner -clonefunc=true -type SliceContainer,InterfaceContainer,MapWithPointers,DeeplyNestedMap,NamedMapContainer,MapSlicePointerContainer,MapWithNamedSliceValues // Package clonerex is an example package for the cloner tool. package clonerex +import "maps" + type SliceContainer struct { Slice []*int } @@ -49,9 +51,7 @@ func (m NamedMap) Clone() NamedMap { return nil } m2 := make(NamedMap, len(m)) - for k, v := range m { - m2[k] = v - } + maps.Copy(m2, m) return m2 } @@ -60,8 +60,24 @@ type NamedMapContainer struct { Attrs NamedMap } +// MapSlicePointerContainer has a map whose values are slices of pointers. +// This tests that the cloner deep-clones the pointer elements in the slice, +// not just the slice itself (which would leave aliased pointers). +type MapSlicePointerContainer struct { + Routes map[string][]*SliceContainer +} + // DeeplyNestedMap tests arbitrary depth of map nesting (3+ levels) type DeeplyNestedMap struct { ThreeLevels map[string]map[string]map[string]int FourLevels map[string]map[string]map[string]map[string]*SliceContainer } + +// MapWithNamedSliceValues has a map with a named slice type for values. This +// tests that the generator treats these values like any other slice and not a +// struct. +type MapWithNamedSliceValues struct { + M map[string]NamedSlice +} + +type NamedSlice []string diff --git a/cmd/cloner/clonerex/clonerex_clone.go b/cmd/cloner/clonerex/clonerex_clone.go index 7d94688a3..dbe9f5387 100644 --- a/cmd/cloner/clonerex/clonerex_clone.go +++ b/cmd/cloner/clonerex/clonerex_clone.go @@ -176,9 +176,64 @@ var _NamedMapContainerCloneNeedsRegeneration = NamedMapContainer(struct { Attrs NamedMap }{}) +// Clone makes a deep copy of MapSlicePointerContainer. +// The result aliases no memory with the original. +func (src *MapSlicePointerContainer) Clone() *MapSlicePointerContainer { + if src == nil { + return nil + } + dst := new(MapSlicePointerContainer) + *dst = *src + if dst.Routes != nil { + dst.Routes = map[string][]*SliceContainer{} + for k, sv := range src.Routes { + if sv == nil { + dst.Routes[k] = nil + continue + } + dst.Routes[k] = make([]*SliceContainer, len(sv)) + for i := range sv { + if sv[i] == nil { + dst.Routes[k][i] = nil + } else { + dst.Routes[k][i] = sv[i].Clone() + } + } + } + } + return dst +} + +// A compilation failure here means this code must be regenerated, with the command at the top of this file. +var _MapSlicePointerContainerCloneNeedsRegeneration = MapSlicePointerContainer(struct { + Routes map[string][]*SliceContainer +}{}) + +// Clone makes a deep copy of MapWithNamedSliceValues. +// The result aliases no memory with the original. +func (src *MapWithNamedSliceValues) Clone() *MapWithNamedSliceValues { + if src == nil { + return nil + } + dst := new(MapWithNamedSliceValues) + *dst = *src + if dst.M != nil { + dst.M = map[string]NamedSlice{} + for k := range src.M { + dst.M[k] = append([]string{}, src.M[k]...) + } + } + return dst +} + +// A compilation failure here means this code must be regenerated, with the command at the top of this file. +var _MapWithNamedSliceValuesCloneNeedsRegeneration = MapWithNamedSliceValues(struct { + M map[string]NamedSlice +}{}) + // Clone duplicates src into dst and reports whether it succeeded. // To succeed, must be of types <*T, *T> or <*T, **T>, -// where T is one of SliceContainer,InterfaceContainer,MapWithPointers,DeeplyNestedMap,NamedMapContainer. +// where T is one of SliceContainer,InterfaceContainer,MapWithPointers,DeeplyNestedMap,NamedMapContainer,MapSlicePointerContainer,MapWithNamedSliceValues. func Clone(dst, src any) bool { switch src := src.(type) { case *SliceContainer: @@ -226,6 +281,24 @@ func Clone(dst, src any) bool { *dst = src.Clone() return true } + case *MapSlicePointerContainer: + switch dst := dst.(type) { + case *MapSlicePointerContainer: + *dst = *src.Clone() + return true + case **MapSlicePointerContainer: + *dst = src.Clone() + return true + } + case *MapWithNamedSliceValues: + switch dst := dst.(type) { + case *MapWithNamedSliceValues: + *dst = *src.Clone() + return true + case **MapWithNamedSliceValues: + *dst = src.Clone() + return true + } } return false } diff --git a/cmd/containerboot/egressservices.go b/cmd/containerboot/egressservices.go index e60d65c04..0b2ab024e 100644 --- a/cmd/containerboot/egressservices.go +++ b/cmd/containerboot/egressservices.go @@ -22,11 +22,12 @@ import ( "time" "github.com/fsnotify/fsnotify" + "tailscale.com/client/local" - "tailscale.com/ipn" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" + "tailscale.com/types/views" "tailscale.com/util/httpm" "tailscale.com/util/linuxfw" "tailscale.com/util/mak" @@ -54,9 +55,10 @@ type egressProxy struct { tsClient *local.Client // never nil - netmapChan chan ipn.Notify // chan to receive netmap updates on + netmapChan chan netmapState // chan to receive netmap state updates on - podIPv4 string // never empty string, currently only IPv4 is supported + podIPv4 string // empty if Pod does not have IPv4 address + podIPv6 string // empty if Pod does not have IPv6 address // tailnetFQDNs is the egress service FQDN to tailnet IP mappings that // were last used to configure firewall rules for this proxy. @@ -86,7 +88,7 @@ type httpClient interface { // - the mounted egress config has changed // - the proxy's tailnet IP addresses have changed // - tailnet IPs have changed for any backend targets specified by tailnet FQDN -func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRunOpts) error { +func (ep *egressProxy) run(ctx context.Context, nm netmapState, opts egressProxyRunOpts) error { ep.configure(opts) var tickChan <-chan time.Time var eventChan <-chan fsnotify.Event @@ -105,7 +107,7 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRu eventChan = w.Events } - if err := ep.sync(ctx, n); err != nil { + if err := ep.sync(ctx, nm); err != nil { return err } for { @@ -116,14 +118,14 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRu log.Printf("periodic sync, ensuring firewall config is up to date...") case <-eventChan: log.Printf("config file change detected, ensuring firewall config is up to date...") - case n = <-ep.netmapChan: - shouldResync := ep.shouldResync(n) + case nm = <-ep.netmapChan: + shouldResync := ep.shouldResync(nm) if !shouldResync { continue } log.Printf("netmap change detected, ensuring firewall config is up to date...") } - if err := ep.sync(ctx, n); err != nil { + if err := ep.sync(ctx, nm); err != nil { return fmt.Errorf("error syncing egress service config: %w", err) } } @@ -135,8 +137,9 @@ type egressProxyRunOpts struct { kc kubeclient.Client tsClient *local.Client stateSecret string - netmapChan chan ipn.Notify + netmapChan chan netmapState podIPv4 string + podIPv6 string tailnetAddrs []netip.Prefix } @@ -149,6 +152,7 @@ func (ep *egressProxy) configure(opts egressProxyRunOpts) { ep.stateSecret = opts.stateSecret ep.netmapChan = opts.netmapChan ep.podIPv4 = opts.podIPv4 + ep.podIPv6 = opts.podIPv6 ep.tailnetAddrs = opts.tailnetAddrs ep.client = &http.Client{} // default HTTP client sleepDuration := time.Second @@ -164,7 +168,7 @@ func (ep *egressProxy) configure(opts egressProxyRunOpts) { // any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current // firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such // as failed firewall update -func (ep *egressProxy) sync(ctx context.Context, n ipn.Notify) error { +func (ep *egressProxy) sync(ctx context.Context, nm netmapState) error { cfgs, err := ep.getConfigs() if err != nil { return fmt.Errorf("error retrieving egress service configs: %w", err) @@ -173,28 +177,27 @@ func (ep *egressProxy) sync(ctx context.Context, n ipn.Notify) error { if err != nil { return fmt.Errorf("error retrieving current egress proxy status: %w", err) } - newStatus, err := ep.syncEgressConfigs(cfgs, status, n) + newStatus, err := ep.syncEgressConfigs(cfgs, status, nm) if err != nil { return fmt.Errorf("error syncing egress service configs: %w", err) } if !servicesStatusIsEqual(newStatus, status) { - if err := ep.setStatus(ctx, newStatus, n); err != nil { + if err := ep.setStatus(ctx, newStatus, nm); err != nil { return fmt.Errorf("error setting egress proxy status: %w", err) } } return nil } -// addrsHaveChanged returns true if the provided netmap update contains tailnet address change for this proxy node. -// Netmap must not be nil. -func (ep *egressProxy) addrsHaveChanged(n ipn.Notify) bool { - return !reflect.DeepEqual(ep.tailnetAddrs, n.NetMap.SelfNode.Addresses()) +// addrsHaveChanged returns true if the provided netmap state contains tailnet address change for this proxy node. +func (ep *egressProxy) addrsHaveChanged(nm netmapState) bool { + return !views.SliceEqual(views.SliceOf(ep.tailnetAddrs), nm.self.Addresses()) } // syncEgressConfigs adds and deletes firewall rules to match the desired // configuration. It uses the provided status to determine what is currently // applied and updates the status after a successful sync. -func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *egressservices.Status, n ipn.Notify) (*egressservices.Status, error) { +func (ep *egressProxy) syncEgressConfigs(cfgs egressservices.Configs, status *egressservices.Status, nm netmapState) (*egressservices.Status, error) { if !(wantsServicesConfigured(cfgs) || hasServicesConfigured(status)) { return nil, nil } @@ -212,8 +215,8 @@ func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *e // Add new services, update rules for any that have changed. rulesPerSvcToAdd := make(map[string][]rule, 0) rulesPerSvcToDelete := make(map[string][]rule, 0) - for svcName, cfg := range *cfgs { - tailnetTargetIPs, err := ep.tailnetTargetIPsForSvc(cfg, n) + for svcName, cfg := range cfgs { + tailnetTargetIPs, err := ep.tailnetTargetIPsForSvc(cfg, nm) if err != nil { return nil, fmt.Errorf("error determining tailnet target IPs: %w", err) } @@ -228,12 +231,12 @@ func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *e if len(rulesToDelete) != 0 { mak.Set(&rulesPerSvcToDelete, svcName, rulesToDelete) } - if len(rulesToAdd) != 0 || ep.addrsHaveChanged(n) { + if len(rulesToAdd) != 0 || ep.addrsHaveChanged(nm) { // For each tailnet target, set up SNAT from the local tailnet device address of the matching // family. for _, t := range tailnetTargetIPs { var local netip.Addr - for _, pfx := range n.NetMap.SelfNode.Addresses().All() { + for _, pfx := range nm.self.Addresses().All() { if !pfx.IsSingleIP() { continue } @@ -249,6 +252,9 @@ func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *e if err := ep.nfr.EnsureSNATForDst(local, t); err != nil { return nil, fmt.Errorf("error setting up SNAT rule: %w", err) } + if err := ep.nfr.ClampMSSToPMTU(tailscaleTunInterface, t); err != nil { + return nil, fmt.Errorf("error clamping MSS to PMTU: %w", err) + } } } // Update the status. Status will be written back to the state Secret by the caller. @@ -352,7 +358,7 @@ func updatesForCfg(svcName string, cfg egressservices.Config, status *egressserv // deleteUnneccessaryServices ensure that any services found on status, but not // present in config are deleted. -func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, status *egressservices.Status) error { +func (ep *egressProxy) deleteUnnecessaryServices(cfgs egressservices.Configs, status *egressservices.Status) error { if !hasServicesConfigured(status) { return nil } @@ -367,7 +373,7 @@ func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, s } for svcName, svc := range status.Services { - if _, ok := (*cfgs)[svcName]; !ok { + if _, ok := cfgs[svcName]; !ok { log.Printf("service %s is no longer required, deleting", svcName) if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil { return fmt.Errorf("error deleting service %s: %w", svcName, err) @@ -379,7 +385,7 @@ func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, s } // getConfigs gets the mounted egress service configuration. -func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) { +func (ep *egressProxy) getConfigs() (egressservices.Configs, error) { svcsCfg := filepath.Join(ep.cfgPath, egressservices.KeyEgressServices) j, err := os.ReadFile(svcsCfg) if os.IsNotExist(err) { @@ -391,7 +397,7 @@ func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) { if len(j) == 0 || string(j) == "" { return nil, nil } - cfg := &egressservices.Configs{} + cfg := egressservices.Configs{} if err := json.Unmarshal(j, &cfg); err != nil { return nil, err } @@ -415,7 +421,7 @@ func (ep *egressProxy) getStatus(ctx context.Context) (*egressservices.Status, e if err := json.Unmarshal([]byte(raw), status); err != nil { return nil, fmt.Errorf("error unmarshalling previous config: %w", err) } - if reflect.DeepEqual(status.PodIPv4, ep.podIPv4) { + if status.PodIPv4 == ep.podIPv4 && status.PodIPv6 == ep.podIPv6 { return status, nil } return nil, nil @@ -423,12 +429,13 @@ func (ep *egressProxy) getStatus(ctx context.Context) (*egressservices.Status, e // setStatus writes egress proxy's currently configured firewall to the state // Secret and updates proxy's tailnet addresses. -func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Status, n ipn.Notify) error { +func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Status, nm netmapState) error { // Pod IP is used to determine if a stored status applies to THIS proxy Pod. if status == nil { status = &egressservices.Status{} } status.PodIPv4 = ep.podIPv4 + status.PodIPv6 = ep.podIPv6 secret, err := ep.kc.GetSecret(ctx, ep.stateSecret) if err != nil { return fmt.Errorf("error retrieving state Secret: %w", err) @@ -446,7 +453,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta if err := ep.kc.JSONPatchResource(ctx, ep.stateSecret, kubeclient.TypeSecrets, []kubeclient.JSONPatch{patch}); err != nil { return fmt.Errorf("error patching state Secret: %w", err) } - ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice() + ep.tailnetAddrs = nm.self.Addresses().AsSlice() return nil } @@ -456,7 +463,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta // FQDN, resolve the FQDN and return the resolved IPs. It checks if the // netfilter runner supports IPv6 NAT and skips any IPv6 addresses if it // doesn't. -func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.Notify) (addrs []netip.Addr, err error) { +func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, nm netmapState) (addrs []netip.Addr, err error) { if svc.TailnetTarget.IP != "" { addr, err := netip.ParseAddr(svc.TailnetTarget.IP) if err != nil { @@ -472,11 +479,11 @@ func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.N if svc.TailnetTarget.FQDN == "" { return nil, errors.New("unexpected egress service config- neither tailnet target IP nor FQDN is set") } - if n.NetMap == nil { - log.Printf("netmap is not available, unable to determine backend addresses for %s", svc.TailnetTarget.FQDN) + if !nm.self.Valid() { + log.Printf("netmap state is not available, unable to determine backend addresses for %s", svc.TailnetTarget.FQDN) return addrs, nil } - egressAddrs, err := resolveTailnetFQDN(n.NetMap, svc.TailnetTarget.FQDN) + egressAddrs, err := resolveTailnetFQDN(nm, svc.TailnetTarget.FQDN) if err != nil { log.Printf("error fetching backend addresses for %q: %v", svc.TailnetTarget.FQDN, err) return addrs, nil @@ -500,26 +507,26 @@ func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.N return addrs, nil } -// shouldResync parses netmap update and returns true if the update contains +// shouldResync parses netmap state update and returns true if the update contains // changes for which the egress proxy's firewall should be reconfigured. -func (ep *egressProxy) shouldResync(n ipn.Notify) bool { - if n.NetMap == nil { +func (ep *egressProxy) shouldResync(nm netmapState) bool { + if !nm.self.Valid() { return false } // If proxy's tailnet addresses have changed, resync. - if !reflect.DeepEqual(n.NetMap.SelfNode.Addresses().AsSlice(), ep.tailnetAddrs) { + if !views.SliceEqual(nm.self.Addresses(), views.SliceOf(ep.tailnetAddrs)) { log.Printf("node addresses have changed, trigger egress config resync") - ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice() + ep.tailnetAddrs = nm.self.Addresses().AsSlice() return true } // If the IPs for any of the egress services configured via FQDN have // changed, resync. for fqdn, ips := range ep.targetFQDNs { - for _, nn := range n.NetMap.Peers { + for nn := range nm.peers() { if equalFQDNs(nn.Name(), fqdn) { - if !reflect.DeepEqual(ips, nn.Addresses().AsSlice()) { + if !views.SliceEqual(views.SliceOf(ips), nn.Addresses()) { log.Printf("backend addresses for egress target %q have changed old IPs %v, new IPs %v trigger egress config resync", nn.Name(), ips, nn.Addresses().AsSlice()) return true } @@ -602,8 +609,8 @@ type rule struct { protocol string } -func wantsServicesConfigured(cfgs *egressservices.Configs) bool { - return cfgs != nil && len(*cfgs) != 0 +func wantsServicesConfigured(cfgs egressservices.Configs) bool { + return cfgs != nil && len(cfgs) != 0 } func hasServicesConfigured(status *egressservices.Status) bool { @@ -619,6 +626,8 @@ func servicesStatusIsEqual(st, st1 *egressservices.Status) bool { } st.PodIPv4 = "" st1.PodIPv4 = "" + st.PodIPv6 = "" + st1.PodIPv6 = "" return reflect.DeepEqual(*st, *st1) } @@ -657,37 +666,42 @@ func (ep *egressProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { // would normally be this Pod. When this Pod is being deleted, the operator should have removed it from the Service // backends and eventually kube proxy routing rules should be updated to no longer route traffic for the Service to this // Pod. -func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs *egressservices.Configs, hp int) { - if cfgs == nil || len(*cfgs) == 0 { // avoid sleeping if no services are configured +func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs egressservices.Configs, hp int) { + if cfgs == nil || len(cfgs) == 0 { // avoid sleeping if no services are configured return } log.Printf("Ensuring that cluster traffic for egress targets is no longer routed via this Pod...") var wg sync.WaitGroup - for s, cfg := range *cfgs { + for s, cfg := range cfgs { hep := cfg.HealthCheckEndpoint if hep == "" { log.Printf("Tailnet target %q does not have a cluster healthcheck specified, unable to verify if cluster traffic for the target is still routed via this Pod", s) continue } svc := s + // TODO(beckypauley): In dual-stack clusters, this is a best-effort check as we do not control which IP family is used. + // This confirms removal from routing on this node for one family only. The other IP family then relies on the longSleep below. wg.Go(func() { log.Printf("Ensuring that cluster traffic is no longer routed to %q via this Pod...", svc) + podIP, header := ep.podIPv4, kubetypes.PodIPv4Header + if podIP == "" { + podIP, header = ep.podIPv6, kubetypes.PodIPv6Header + } + if ep.podDrained(ctx, svc, hep, podIP, header, hp) { + return + } + ticker := time.NewTicker(ep.shortSleep) + defer ticker.Stop() for { - if ctx.Err() != nil { // kubelet's HTTP request timeout + select { + case <-ctx.Done(): // kubelet's HTTP request timeout log.Printf("Cluster traffic for %s did not stop being routed to this Pod.", svc) return + case <-ticker.C: + if ep.podDrained(ctx, svc, hep, podIP, header, hp) { + return + } } - found, err := lookupPodRoute(ctx, hep, ep.podIPv4, hp, ep.client) - if err != nil { - log.Printf("unable to reach endpoint %q, assuming the routing rules for this Pod have been deleted: %v", hep, err) - break - } - if !found { - log.Printf("service %q is no longer routed through this Pod", svc) - break - } - log.Printf("service %q is still routed through this Pod, waiting...", svc) - time.Sleep(ep.shortSleep) } }) } @@ -701,9 +715,9 @@ func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs *egresss // lookupPodRoute calls the healthcheck endpoint repeat times and returns true if the endpoint returns with the podIP // header at least once. -func lookupPodRoute(ctx context.Context, hep, podIP string, repeat int, client httpClient) (bool, error) { +func lookupPodRoute(ctx context.Context, hep, podIP, podIPHeader string, repeat int, client httpClient) (bool, error) { for range repeat { - f, err := lookup(ctx, hep, podIP, client) + f, err := lookup(ctx, hep, podIP, podIPHeader, client) if err != nil { return false, err } @@ -715,7 +729,7 @@ func lookupPodRoute(ctx context.Context, hep, podIP string, repeat int, client h } // lookup calls the healthcheck endpoint and returns true if the response contains the podIP header. -func lookup(ctx context.Context, hep, podIP string, client httpClient) (bool, error) { +func lookup(ctx context.Context, hep, podIP, podIPHeader string, client httpClient) (bool, error) { req, err := http.NewRequestWithContext(ctx, httpm.GET, hep, nil) if err != nil { return false, fmt.Errorf("error creating new HTTP request: %v", err) @@ -730,7 +744,7 @@ func lookup(ctx context.Context, hep, podIP string, client httpClient) (bool, er return true, nil } defer resp.Body.Close() - gotIP := resp.Header.Get(kubetypes.PodIPv4Header) + gotIP := resp.Header.Get(podIPHeader) return strings.EqualFold(podIP, gotIP), nil } @@ -759,3 +773,17 @@ func (ep *egressProxy) getHEPPings() (int, error) { } return hp, nil } + +func (ep *egressProxy) podDrained(ctx context.Context, svc, hep, podIP, header string, hp int) bool { + found, err := lookupPodRoute(ctx, hep, podIP, header, hp, ep.client) + if err != nil { + log.Printf("unable to reach endpoint %q, assuming the routing rules for this Pod have been deleted: %v", hep, err) + return true + } + if !found { + log.Printf("service %q is no longer routed through this Pod", svc) + return true + } + log.Printf("service %q is still routed through this Pod, waiting...", svc) + return false +} diff --git a/cmd/containerboot/egressservices_test.go b/cmd/containerboot/egressservices_test.go index 0d8504bda..e41cc27f6 100644 --- a/cmd/containerboot/egressservices_test.go +++ b/cmd/containerboot/egressservices_test.go @@ -15,6 +15,7 @@ import ( "strings" "sync" "testing" + "time" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubetypes" @@ -255,13 +256,13 @@ func TestWaitTillSafeToShutdown(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfgs := &egressservices.Configs{} + cfgs := egressservices.Configs{} switches := make(map[string]int) for svc, callsToSwitch := range tt.services { endpoint := fmt.Sprintf("http://%s.local", svc) if tt.healthCheckSet { - (*cfgs)[svc] = egressservices.Config{ + cfgs[svc] = egressservices.Config{ HealthCheckEndpoint: endpoint, } } @@ -269,7 +270,8 @@ func TestWaitTillSafeToShutdown(t *testing.T) { } ep := &egressProxy{ - podIPv4: podIP, + podIPv4: podIP, + shortSleep: time.Millisecond, client: &mockHTTPClient{ podIP: podIP, anotherIP: anotherIP, diff --git a/cmd/containerboot/ingressservices.go b/cmd/containerboot/ingressservices.go index d76bf86e0..d8ad01717 100644 --- a/cmd/containerboot/ingressservices.go +++ b/cmd/containerboot/ingressservices.go @@ -265,7 +265,13 @@ func ensureIngressRulesAdded(cfgs map[string]ingressservices.Config, nfr linuxfw func addDNATRuleForSvc(nfr linuxfw.NetfilterRunner, serviceName string, tsIP, clusterIP netip.Addr) error { log.Printf("adding DNAT rule for Tailscale Service %s with IP %s to Kubernetes Service IP %s", serviceName, tsIP, clusterIP) - return nfr.EnsureDNATRuleForSvc(serviceName, tsIP, clusterIP) + if err := nfr.EnsureDNATRuleForSvc(serviceName, tsIP, clusterIP); err != nil { + return err + } + if err := nfr.ClampMSSToPMTU(tailscaleTunInterface, clusterIP); err != nil { + return fmt.Errorf("error clamping MSS to PMTU: %w", err) + } + return nil } // ensureIngressRulesDeleted takes a map of Tailscale Services and rules and ensures that the firewall rules are deleted. diff --git a/cmd/containerboot/ingressservices_test.go b/cmd/containerboot/ingressservices_test.go index 46330103e..1643bb11c 100644 --- a/cmd/containerboot/ingressservices_test.go +++ b/cmd/containerboot/ingressservices_test.go @@ -7,6 +7,7 @@ package main import ( "net/netip" + "slices" "testing" "tailscale.com/kube/ingressservices" @@ -22,6 +23,7 @@ func TestSyncIngressConfigs(t *testing.T) { TailscaleServiceIP netip.Addr ClusterIP netip.Addr } + wantClampedAddrs []netip.Addr // cluster IPs that should have MSS clamping applied }{ { name: "add_new_rules_when_no_existing_config", @@ -35,6 +37,7 @@ func TestSyncIngressConfigs(t *testing.T) { }{ "svc:foo": makeWantService("100.64.0.1", "10.0.0.1"), }, + wantClampedAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, }, { name: "add_multiple_services", @@ -52,6 +55,11 @@ func TestSyncIngressConfigs(t *testing.T) { "svc:bar": makeWantService("100.64.0.2", "10.0.0.2"), "svc:baz": makeWantService("100.64.0.3", "10.0.0.3"), }, + wantClampedAddrs: []netip.Addr{ + netip.MustParseAddr("10.0.0.1"), + netip.MustParseAddr("10.0.0.2"), + netip.MustParseAddr("10.0.0.3"), + }, }, { name: "add_both_ipv4_and_ipv6_rules", @@ -65,6 +73,10 @@ func TestSyncIngressConfigs(t *testing.T) { }{ "svc:foo": makeWantService("2001:db8::1", "2001:db8::2"), }, + wantClampedAddrs: []netip.Addr{ + netip.MustParseAddr("10.0.0.1"), + netip.MustParseAddr("2001:db8::2"), + }, }, { name: "add_ipv6_only_rules", @@ -78,6 +90,7 @@ func TestSyncIngressConfigs(t *testing.T) { }{ "svc:ipv6": makeWantService("2001:db8::10", "2001:db8::20"), }, + wantClampedAddrs: []netip.Addr{netip.MustParseAddr("2001:db8::20")}, }, { name: "delete_all_rules_when_config_removed", @@ -94,6 +107,7 @@ func TestSyncIngressConfigs(t *testing.T) { TailscaleServiceIP netip.Addr ClusterIP netip.Addr }{}, + wantClampedAddrs: nil, // no rules added, no clamping }, { name: "add_remove_modify", @@ -117,6 +131,10 @@ func TestSyncIngressConfigs(t *testing.T) { "svc:foo": makeWantService("100.64.0.1", "10.0.0.2"), "svc:new": makeWantService("100.64.0.4", "10.0.0.4"), }, + wantClampedAddrs: []netip.Addr{ + netip.MustParseAddr("10.0.0.2"), + netip.MustParseAddr("10.0.0.4"), + }, }, { name: "update_with_outdated_status", @@ -152,12 +170,17 @@ func TestSyncIngressConfigs(t *testing.T) { "svc:web-ipv6": makeWantService("2001:db8::10", "2001:db8::20"), "svc:api": makeWantService("100.64.0.20", "10.0.0.20"), }, + wantClampedAddrs: []netip.Addr{ + netip.MustParseAddr("10.0.0.10"), + netip.MustParseAddr("10.0.0.20"), + netip.MustParseAddr("2001:db8::20"), + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var nfr linuxfw.NetfilterRunner = linuxfw.NewFakeNetfilterRunner() + nfr := linuxfw.NewFakeNetfilterRunner() ep := &ingressProxy{ nfr: nfr, @@ -170,8 +193,7 @@ func TestSyncIngressConfigs(t *testing.T) { t.Fatalf("syncIngressConfigs failed: %v", err) } - fake := nfr.(*linuxfw.FakeNetfilterRunner) - gotServices := fake.GetServiceState() + gotServices := nfr.GetServiceState() if len(gotServices) != len(tt.wantServices) { t.Errorf("got %d services, want %d", len(gotServices), len(tt.wantServices)) } @@ -188,6 +210,20 @@ func TestSyncIngressConfigs(t *testing.T) { t.Errorf("service %s: got ClusterIP %v, want %v", svc, got.ClusterIP, want.ClusterIP) } } + + gotClamped := nfr.GetClampedAddrs() + slices.SortFunc(gotClamped, func(a, b netip.Addr) int { return a.Compare(b) }) + slices.SortFunc(tt.wantClampedAddrs, func(a, b netip.Addr) int { return a.Compare(b) }) + if len(gotClamped) != len(tt.wantClampedAddrs) { + t.Errorf("ClampMSSToPMTU: got %v, want %v", gotClamped, tt.wantClampedAddrs) + } else { + for i := range gotClamped { + if gotClamped[i] != tt.wantClampedAddrs[i] { + t.Errorf("ClampMSSToPMTU: got %v, want %v", gotClamped, tt.wantClampedAddrs) + break + } + } + } }) } } diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index 73f5819b4..3e97710da 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -21,6 +21,7 @@ import ( "github.com/fsnotify/fsnotify" "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/kube/authkey" "tailscale.com/kube/egressservices" "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubeapi" @@ -32,7 +33,6 @@ import ( ) const fieldManager = "tailscale-container" -const kubeletMountedConfigLn = "..data" // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use // this rather than any of the upstream Kubernetes client libaries to avoid extra imports. @@ -127,6 +127,9 @@ func (kc *kubeClient) deleteAuthKey(ctx context.Context) error { // resetContainerbootState resets state from previous runs of containerboot to // ensure the operator doesn't use stale state when a Pod is first recreated. +// +// Device identity keys (device_id, device_fqdn, device_ips) are preserved so +// the operator can clean up the old device from the control plane. func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string, tailscaledConfigAuthkey string) error { existingSecret, err := kc.GetSecret(ctx, kc.stateSecret) switch { @@ -139,12 +142,7 @@ func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string s := &kubeapi.Secret{ Data: map[string][]byte{ - kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion), - - // TODO(tomhjp): Perhaps shouldn't clear device ID and use a different signal, as this could leak tailnet devices. - kubetypes.KeyDeviceID: nil, - kubetypes.KeyDeviceFQDN: nil, - kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion), kubetypes.KeyHTTPSEndpoint: nil, egressservices.KeyEgressServices: nil, ingressservices.IngressConfigKey: nil, @@ -169,47 +167,18 @@ func (kc *kubeClient) setAndWaitForAuthKeyReissue(ctx context.Context, client *l return fmt.Errorf("error disconnecting from control: %w", err) } - err = kc.setReissueAuthKey(ctx, tailscaledConfigAuthKey) + err = authkey.SetReissueAuthKey(ctx, kc.Client, kc.stateSecret, tailscaledConfigAuthKey, authkey.TailscaleContainerFieldManager) if err != nil { return fmt.Errorf("failed to set reissue_authkey in Kubernetes Secret: %w", err) } - err = kc.waitForAuthKeyReissue(ctx, cfg.TailscaledConfigFilePath, tailscaledConfigAuthKey, 10*time.Minute) - if err != nil { - return fmt.Errorf("failed to receive new auth key: %w", err) + clearFn := func(ctx context.Context) error { + return authkey.ClearReissueAuthKey(ctx, kc.Client, kc.stateSecret, authkey.TailscaleContainerFieldManager) } - return nil -} - -func (kc *kubeClient) setReissueAuthKey(ctx context.Context, authKey string) error { - s := &kubeapi.Secret{ - Data: map[string][]byte{ - kubetypes.KeyReissueAuthkey: []byte(authKey), - }, - } - - log.Printf("Requesting a new auth key from operator") - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) -} - -func (kc *kubeClient) waitForAuthKeyReissue(ctx context.Context, configPath string, oldAuthKey string, maxWait time.Duration) error { - log.Printf("Waiting for operator to provide new auth key (max wait: %v)", maxWait) - - ctx, cancel := context.WithTimeout(ctx, maxWait) - defer cancel() - - tailscaledCfgDir := filepath.Dir(configPath) - toWatch := filepath.Join(tailscaledCfgDir, kubeletMountedConfigLn) - - var ( - pollTicker <-chan time.Time - eventChan <-chan fsnotify.Event - ) - - pollInterval := 5 * time.Second - - // Try to use fsnotify for faster notification + getAuthKey := func() string { return authkey.AuthKeyFromConfig(cfg.TailscaledConfigFilePath) } + tailscaledCfgDir := filepath.Dir(cfg.TailscaledConfigFilePath) + var notify <-chan struct{} if w, err := fsnotify.NewWatcher(); err != nil { log.Printf("auth key reissue: fsnotify unavailable, using polling: %v", err) } else if err := w.Add(tailscaledCfgDir); err != nil { @@ -217,54 +186,28 @@ func (kc *kubeClient) waitForAuthKeyReissue(ctx context.Context, configPath stri log.Printf("auth key reissue: fsnotify watch failed, using polling: %v", err) } else { defer w.Close() + ch := make(chan struct{}, 1) + toWatch := filepath.Join(tailscaledCfgDir, "..data") + go func() { + for ev := range w.Events { + if ev.Name == toWatch { + select { + case ch <- struct{}{}: + default: + } + } + } + }() + notify = ch log.Printf("auth key reissue: watching for config changes via fsnotify") - eventChan = w.Events } - // still keep polling if using fsnotify, for logging and in case fsnotify fails - pt := time.NewTicker(pollInterval) - defer pt.Stop() - pollTicker = pt.C - - start := time.Now() - - for { - select { - case <-ctx.Done(): - return fmt.Errorf("timeout waiting for auth key reissue after %v", maxWait) - case <-pollTicker: // Waits for polling tick, continues when received - case event := <-eventChan: - if event.Name != toWatch { - continue - } - } - - newAuthKey := authkeyFromTailscaledConfig(configPath) - if newAuthKey != "" && newAuthKey != oldAuthKey { - log.Printf("New auth key received from operator after %v", time.Since(start).Round(time.Second)) - - if err := kc.clearReissueAuthKeyRequest(ctx); err != nil { - log.Printf("Warning: failed to clear reissue request: %v", err) - } - - return nil - } - - if eventChan == nil && pollTicker != nil { - log.Printf("Waiting for new auth key from operator (%v elapsed)", time.Since(start).Round(time.Second)) - } + err = authkey.WaitForAuthKeyReissue(ctx, tailscaledConfigAuthKey, 10*time.Minute, getAuthKey, clearFn, notify) + if err != nil { + return fmt.Errorf("failed to receive new auth key: %w", err) } -} -// clearReissueAuthKeyRequest removes the reissue_authkey marker from the Secret -// to signal to the operator that we've successfully received the new key. -func (kc *kubeClient) clearReissueAuthKeyRequest(ctx context.Context) error { - s := &kubeapi.Secret{ - Data: map[string][]byte{ - kubetypes.KeyReissueAuthkey: nil, - }, - } - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) + return nil } // waitForConsistentState waits for tailscaled to finish writing state if it diff --git a/cmd/containerboot/kube_test.go b/cmd/containerboot/kube_test.go index b2e89a36c..fec0b74f7 100644 --- a/cmd/containerboot/kube_test.go +++ b/cmd/containerboot/kube_test.go @@ -257,12 +257,8 @@ func TestResetContainerbootState(t *testing.T) { authkey: "new-authkey", initial: map[string][]byte{}, expected: map[string][]byte{ - kubetypes.KeyCapVer: capver, - kubetypes.KeyPodUID: []byte("1234"), - // Cleared keys. - kubetypes.KeyDeviceID: nil, - kubetypes.KeyDeviceFQDN: nil, - kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: []byte("1234"), kubetypes.KeyHTTPSEndpoint: nil, egressservices.KeyEgressServices: nil, ingressservices.IngressConfigKey: nil, @@ -271,11 +267,7 @@ func TestResetContainerbootState(t *testing.T) { "empty_initial_no_pod_uid": { initial: map[string][]byte{}, expected: map[string][]byte{ - kubetypes.KeyCapVer: capver, - // Cleared keys. - kubetypes.KeyDeviceID: nil, - kubetypes.KeyDeviceFQDN: nil, - kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyCapVer: capver, kubetypes.KeyHTTPSEndpoint: nil, egressservices.KeyEgressServices: nil, ingressservices.IngressConfigKey: nil, @@ -303,9 +295,6 @@ func TestResetContainerbootState(t *testing.T) { kubetypes.KeyCapVer: capver, kubetypes.KeyPodUID: []byte("1234"), // Cleared keys. - kubetypes.KeyDeviceID: nil, - kubetypes.KeyDeviceFQDN: nil, - kubetypes.KeyDeviceIPs: nil, kubetypes.KeyHTTPSEndpoint: nil, egressservices.KeyEgressServices: nil, ingressservices.IngressConfigKey: nil, @@ -321,9 +310,6 @@ func TestResetContainerbootState(t *testing.T) { kubetypes.KeyCapVer: capver, kubetypes.KeyReissueAuthkey: nil, // Cleared keys. - kubetypes.KeyDeviceID: nil, - kubetypes.KeyDeviceFQDN: nil, - kubetypes.KeyDeviceIPs: nil, kubetypes.KeyHTTPSEndpoint: nil, egressservices.KeyEgressServices: nil, ingressservices.IngressConfigKey: nil, @@ -338,9 +324,6 @@ func TestResetContainerbootState(t *testing.T) { kubetypes.KeyCapVer: capver, // reissue_authkey not cleared. // Cleared keys. - kubetypes.KeyDeviceID: nil, - kubetypes.KeyDeviceFQDN: nil, - kubetypes.KeyDeviceIPs: nil, kubetypes.KeyHTTPSEndpoint: nil, egressservices.KeyEgressServices: nil, ingressservices.IngressConfigKey: nil, @@ -355,9 +338,6 @@ func TestResetContainerbootState(t *testing.T) { kubetypes.KeyCapVer: capver, // reissue_authkey not cleared. // Cleared keys. - kubetypes.KeyDeviceID: nil, - kubetypes.KeyDeviceFQDN: nil, - kubetypes.KeyDeviceIPs: nil, kubetypes.KeyHTTPSEndpoint: nil, egressservices.KeyEgressServices: nil, ingressservices.IngressConfigKey: nil, diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index e80192a31..04651cd48 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -120,6 +120,7 @@ import ( "errors" "fmt" "io/fs" + "iter" "log" "math" "net" @@ -135,12 +136,15 @@ import ( "syscall" "time" + "github.com/benbjohnson/immutable" "golang.org/x/sys/unix" + "tailscale.com/client/local" "tailscale.com/health" "tailscale.com/ipn" - "tailscale.com/ipn/conffile" + "tailscale.com/ipn/ipnstate" kubeutils "tailscale.com/k8s-operator" + "tailscale.com/kube/authkey" healthz "tailscale.com/kube/health" "tailscale.com/kube/kubetypes" klc "tailscale.com/kube/localclient" @@ -148,21 +152,170 @@ import ( "tailscale.com/kube/services" "tailscale.com/tailcfg" "tailscale.com/types/logger" - "tailscale.com/types/netmap" + "tailscale.com/types/views" "tailscale.com/util/deephash" + "tailscale.com/util/def" "tailscale.com/util/dnsname" "tailscale.com/util/linuxfw" ) func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) { - if defaultBool("TS_TEST_FAKE_NETFILTER", false) { + if def.Bool(os.Getenv("TS_TEST_FAKE_NETFILTER"), false) { return linuxfw.NewFakeIPTablesRunner(), nil } return linuxfw.New(logf, "") } func getAutoAdvertiseBool() bool { - return defaultBool("TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", true) + return def.Bool(os.Getenv("TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT"), true) +} + +const containerbootWatchMask = ipn.NotifyInitialStatus | + ipn.NotifyPeerChanges | + ipn.NotifyNoNetMap + +func notifyState(n ipn.Notify) (_ ipn.State, ok bool) { + if n.State != nil { + return *n.State, true + } + if n.InitialStatus != nil && n.InitialStatus.BackendState != "" { + if state, ok := ipn.StateFromString(n.InitialStatus.BackendState); ok { + return state, true + } + } + return ipn.NoState, false +} + +var netmapStatePeerIDHasher = immutable.NewHasher(tailcfg.NodeID(0)) + +type netmapState struct { + self tailcfg.NodeView + peersByID *immutable.Map[tailcfg.NodeID, tailcfg.NodeView] + peersByName *immutable.Map[string, tailcfg.NodeView] // keyed by tailcfg.Node.Name when NodeID is unavailable + certDomains views.Slice[string] + dnsExtraRecords views.Slice[tailcfg.DNSRecord] +} + +func (s netmapState) updateFromNotify(n ipn.Notify) netmapState { + if n.InitialStatus != nil { + s = s.updateFromStatus(n.InitialStatus) + } + if n.SelfChange != nil { + s.self = n.SelfChange.View() + } + for _, p := range n.PeersChanged { + s = s.upsertPeer(p.View()) + } + for _, id := range n.PeersRemoved { + if s.peersByID != nil { + s.peersByID = s.peersByID.Delete(id) + } + } + return s +} + +// processNotify updates the netmap state from an IPN bus Notify. On +// SelfChange it also refetches DNS via the LocalAPI dns-config +// endpoint; the bus carries no DNS delta. +func (s netmapState) processNotify(ctx context.Context, client *local.Client, n ipn.Notify) netmapState { + s = s.updateFromNotify(n) + if n.SelfChange != nil { + dns, err := client.DNSConfig(ctx) + if err != nil { + log.Printf("error refreshing DNS config from tailscaled: %v", err) + } else if dns != nil { + s.dnsExtraRecords = views.SliceOf(dns.ExtraRecords) + s.certDomains = views.SliceOf(dns.CertDomains) + } + } + return s +} + +func (s netmapState) updateFromStatus(st *ipnstate.Status) netmapState { + s.certDomains = views.SliceOf(st.CertDomains) + s.dnsExtraRecords = views.SliceOf(st.ExtraRecords) + if st.Self != nil { + s.self = nodeFromPeerStatus(st.Self).View() + } + if len(st.Peer) != 0 { + s.peersByID = nil + s.peersByName = nil + for _, ps := range st.Peer { + s = s.upsertPeer(nodeFromPeerStatus(ps).View()) + } + } + return s +} + +func (s netmapState) upsertPeer(n tailcfg.NodeView) netmapState { + if !n.Valid() { + return s + } + if s.peersByID == nil { + s.peersByID = immutable.NewMap[tailcfg.NodeID, tailcfg.NodeView](netmapStatePeerIDHasher) + } + if s.peersByName == nil { + s.peersByName = immutable.NewMap[string, tailcfg.NodeView](nil) + } + if n.ID() != 0 { + s.peersByID = s.peersByID.Set(n.ID(), n) + if name := n.Name(); name != "" { + s.peersByName = s.peersByName.Delete(name) + } + return s + } + if n.Name() != "" { + s.peersByName = s.peersByName.Set(n.Name(), n) + } + return s +} + +func nodeFromPeerStatus(ps *ipnstate.PeerStatus) *tailcfg.Node { + if ps == nil { + return nil + } + n := &tailcfg.Node{ + ID: ps.NodeID, + StableID: ps.ID, + Name: ps.DNSName, + Key: ps.PublicKey, + } + for _, ip := range ps.TailscaleIPs { + n.Addresses = append(n.Addresses, netip.PrefixFrom(ip, ip.BitLen())) + } + if ps.AllowedIPs != nil { + n.AllowedIPs = ps.AllowedIPs.AsSlice() + } + return n +} + +func (s netmapState) peers() iter.Seq[tailcfg.NodeView] { + return func(yield func(tailcfg.NodeView) bool) { + if s.peersByID != nil { + it := s.peersByID.Iterator() + for { + _, p, ok := it.Next() + if !ok { + break + } + if !yield(p) { + return + } + } + } + if s.peersByName != nil { + it := s.peersByName.Iterator() + for { + _, p, ok := it.Next() + if !ok { + break + } + if !yield(p) { + return + } + } + } + } } func main() { @@ -209,7 +362,7 @@ func run() error { var tailscaledConfigAuthkey string if isOneStepConfig(cfg) { - tailscaledConfigAuthkey = authkeyFromTailscaledConfig(cfg.TailscaledConfigFilePath) + tailscaledConfigAuthkey = authkey.AuthKeyFromConfig(cfg.TailscaledConfigFilePath) } var kc *kubeClient @@ -271,7 +424,7 @@ func run() error { mux := http.NewServeMux() log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort) - healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf) + healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, cfg.PodIPv6, log.Printf) close := runHTTPServer(mux, cfg.HealthCheckAddrPort) defer close() @@ -287,7 +440,7 @@ func run() error { if cfg.localHealthEnabled() { log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort) - healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf) + healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, cfg.PodIPv6, log.Printf) } if cfg.egressSvcsTerminateEPEnabled() { @@ -305,7 +458,7 @@ func run() error { } } - w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState|ipn.NotifyInitialHealthState|ipn.NotifyRateLimit) + w, err := client.WatchIPNBus(bootCtx, containerbootWatchMask|ipn.NotifyInitialPrefs|ipn.NotifyInitialHealthState) if err != nil { return fmt.Errorf("failed to watch tailscaled for updates: %w", err) } @@ -345,7 +498,7 @@ func run() error { if err := tailscaleUp(bootCtx, cfg); err != nil { return fmt.Errorf("failed to auth tailscale: %w", err) } - w, err = client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState|ipn.NotifyRateLimit) + w, err = client.WatchIPNBus(bootCtx, containerbootWatchMask) if err != nil { return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) } @@ -365,8 +518,8 @@ authLoop: return fmt.Errorf("failed to read from tailscaled: %w", err) } - if n.State != nil { - switch *n.State { + if state, ok := notifyState(n); ok { + switch state { case ipn.NeedsLogin: if isOneStepConfig(cfg) { // This could happen if this is the first time tailscaled was run for this @@ -374,7 +527,7 @@ authLoop: if hasKubeStateStore(cfg) { log.Printf("Auth key missing or invalid (NeedsLogin state), disconnecting from control and requesting new key from operator") - err := kc.setAndWaitForAuthKeyReissue(bootCtx, client, cfg, tailscaledConfigAuthkey) + err := kc.setAndWaitForAuthKeyReissue(ctx, client, cfg, tailscaledConfigAuthkey) if err != nil { return fmt.Errorf("failed to get a reissued authkey: %w", err) } @@ -402,7 +555,7 @@ authLoop: // deadline to continue monitoring for changes. break authLoop default: - log.Printf("tailscaled in state %q, waiting", *n.State) + log.Printf("tailscaled in state %q, waiting", state) } } @@ -414,7 +567,7 @@ authLoop: if isOneStepConfig(cfg) && hasKubeStateStore(cfg) { log.Printf("Auth key failed to authenticate (may be expired or single-use), disconnecting from control and requesting new key from operator") - err := kc.setAndWaitForAuthKeyReissue(bootCtx, client, cfg, tailscaledConfigAuthkey) + err := kc.setAndWaitForAuthKeyReissue(ctx, client, cfg, tailscaledConfigAuthkey) if err != nil { return fmt.Errorf("failed to get a reissued authkey: %w", err) } @@ -457,7 +610,7 @@ authLoop: } } - w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState|ipn.NotifyRateLimit) + w, err = client.WatchIPNBus(ctx, containerbootWatchMask) if err != nil { return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) } @@ -536,7 +689,7 @@ authLoop: failedResolveAttempts++ } - var egressSvcsNotify chan ipn.Notify + var egressSvcsNotify chan netmapState notifyChan := make(chan ipn.Notify) errChan := make(chan error) go func() { @@ -550,10 +703,12 @@ authLoop: } } }() + var nmState netmapState var wg sync.WaitGroup runLoop: for { + var processNetmap bool select { case <-ctx.Done(): // Although killTailscaled() is deferred earlier, if we @@ -567,244 +722,17 @@ runLoop: case err := <-cfgWatchErrChan: return fmt.Errorf("failed to watch tailscaled config: %w", err) case n := <-notifyChan: - // TODO: (ChaosInTheCRD) Add node removed check when supported by ipn - if n.State != nil && *n.State != ipn.Running { + nmState = nmState.processNotify(ctx, client, n) + if state, ok := notifyState(n); ok && state != ipn.Running { // Something's gone wrong and we've left the authenticated state. // Our container image never recovered gracefully from this, and the // control flow required to make it work now is hard. So, just crash // the container and rely on the container runtime to restart us, // whereupon we'll go through initial auth again. - return fmt.Errorf("tailscaled left running state (now in state %q), exiting", *n.State) + return fmt.Errorf("tailscaled left running state (now in state %q), exiting", state) } - if n.NetMap != nil { - addrs = n.NetMap.SelfNode.Addresses().AsSlice() - newCurrentIPs := deephash.Hash(&addrs) - ipsHaveChanged := newCurrentIPs != currentIPs - - // Store device ID in a Kubernetes Secret before - // setting up any routing rules. This ensures - // that, for containerboot instances that are - // Kubernetes operator proxies, the operator is - // able to retrieve the device ID from the - // Kubernetes Secret to clean up tailnet nodes - // for proxies whose route setup continuously - // fails. - deviceID := n.NetMap.SelfNode.StableID() - if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceID, &deviceID) { - if err := kc.storeDeviceID(ctx, n.NetMap.SelfNode.StableID()); err != nil { - return fmt.Errorf("storing device ID in Kubernetes Secret: %w", err) - } - } - if cfg.TailnetTargetFQDN != "" { - egressAddrs, err := resolveTailnetFQDN(n.NetMap, cfg.TailnetTargetFQDN) - if err != nil { - log.Print(err.Error()) - break - } - - newCurentEgressIPs := deephash.Hash(&egressAddrs) - egressIPsHaveChanged := newCurentEgressIPs != currentEgressIPs - // The firewall rules get (re-)installed: - // - on startup - // - when the tailnet IPs of the tailnet target have changed - // - when the tailnet IPs of this node have changed - if (egressIPsHaveChanged || ipsHaveChanged) && len(egressAddrs) != 0 { - var rulesInstalled bool - for _, egressAddr := range egressAddrs { - ea := egressAddr.Addr() - if ea.Is4() || (ea.Is6() && nfr.HasIPV6NAT()) { - rulesInstalled = true - log.Printf("Installing forwarding rules for destination %v", ea.String()) - if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil { - return fmt.Errorf("installing egress proxy rules for destination %s: %v", ea.String(), err) - } - } - } - if !rulesInstalled { - return fmt.Errorf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT()) - } - } - currentEgressIPs = newCurentEgressIPs - } - if cfg.ProxyTargetIP != "" && len(addrs) != 0 && ipsHaveChanged { - log.Printf("Installing proxy rules") - if err := installIngressForwardingRule(ctx, cfg.ProxyTargetIP, addrs, nfr); err != nil { - return fmt.Errorf("installing ingress proxy rules: %w", err) - } - } - if cfg.ProxyTargetDNSName != "" && len(addrs) != 0 && ipsHaveChanged { - newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName) - if err != nil { - log.Printf("[unexpected] error resolving DNS name %s: %v", cfg.ProxyTargetDNSName, err) - resetTimer(true) - continue - } - backendsHaveChanged := !(slices.EqualFunc(backendAddrs, newBackendAddrs, func(ip1 net.IP, ip2 net.IP) bool { - return slices.ContainsFunc(newBackendAddrs, func(ip net.IP) bool { return ip.Equal(ip1) }) - })) - if backendsHaveChanged { - log.Printf("installing ingress proxy rules for backends %v", newBackendAddrs) - if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil { - return fmt.Errorf("error installing ingress proxy rules: %w", err) - } - } - resetTimer(false) - backendAddrs = newBackendAddrs - } - if cfg.ServeConfigPath != "" { - cd := certDomainFromNetmap(n.NetMap) - if cd == "" { - cd = kubetypes.ValueNoHTTPS - } - prev := certDomain.Swap(new(cd)) - if prev == nil || *prev != cd { - select { - case certDomainChanged <- true: - default: - } - } - } - if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) != 0 { - log.Printf("Installing forwarding rules for destination %v", cfg.TailnetTargetIP) - if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs, nfr); err != nil { - return fmt.Errorf("installing egress proxy rules: %w", err) - } - } - // If this is a L7 cluster ingress proxy (set up - // by Kubernetes operator) and proxying of - // cluster traffic to the ingress target is - // enabled, set up proxy rule each time the - // tailnet IPs of this node change (including - // the first time they become available). - if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) != 0 { - log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP) - if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil { - return fmt.Errorf("installing rules to forward traffic to node's tailnet IP: %w", err) - } - } - currentIPs = newCurrentIPs - - // Only store device FQDN and IP addresses to - // Kubernetes Secret when any required proxy - // route setup has succeeded. IPs and FQDN are - // read from the Secret by the Tailscale - // Kubernetes operator and, for some proxy - // types, such as Tailscale Ingress, advertized - // on the Ingress status. Writing them to the - // Secret only after the proxy routing has been - // set up ensures that the operator does not - // advertize endpoints of broken proxies. - // TODO (irbekrm): instead of using the IP and FQDN, have some other mechanism for the proxy signal that it is 'Ready'. - deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()} - if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceEndpoints, &deviceEndpoints) { - if err := kc.storeDeviceEndpoints(ctx, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil { - return fmt.Errorf("storing device IPs and FQDN in Kubernetes Secret: %w", err) - } - } - - if healthCheck != nil { - healthCheck.Update(len(addrs) != 0) - } - - var prevServeConfig *ipn.ServeConfig - if getAutoAdvertiseBool() { - prevServeConfig, err = client.GetServeConfig(ctx) - if err != nil { - return fmt.Errorf("autoadvertisement: failed to get serve config: %w", err) - } - - err = refreshAdvertiseServices(ctx, prevServeConfig, klc.New(client)) - if err != nil { - return fmt.Errorf("autoadvertisement: failed to refresh advertise services: %w", err) - } - } - - if cfg.ServeConfigPath != "" { - triggerWatchServeConfigChanges.Do(func() { - go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg, prevServeConfig) - }) - } - - if egressSvcsNotify != nil { - egressSvcsNotify <- n - } - } - if !startupTasksDone { - // For containerboot instances that act as TCP proxies (proxying traffic to an endpoint - // passed via one of the env vars that containerboot reads) and store state in a - // Kubernetes Secret, we consider startup tasks done at the point when device info has - // been successfully stored to state Secret. For all other containerboot instances, if - // we just get to this point the startup tasks can be considered done. - if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) { - // This log message is used in tests to detect when all - // post-auth configuration is done. - log.Println("Startup complete, waiting for shutdown signal") - startupTasksDone = true - - // Configure egress proxy. Egress proxy will set up firewall rules to proxy - // traffic to tailnet targets configured in the provided configuration file. It - // will then continuously monitor the config file and netmap updates and - // reconfigure the firewall rules as needed. If any of its operations fail, it - // will crash this node. - if cfg.EgressProxiesCfgPath != "" { - log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressProxiesCfgPath) - egressSvcsNotify = make(chan ipn.Notify) - opts := egressProxyRunOpts{ - cfgPath: cfg.EgressProxiesCfgPath, - nfr: nfr, - kc: kc, - tsClient: client, - stateSecret: cfg.KubeSecret, - netmapChan: egressSvcsNotify, - podIPv4: cfg.PodIPv4, - tailnetAddrs: addrs, - } - go func() { - if err := ep.run(ctx, n, opts); err != nil { - egressSvcsErrorChan <- err - } - }() - } - ip := ingressProxy{} - if cfg.IngressProxiesCfgPath != "" { - log.Printf("configuring ingress proxy using configuration file at %s", cfg.IngressProxiesCfgPath) - opts := ingressProxyOpts{ - cfgPath: cfg.IngressProxiesCfgPath, - nfr: nfr, - kc: kc, - stateSecret: cfg.KubeSecret, - podIPv4: cfg.PodIPv4, - podIPv6: cfg.PodIPv6, - } - go func() { - if err := ip.run(ctx, opts); err != nil { - ingressSvcsErrorChan <- err - } - }() - } - - // Wait on tailscaled process. It won't be cleaned up by default when the - // container exits as it is not PID1. TODO (irbekrm): perhaps we can replace the - // reaper by a running cmd.Wait in a goroutine immediately after starting - // tailscaled? - reaper := func() { - defer wg.Done() - for { - var status unix.WaitStatus - _, err := unix.Wait4(daemonProcess.Pid, &status, 0, nil) - if errors.Is(err, unix.EINTR) { - continue - } - if err != nil { - log.Fatalf("Waiting for tailscaled to exit: %v", err) - } - log.Print("tailscaled exited") - os.Exit(0) - } - } - wg.Add(1) - go reaper() - } + if n.InitialStatus != nil || n.SelfChange != nil || len(n.PeersChanged) != 0 || len(n.PeersRemoved) != 0 || len(n.PeerChangedPatch) != 0 { + processNetmap = true } case <-tc: newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName) @@ -824,11 +752,253 @@ runLoop: } backendAddrs = newBackendAddrs resetTimer(false) + continue case e := <-egressSvcsErrorChan: return fmt.Errorf("egress proxy failed: %v", e) case e := <-ingressSvcsErrorChan: return fmt.Errorf("ingress proxy failed: %v", e) } + if !processNetmap { + continue + } + self := nmState.self + if !self.Valid() { + continue + } + { + addrs = self.Addresses().AsSlice() + newCurrentIPs := deephash.Hash(&addrs) + ipsHaveChanged := newCurrentIPs != currentIPs + + // Store device ID in a Kubernetes Secret before + // setting up any routing rules. This ensures + // that, for containerboot instances that are + // Kubernetes operator proxies, the operator is + // able to retrieve the device ID from the + // Kubernetes Secret to clean up tailnet nodes + // for proxies whose route setup continuously + // fails. + deviceID := self.StableID() + if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceID, &deviceID) { + if err := kc.storeDeviceID(ctx, deviceID); err != nil { + return fmt.Errorf("storing device ID in Kubernetes Secret: %w", err) + } + } + if cfg.TailnetTargetFQDN != "" { + egressAddrs, err := resolveTailnetFQDN(nmState, cfg.TailnetTargetFQDN) + if err != nil { + log.Print(err.Error()) + break + } + + newCurentEgressIPs := deephash.Hash(&egressAddrs) + egressIPsHaveChanged := newCurentEgressIPs != currentEgressIPs + // The firewall rules get (re-)installed: + // - on startup + // - when the tailnet IPs of the tailnet target have changed + // - when the tailnet IPs of this node have changed + if (egressIPsHaveChanged || ipsHaveChanged) && len(egressAddrs) != 0 { + var rulesInstalled bool + for _, egressAddr := range egressAddrs { + ea := egressAddr.Addr() + if ea.Is4() || (ea.Is6() && nfr.HasIPV6NAT()) { + rulesInstalled = true + log.Printf("Installing forwarding rules for destination %v", ea.String()) + if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil { + return fmt.Errorf("installing egress proxy rules for destination %s: %v", ea.String(), err) + } + } + } + if !rulesInstalled { + return fmt.Errorf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT()) + } + } + currentEgressIPs = newCurentEgressIPs + } + if cfg.ProxyTargetIP != "" && len(addrs) != 0 && ipsHaveChanged { + log.Printf("Installing proxy rules") + if err := installIngressForwardingRule(ctx, cfg.ProxyTargetIP, addrs, nfr); err != nil { + return fmt.Errorf("installing ingress proxy rules: %w", err) + } + } + if cfg.ProxyTargetDNSName != "" && len(addrs) != 0 && ipsHaveChanged { + newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName) + if err != nil { + log.Printf("[unexpected] error resolving DNS name %s: %v", cfg.ProxyTargetDNSName, err) + resetTimer(true) + continue + } + backendsHaveChanged := !(slices.EqualFunc(backendAddrs, newBackendAddrs, func(ip1 net.IP, ip2 net.IP) bool { + return slices.ContainsFunc(newBackendAddrs, func(ip net.IP) bool { return ip.Equal(ip1) }) + })) + if backendsHaveChanged { + log.Printf("installing ingress proxy rules for backends %v", newBackendAddrs) + if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil { + return fmt.Errorf("error installing ingress proxy rules: %w", err) + } + } + resetTimer(false) + backendAddrs = newBackendAddrs + } + if cfg.ServeConfigPath != "" { + var cd string + if nmState.certDomains.Len() != 0 { + cd = nmState.certDomains.At(0) + } + if cd == "" { + cd = kubetypes.ValueNoHTTPS + } + prev := certDomain.Swap(new(cd)) + if prev == nil || *prev != cd { + select { + case certDomainChanged <- true: + default: + } + } + } + if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) != 0 { + log.Printf("Installing forwarding rules for destination %v", cfg.TailnetTargetIP) + if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs, nfr); err != nil { + return fmt.Errorf("installing egress proxy rules: %w", err) + } + } + // If this is a L7 cluster ingress proxy (set up + // by Kubernetes operator) and proxying of + // cluster traffic to the ingress target is + // enabled, set up proxy rule each time the + // tailnet IPs of this node change (including + // the first time they become available). + if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) != 0 { + log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP) + if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil { + return fmt.Errorf("installing rules to forward traffic to node's tailnet IP: %w", err) + } + } + currentIPs = newCurrentIPs + + // Only store device FQDN and IP addresses to + // Kubernetes Secret when any required proxy + // route setup has succeeded. IPs and FQDN are + // read from the Secret by the Tailscale + // Kubernetes operator and, for some proxy + // types, such as Tailscale Ingress, advertized + // on the Ingress status. Writing them to the + // Secret only after the proxy routing has been + // set up ensures that the operator does not + // advertize endpoints of broken proxies. + // TODO (irbekrm): instead of using the IP and FQDN, have some other mechanism for the proxy signal that it is 'Ready'. + deviceEndpoints := []any{self.Name(), self.Addresses()} + if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceEndpoints, &deviceEndpoints) { + if err := kc.storeDeviceEndpoints(ctx, self.Name(), addrs); err != nil { + return fmt.Errorf("storing device IPs and FQDN in Kubernetes Secret: %w", err) + } + } + + if healthCheck != nil { + healthCheck.Update(len(addrs) != 0) + } + + var prevServeConfig *ipn.ServeConfig + if getAutoAdvertiseBool() { + prevServeConfig, err = client.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("autoadvertisement: failed to get serve config: %w", err) + } + + err = refreshAdvertiseServices(ctx, prevServeConfig, klc.New(client)) + if err != nil { + return fmt.Errorf("autoadvertisement: failed to refresh advertise services: %w", err) + } + } + + if cfg.ServeConfigPath != "" { + triggerWatchServeConfigChanges.Do(func() { + go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg, prevServeConfig) + }) + } + + if egressSvcsNotify != nil { + egressSvcsNotify <- nmState + } + } + if !startupTasksDone { + // For containerboot instances that act as TCP proxies (proxying traffic to an endpoint + // passed via one of the env vars that containerboot reads) and store state in a + // Kubernetes Secret, we consider startup tasks done at the point when device info has + // been successfully stored to state Secret. For all other containerboot instances, if + // we just get to this point the startup tasks can be considered done. + if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) { + // This log message is used in tests to detect when all + // post-auth configuration is done. + log.Println("Startup complete, waiting for shutdown signal") + startupTasksDone = true + + // Configure egress proxy. Egress proxy will set up firewall rules to proxy + // traffic to tailnet targets configured in the provided configuration file. It + // will then continuously monitor the config file and netmap updates and + // reconfigure the firewall rules as needed. If any of its operations fail, it + // will crash this node. + if cfg.EgressProxiesCfgPath != "" { + log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressProxiesCfgPath) + egressSvcsNotify = make(chan netmapState) + opts := egressProxyRunOpts{ + cfgPath: cfg.EgressProxiesCfgPath, + nfr: nfr, + kc: kc, + tsClient: client, + stateSecret: cfg.KubeSecret, + netmapChan: egressSvcsNotify, + podIPv4: cfg.PodIPv4, + podIPv6: cfg.PodIPv6, + tailnetAddrs: addrs, + } + go func() { + if err := ep.run(ctx, nmState, opts); err != nil { + egressSvcsErrorChan <- err + } + }() + } + ip := ingressProxy{} + if cfg.IngressProxiesCfgPath != "" { + log.Printf("configuring ingress proxy using configuration file at %s", cfg.IngressProxiesCfgPath) + opts := ingressProxyOpts{ + cfgPath: cfg.IngressProxiesCfgPath, + nfr: nfr, + kc: kc, + stateSecret: cfg.KubeSecret, + podIPv4: cfg.PodIPv4, + podIPv6: cfg.PodIPv6, + } + go func() { + if err := ip.run(ctx, opts); err != nil { + ingressSvcsErrorChan <- err + } + }() + } + + // Wait on tailscaled process. It won't be cleaned up by default when the + // container exits as it is not PID1. TODO (irbekrm): perhaps we can replace the + // reaper by a running cmd.Wait in a goroutine immediately after starting + // tailscaled? + reaper := func() { + defer wg.Done() + for { + var status unix.WaitStatus + _, err := unix.Wait4(daemonProcess.Pid, &status, 0, nil) + if errors.Is(err, unix.EINTR) { + continue + } + if err != nil { + log.Fatalf("Waiting for tailscaled to exit: %v", err) + } + log.Print("tailscaled exited") + os.Exit(0) + } + } + wg.Add(1) + go reaper() + } + } } wg.Wait() @@ -964,34 +1134,52 @@ func runHTTPServer(mux *http.ServeMux, addr string) (close func() error) { } // resolveTailnetFQDN resolves a tailnet FQDN to a list of IP prefixes, which -// can be either a peer device or a Tailscale Service. -func resolveTailnetFQDN(nm *netmap.NetworkMap, fqdn string) ([]netip.Prefix, error) { +// can be either a peer device, a Tailscale Service, or a 4via6 synthesized +// DNS name (e.g. "10-1-0-5-via-7.tailnet.ts.net"). +func resolveTailnetFQDN(nm netmapState, fqdn string) ([]netip.Prefix, error) { dnsFQDN, err := dnsname.ToFQDN(fqdn) if err != nil { return nil, fmt.Errorf("error parsing %q as FQDN: %w", fqdn, err) } // Check all peer devices first. - for _, p := range nm.Peers { + var ret []netip.Prefix + for p := range nm.peers() { if strings.EqualFold(p.Name(), dnsFQDN.WithTrailingDot()) { - return p.Addresses().AsSlice(), nil + ret = p.Addresses().AsSlice() + break } } + if ret != nil { + return ret, nil + } // If not found yet, check for a matching Tailscale Service. if svcIPs := serviceIPsFromNetMap(nm, dnsFQDN); len(svcIPs) != 0 { return svcIPs, nil } + // If not found yet, check for a matching 4via6 DNS name. + if addr, ok := kubeutils.ResolveViaDomain(dnsFQDN.WithTrailingDot()); ok { + prefix := netip.PrefixFrom(addr, addr.BitLen()) + for nn := range nm.peers() { + for _, allowedIP := range nn.AllowedIPs().All() { + if allowedIP.Contains(addr) { + return []netip.Prefix{prefix}, nil + } + } + } + return nil, fmt.Errorf("resolved 4via6 address %v for %q but no peer advertises a route containing it", addr, fqdn) + } - return nil, fmt.Errorf("could not find Tailscale node or service %q; it either does not exist, or not reachable because of ACLs", fqdn) + return nil, fmt.Errorf("could not find Tailscale node, service or 4via6 address %q; it either does not exist, or not reachable because of ACLs", fqdn) } // serviceIPsFromNetMap returns all IPs of a Tailscale Service if its FQDN is // found in the netmap. Note that Tailscale Services are not a first-class // object in the netmap, so we guess based on DNS ExtraRecords and AllowedIPs. -func serviceIPsFromNetMap(nm *netmap.NetworkMap, fqdn dnsname.FQDN) []netip.Prefix { +func serviceIPsFromNetMap(nm netmapState, fqdn dnsname.FQDN) []netip.Prefix { var extraRecords []tailcfg.DNSRecord - for _, rec := range nm.DNS.ExtraRecords { + for _, rec := range nm.dnsExtraRecords.All() { recFQDN, err := dnsname.ToFQDN(rec.Name) if err != nil { continue @@ -1013,7 +1201,7 @@ func serviceIPsFromNetMap(nm *netmap.NetworkMap, fqdn dnsname.FQDN) []netip.Pref continue } ipPrefix := netip.PrefixFrom(ip, ip.BitLen()) - for _, ps := range nm.Peers { + for ps := range nm.peers() { for _, allowedIP := range ps.AllowedIPs().All() { if allowedIP == ipPrefix { prefixes = append(prefixes, ipPrefix) @@ -1024,11 +1212,3 @@ func serviceIPsFromNetMap(nm *netmap.NetworkMap, fqdn dnsname.FQDN) []netip.Pref return prefixes } - -func authkeyFromTailscaledConfig(path string) string { - if cfg, err := conffile.Load(path); err == nil && cfg.Parsed.AuthKey != nil { - return *cfg.Parsed.AuthKey - } - - return "" -} diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index 5ea402f66..0013bb8e4 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -7,6 +7,7 @@ package main import ( "bytes" + "context" _ "embed" "encoding/base64" "encoding/json" @@ -32,24 +33,30 @@ import ( "github.com/google/go-cmp/cmp" "golang.org/x/sys/unix" + "tailscale.com/client/local" + "tailscale.com/cmd/testwrapper/flakytest" "tailscale.com/health" "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" + "tailscale.com/net/memnet" "tailscale.com/tailcfg" "tailscale.com/tstest" - "tailscale.com/types/netmap" + "tailscale.com/types/key" ) const configFileAuthKey = "some-auth-key" func TestContainerBoot(t *testing.T) { + flakytest.Mark(t, "https://github.com/tailscale/tailscale/issues/19380") boot := filepath.Join(t.TempDir(), "containerboot") if err := exec.Command("go", "build", "-ldflags", "-X main.testSleepDuration=1ms", "-o", boot, "tailscale.com/cmd/containerboot").Run(); err != nil { t.Fatalf("Building containerboot: %v", err) } egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net", "100.64.0.2") + egressStatusUpdated := egressSvcStatus("foo", "foo.tailnetxyz.ts.net", "100.64.0.3") metricsURL := func(port int) string { return fmt.Sprintf("http://127.0.0.1:%d/metrics", port) @@ -103,12 +110,10 @@ func TestContainerBoot(t *testing.T) { } runningNotify := &ipn.Notify{ State: new(ipn.Running), - NetMap: &netmap.NetworkMap{ - SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("myID"), - Name: "test-node.test.ts.net.", - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, - }).View(), + SelfChange: &tailcfg.Node{ + StableID: tailcfg.StableNodeID("myID"), + Name: "test-node.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, } type testCase struct { @@ -381,18 +386,16 @@ func TestContainerBoot(t *testing.T) { { Notify: &ipn.Notify{ State: new(ipn.Running), - NetMap: &netmap.NetworkMap{ - SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("myID"), - Name: "test-node.test.ts.net.", - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, - }).View(), - Peers: []tailcfg.NodeView{ - (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("ipv6ID"), - Name: "ipv6-node.test.ts.net.", - Addresses: []netip.Prefix{netip.MustParsePrefix("::1/128")}, - }).View(), + SelfChange: &tailcfg.Node{ + StableID: tailcfg.StableNodeID("myID"), + Name: "test-node.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, + }, + PeersChanged: []*tailcfg.Node{ + { + StableID: tailcfg.StableNodeID("ipv6ID"), + Name: "ipv6-node.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("::1/128")}, }, }, }, @@ -629,12 +632,10 @@ func TestContainerBoot(t *testing.T) { { Notify: &ipn.Notify{ State: new(ipn.Running), - NetMap: &netmap.NetworkMap{ - SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("newID"), - Name: "new-name.test.ts.net.", - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, - }).View(), + SelfChange: &tailcfg.Node{ + StableID: tailcfg.StableNodeID("newID"), + Name: "new-name.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, }, WantKubeSecret: map[string]string{ @@ -1093,18 +1094,16 @@ func TestContainerBoot(t *testing.T) { { Notify: &ipn.Notify{ State: new(ipn.Running), - NetMap: &netmap.NetworkMap{ - SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("myID"), - Name: "test-node.test.ts.net.", - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, - }).View(), - Peers: []tailcfg.NodeView{ - (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("fooID"), - Name: "foo.tailnetxyz.ts.net.", - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}, - }).View(), + SelfChange: &tailcfg.Node{ + StableID: tailcfg.StableNodeID("myID"), + Name: "test-node.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, + }, + PeersChanged: []*tailcfg.Node{ + { + StableID: tailcfg.StableNodeID("fooID"), + Name: "foo.tailnetxyz.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}, }, }, }, @@ -1120,6 +1119,23 @@ func TestContainerBoot(t *testing.T) { egressSvcTerminateURL(env.localAddrPort): 200, }, }, + { + Notify: &ipn.Notify{ + PeersChanged: []*tailcfg.Node{{ + StableID: tailcfg.StableNodeID("fooID"), + Name: "foo.tailnetxyz.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.3/32")}, + }}, + }, + WantKubeSecret: map[string]string{ + "egress-services": string(mustJSON(t, egressStatusUpdated)), + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net.", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, + }, + }, }, } }, @@ -1274,6 +1290,12 @@ func TestContainerBoot(t *testing.T) { t.Fatalf("phase %d: updating mtime for %q: %v", i, path, err) } } + if p.Notify != nil && p.Notify.InitialStatus == nil { + // Shallow-copy before mutating to avoid a race with + // parallel subtests that share the same *ipn.Notify. + p.Notify = new(*p.Notify) + p.Notify.InitialStatus = statusFromNotify(p.Notify) + } env.lapi.Notify(p.Notify) if p.Signal != nil { cmd.Process.Signal(*p.Signal) @@ -1502,6 +1524,43 @@ func (lc *localAPI) Notify(n *ipn.Notify) { lc.cond.Broadcast() } +func statusFromNotify(n *ipn.Notify) *ipnstate.Status { + st := new(ipnstate.Status) + if n.State != nil { + st.BackendState = n.State.String() + } + if n.SelfChange != nil { + st.Self = peerStatusFromNode(n.SelfChange.View()) + } + if len(n.PeersChanged) != 0 { + st.Peer = map[key.NodePublic]*ipnstate.PeerStatus{} + for _, p := range n.PeersChanged { + pv := p.View() + st.Peer[pv.Key()] = peerStatusFromNode(pv) + } + } + return st +} + +func peerStatusFromNode(n tailcfg.NodeView) *ipnstate.PeerStatus { + ps := &ipnstate.PeerStatus{ + ID: n.StableID(), + NodeID: n.ID(), + PublicKey: n.Key(), + DNSName: n.Name(), + } + for _, p := range n.Addresses().All() { + if p.IsSingleIP() { + ps.TailscaleIPs = append(ps.TailscaleIPs, p.Addr()) + } + } + if n.AllowedIPs().Len() != 0 { + v := n.AllowedIPs() + ps.AllowedIPs = &v + } + return ps +} + func (lc *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/localapi/v0/serve-config": @@ -1889,3 +1948,57 @@ func newTestEnv(t *testing.T) testEnv { healthAddrPort: healthAddrPort, } } + +// TestProcessNotifyRefreshesDNSOnSelfChange verifies that a SelfChange +// notification triggers a DNS refresh; without it, VIPServices created +// after pod boot are invisible to resolveTailnetFQDN. +func TestProcessNotifyRefreshesDNSOnSelfChange(t *testing.T) { + extraRec := tailcfg.DNSRecord{ + Name: "my-ingress.tailnet.ts.net.", + Type: "A", + Value: "100.99.10.20", + } + dnsCfg := &tailcfg.DNSConfig{ + ExtraRecords: []tailcfg.DNSRecord{extraRec}, + CertDomains: []string{"node.tailnet.ts.net"}, + } + + lal := memnet.Listen("local-tailscaled.sock:80") + defer lal.Close() + mux := http.NewServeMux() + mux.HandleFunc("/localapi/v0/dns-config", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(dnsCfg); err != nil { + t.Errorf("encoding dns config: %v", err) + } + }) + srv := &http.Server{Handler: mux} + go srv.Serve(lal) + t.Cleanup(func() { srv.Shutdown(context.Background()) }) + + client := &local.Client{Dial: lal.Dial} + + // Empty starting state, as if the InitialStatus captured at pod + // boot carried no ExtraRecords because the VIPService didn't exist + // yet at that time. + var s netmapState + + n := ipn.Notify{ + SelfChange: &tailcfg.Node{ + ID: 1, + Name: "self.tailnet.ts.net.", + }, + } + + got := s.processNotify(context.Background(), client, n) + + if got.dnsExtraRecords.Len() != 1 { + t.Fatalf("dnsExtraRecords.Len() = %d, want 1", got.dnsExtraRecords.Len()) + } + if rec := got.dnsExtraRecords.At(0); rec.Name != extraRec.Name { + t.Errorf("dnsExtraRecords[0].Name = %q, want %q", rec.Name, extraRec.Name) + } + if got.certDomains.Len() != 1 || got.certDomains.At(0) != "node.tailnet.ts.net" { + t.Errorf("certDomains = %v, want [node.tailnet.ts.net]", got.certDomains.AsSlice()) + } +} diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index f64d2d24f..f5423630f 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -24,7 +24,6 @@ import ( "tailscale.com/kube/kubetypes" klc "tailscale.com/kube/localclient" "tailscale.com/kube/services" - "tailscale.com/types/netmap" ) // watchServeConfigChanges watches path for changes, and when it sees one, reads @@ -142,13 +141,6 @@ func refreshAdvertiseServices(ctx context.Context, sc *ipn.ServeConfig, lc klc.L return nil } -func certDomainFromNetmap(nm *netmap.NetworkMap) string { - if len(nm.DNS.CertDomains) == 0 { - return "" - } - return nm.DNS.CertDomains[0] -} - func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc klc.LocalClient) error { if !isValidHTTPSConfig(certDomain, sc) { return nil diff --git a/cmd/containerboot/settings.go b/cmd/containerboot/settings.go index f695f2e5d..17483194f 100644 --- a/cmd/containerboot/settings.go +++ b/cmd/containerboot/settings.go @@ -6,6 +6,7 @@ package main import ( + "cmp" "context" "errors" "fmt" @@ -18,6 +19,7 @@ import ( "tailscale.com/ipn/conffile" "tailscale.com/kube/kubeclient" + "tailscale.com/util/def" ) // settings is all the configuration for containerboot. @@ -89,47 +91,50 @@ type settings struct { func configFromEnv() (*settings, error) { cfg := &settings{ - AuthKey: defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""), - ClientID: defaultEnv("TS_CLIENT_ID", ""), - ClientSecret: defaultEnv("TS_CLIENT_SECRET", ""), - IDToken: defaultEnv("TS_ID_TOKEN", ""), - Audience: defaultEnv("TS_AUDIENCE", ""), - Hostname: defaultEnv("TS_HOSTNAME", ""), + AuthKey: cmp.Or(os.Getenv("TS_AUTHKEY"), os.Getenv("TS_AUTH_KEY")), + ClientID: os.Getenv("TS_CLIENT_ID"), + ClientSecret: os.Getenv("TS_CLIENT_SECRET"), + IDToken: os.Getenv("TS_ID_TOKEN"), + Audience: os.Getenv("TS_AUDIENCE"), + Hostname: os.Getenv("TS_HOSTNAME"), Routes: defaultEnvStringPointer("TS_ROUTES"), - ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), - ProxyTargetIP: defaultEnv("TS_DEST_IP", ""), - ProxyTargetDNSName: defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""), - TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""), - TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""), - DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), - ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), + ServeConfigPath: os.Getenv("TS_SERVE_CONFIG"), + ProxyTargetIP: os.Getenv("TS_DEST_IP"), + ProxyTargetDNSName: os.Getenv("TS_EXPERIMENTAL_DEST_DNS_NAME"), + TailnetTargetIP: os.Getenv("TS_TAILNET_TARGET_IP"), + TailnetTargetFQDN: os.Getenv("TS_TAILNET_TARGET_FQDN"), + DaemonExtraArgs: os.Getenv("TS_TAILSCALED_EXTRA_ARGS"), + ExtraArgs: os.Getenv("TS_EXTRA_ARGS"), InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", - UserspaceMode: defaultBool("TS_USERSPACE", true), - StateDir: defaultEnv("TS_STATE_DIR", ""), + UserspaceMode: def.Bool(os.Getenv("TS_USERSPACE"), true), + StateDir: os.Getenv("TS_STATE_DIR"), AcceptDNS: defaultEnvBoolPointer("TS_ACCEPT_DNS"), KubeSecret: func() string { - if os.Getenv("KUBERNETES_SERVICE_HOST") != "" { - return defaultEnv("TS_KUBE_SECRET", "tailscale") + if os.Getenv("KUBERNETES_SERVICE_HOST") == "" { + return os.Getenv("TS_KUBE_SECRET") } - return defaultEnv("TS_KUBE_SECRET", "") + // An explicitly empty TS_KUBE_SECRET disables Secret storage, so + // unset and empty must stay distinguishable: def.LookupEnv keeps + // an explicit "" rather than falling back to the default. + return def.LookupEnv("TS_KUBE_SECRET", "tailscale") }(), - SOCKSProxyAddr: defaultEnv("TS_SOCKS5_SERVER", ""), - HTTPProxyAddr: defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""), - Socket: defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"), - AuthOnce: defaultBool("TS_AUTH_ONCE", false), - Root: defaultEnv("TS_TEST_ONLY_ROOT", "/"), + SOCKSProxyAddr: os.Getenv("TS_SOCKS5_SERVER"), + HTTPProxyAddr: os.Getenv("TS_OUTBOUND_HTTP_PROXY_LISTEN"), + Socket: cmp.Or(os.Getenv("TS_SOCKET"), "/tmp/tailscaled.sock"), + AuthOnce: def.Bool(os.Getenv("TS_AUTH_ONCE"), false), + Root: cmp.Or(os.Getenv("TS_TEST_ONLY_ROOT"), "/"), TailscaledConfigFilePath: tailscaledConfigFilePath(), - AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false), - PodIP: defaultEnv("POD_IP", ""), - EnableForwardingOptimizations: defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false), - HealthCheckAddrPort: defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""), - LocalAddrPort: defaultEnv("TS_LOCAL_ADDR_PORT", "[::]:9002"), - MetricsEnabled: defaultBool("TS_ENABLE_METRICS", false), - HealthCheckEnabled: defaultBool("TS_ENABLE_HEALTH_CHECK", false), - DebugAddrPort: defaultEnv("TS_DEBUG_ADDR_PORT", ""), - EgressProxiesCfgPath: defaultEnv("TS_EGRESS_PROXIES_CONFIG_PATH", ""), - IngressProxiesCfgPath: defaultEnv("TS_INGRESS_PROXIES_CONFIG_PATH", ""), - PodUID: defaultEnv("POD_UID", ""), + AllowProxyingClusterTrafficViaIngress: def.Bool(os.Getenv("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"), false), + PodIP: os.Getenv("POD_IP"), + EnableForwardingOptimizations: def.Bool(os.Getenv("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS"), false), + HealthCheckAddrPort: os.Getenv("TS_HEALTHCHECK_ADDR_PORT"), + LocalAddrPort: cmp.Or(os.Getenv("TS_LOCAL_ADDR_PORT"), "[::]:9002"), + MetricsEnabled: def.Bool(os.Getenv("TS_ENABLE_METRICS"), false), + HealthCheckEnabled: def.Bool(os.Getenv("TS_ENABLE_HEALTH_CHECK"), false), + DebugAddrPort: os.Getenv("TS_DEBUG_ADDR_PORT"), + EgressProxiesCfgPath: os.Getenv("TS_EGRESS_PROXIES_CONFIG_PATH"), + IngressProxiesCfgPath: os.Getenv("TS_INGRESS_PROXIES_CONFIG_PATH"), + PodUID: os.Getenv("POD_UID"), } podIPs, ok := os.LookupEnv("POD_IPS") @@ -153,7 +158,7 @@ func configFromEnv() (*settings, error) { // If cert share is enabled, set the replica as read or write. Only 0th // replica should be able to write. - isInCertShareMode := defaultBool("TS_EXPERIMENTAL_CERT_SHARE", false) + isInCertShareMode := def.Bool(os.Getenv("TS_EXPERIMENTAL_CERT_SHARE"), false) if isInCertShareMode { cfg.CertShareMode = "ro" podName := os.Getenv("POD_NAME") @@ -454,15 +459,6 @@ func (cfg *settings) egressSvcsTerminateEPEnabled() bool { return cfg.LocalAddrPort != "" && cfg.EgressProxiesCfgPath != "" } -// defaultEnv returns the value of the given envvar name, or defVal if -// unset. -func defaultEnv(name, defVal string) string { - if v, ok := os.LookupEnv(name); ok { - return v - } - return defVal -} - // defaultEnvStringPointer returns a pointer to the given envvar value if set, else // returns nil. This is useful in cases where we need to distinguish between a // variable being set to empty string vs unset. @@ -484,23 +480,3 @@ func defaultEnvBoolPointer(name string) *bool { } return &ret } - -func defaultEnvs(names []string, defVal string) string { - for _, name := range names { - if v, ok := os.LookupEnv(name); ok { - return v - } - } - return defVal -} - -// defaultBool returns the boolean value of the given envvar name, or -// defVal if unset or not a bool. -func defaultBool(name string, defVal bool) bool { - v := os.Getenv(name) - ret, err := strconv.ParseBool(v) - if err != nil { - return defVal - } - return ret -} diff --git a/cmd/containerboot/settings_test.go b/cmd/containerboot/settings_test.go index eca50101b..c2a61b85f 100644 --- a/cmd/containerboot/settings_test.go +++ b/cmd/containerboot/settings_test.go @@ -7,6 +7,7 @@ package main import ( "net/netip" + "os" "strings" "testing" ) @@ -228,6 +229,78 @@ func TestValidateAuthMethods(t *testing.T) { } } +func TestConfigFromEnvEmptyDefaults(t *testing.T) { + tests := []struct { + env string + get func(*settings) string + want string + }{ + { + env: "TS_SOCKET", + get: func(c *settings) string { return c.Socket }, + want: "/tmp/tailscaled.sock", + }, + { + env: "TS_LOCAL_ADDR_PORT", + get: func(c *settings) string { return c.LocalAddrPort }, + want: "[::]:9002", + }, + { + env: "TS_TEST_ONLY_ROOT", + get: func(c *settings) string { return c.Root }, + want: "/", + }, + } + for _, tt := range tests { + t.Run(tt.env, func(t *testing.T) { + t.Setenv(tt.env, "") + cfg, err := configFromEnv() + if err != nil { + t.Fatal(err) + } + if got := tt.get(cfg); got != tt.want { + t.Errorf(`%s set to empty "": got %q, want default %q`, tt.env, got, tt.want) + } + }) + } +} + +func TestConfigFromEnvKubeSecret(t *testing.T) { + tests := []struct { + name string + inKubernetes bool + unset bool + value string + want string + }{ + {name: "in_kubernetes_unset", inKubernetes: true, unset: true, want: "tailscale"}, + {name: "in_kubernetes_empty", inKubernetes: true, value: "", want: ""}, + {name: "in_kubernetes_set", inKubernetes: true, value: "custom", want: "custom"}, + {name: "not_in_kubernetes_unset", inKubernetes: false, unset: true, want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // t.Setenv registers a t.Cleanup to restore the original value, so + // route the unset cases through it rather than a bare os.Unsetenv. + t.Setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") + if !tt.inKubernetes { + os.Unsetenv("KUBERNETES_SERVICE_HOST") + } + t.Setenv("TS_KUBE_SECRET", tt.value) + if tt.unset { + os.Unsetenv("TS_KUBE_SECRET") + } + cfg, err := configFromEnv() + if err != nil { + t.Fatal(err) + } + if cfg.KubeSecret != tt.want { + t.Errorf("KubeSecret = %q, want %q", cfg.KubeSecret, tt.want) + } + }) + } +} + func TestHandlesKubeIPV6(t *testing.T) { t.Setenv("TS_LOCAL_ADDR_PORT", "fd7a:115c:a1e0::6c34:352:9002") t.Setenv("POD_IPS", "fd7a:115c:a1e0::6c34:352") diff --git a/cmd/containerboot/tailscaled.go b/cmd/containerboot/tailscaled.go index 6f4ed77e7..379dd3eeb 100644 --- a/cmd/containerboot/tailscaled.go +++ b/cmd/containerboot/tailscaled.go @@ -150,7 +150,15 @@ func tailscaleUp(ctx context.Context, cfg *settings) error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - return fmt.Errorf("tailscale up failed: %v", err) + if ctxErr := ctx.Err(); ctxErr != nil { + // A canceled context kills the command, and cmd.Run can + // report the subprocess's death ("signal: killed") rather + // than the context error that caused it. Return the + // context error so that callers (and ultimately main) can + // recognize a graceful shutdown with errors.Is. + return fmt.Errorf("tailscale up failed: %w", ctxErr) + } + return fmt.Errorf("tailscale up failed: %w", err) } return nil } @@ -180,7 +188,11 @@ func tailscaleSet(ctx context.Context, cfg *settings) error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - return fmt.Errorf("tailscale set failed: %v", err) + if ctxErr := ctx.Err(); ctxErr != nil { + // See the equivalent check in tailscaleUp. + return fmt.Errorf("tailscale set failed: %w", ctxErr) + } + return fmt.Errorf("tailscale set failed: %w", err) } return nil } diff --git a/cmd/derper/bootstrap_dns_test.go b/cmd/derper/bootstrap_dns_test.go index 5b765f6d3..2055b9751 100644 --- a/cmd/derper/bootstrap_dns_test.go +++ b/cmd/derper/bootstrap_dns_test.go @@ -41,8 +41,28 @@ func (b *bitbucketResponseWriter) Write(p []byte) (int, error) { return len(p), func (b *bitbucketResponseWriter) WriteHeader(statusCode int) {} +// setDNSCache sets the published DNS cache for tests. +func setDNSCache(tb testing.TB, m *dnsEntryMap) { + tb.Helper() + j, err := json.Marshal(m.IPs) + if err != nil { + tb.Fatal(err) + } + tstest.AssertNotParallel(tb) + dnsCache.Store(m) + dnsCacheBytes.Store(j) + tb.Cleanup(func() { + dnsCache.Store(nil) + dnsCacheBytes.Store(nil) + }) +} + func getBootstrapDNS(t *testing.T, q string) map[string][]net.IP { t.Helper() + tstest.AssertNotParallel(t) + if dnsCache.Load() == nil { + t.Fatal("dnsCache not initialized; call setDNSCache before getBootstrapDNS") + } req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape(q), nil) w := httptest.NewRecorder() handleBootstrapDNS(w, req) @@ -100,7 +120,8 @@ func TestUnpublishedDNS(t *testing.T) { } } -func resetMetrics() { +func resetMetrics(tb testing.TB) { + tstest.AssertNotParallel(tb) publishedDNSHits.Set(0) publishedDNSMisses.Set(0) unpublishedDNSHits.Set(0) @@ -114,8 +135,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { pub := &dnsEntryMap{ IPs: map[string][]net.IP{"tailscale.com": {net.IPv4(10, 10, 10, 10)}}, } - dnsCache.Store(pub) - dnsCacheBytes.Store([]byte(`{"tailscale.com":["10.10.10.10"]}`)) + setDNSCache(t, pub) unpublishedDNSCache.Store(&dnsEntryMap{ IPs: map[string][]net.IP{ @@ -131,7 +151,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { t.Run("CacheMiss", func(t *testing.T) { // One domain in map but empty, one not in map at all for _, q := range []string{"log.tailscale.com", "login.tailscale.com"} { - resetMetrics() + resetMetrics(t) ips := getBootstrapDNS(t, q) // Expected our public map to be returned on a cache miss @@ -149,7 +169,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { // Verify that we do get a valid response and metric. t.Run("CacheHit", func(t *testing.T) { - resetMetrics() + resetMetrics(t) ips := getBootstrapDNS(t, "controlplane.tailscale.com") want := map[string][]net.IP{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}} if !reflect.DeepEqual(ips, want) { @@ -166,8 +186,10 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { } func TestLookupMetric(t *testing.T) { + setDNSCache(t, &dnsEntryMap{}) + d := []string{"a.io", "b.io", "c.io", "d.io", "e.io", "e.io", "e.io", "a.io"} - resetMetrics() + resetMetrics(t) for _, q := range d { _ = getBootstrapDNS(t, q) } diff --git a/cmd/derper/cert.go b/cmd/derper/cert.go index 979c0d671..d12a457c2 100644 --- a/cmd/derper/cert.go +++ b/cmd/derper/cert.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "time" "golang.org/x/crypto/acme" @@ -35,21 +36,34 @@ var unsafeHostnameCharacters = regexp.MustCompile(`[^a-zA-Z0-9-\.]`) type certProvider interface { // TLSConfig creates a new TLS config suitable for net/http.Server servers. // - // The returned Config must have a GetCertificate function set and that - // function must return a unique *tls.Certificate for each call. The - // returned *tls.Certificate will be mutated by the caller to append to the - // (*tls.Certificate).Certificate field. + // The returned Config must have a GetCertificate function set. The + // *tls.Certificate values it returns may be shared and cached, so + // callers must not mutate them. TLSConfig() *tls.Config // HTTPHandler handle ACME related request, if any. HTTPHandler(fallback http.Handler) http.Handler } -func certProviderByCertMode(mode, dir, hostname, eabKID, eabKey, email string) (certProvider, error) { +func certProviderByCertMode(mode, dir, hostname string, ipCerts bool, eabKID, eabKey, email string) (certProvider, error) { if dir == "" { return nil, errors.New("missing required --certdir flag") } + if ipCerts && mode != "letsencrypt" { + return nil, errors.New("--acme-ip-certs requires --certmode=letsencrypt") + } switch mode { case "letsencrypt", "gcp": + if net.ParseIP(hostname) != nil { + if mode == "gcp" { + return nil, errors.New("--certmode=gcp requires --hostname to be a DNS name, not an IP address") + } + if !ipCerts { + return nil, errors.New("--hostname is an IP address; use --certmode=manual for a self-signed cert, or set --acme-ip-certs to get LetsEncrypt IP address certs") + } + // IP-only server: certs are issued on demand per + // connection, so there is no hostname cert provider. + return newIPCertManager(dir, email, "", nil) + } certManager := &autocert.Manager{ Prompt: autocert.AcceptTOS, HostPolicy: autocert.HostWhitelist(hostname), @@ -82,6 +96,9 @@ func certProviderByCertMode(mode, dir, hostname, eabKID, eabKey, email string) ( } else if hostname == "derp.tailscale.com" { certManager.Email = "security@tailscale.com" } + if ipCerts { + return newIPCertManager(dir, email, "", certManager) + } return certManager, nil case "manual": return NewManualCertManager(dir, hostname) @@ -157,12 +174,11 @@ func (m *manualCertManager) getCertificate(hi *tls.ClientHelloInfo) (*tls.Certif return nil, fmt.Errorf("cert mismatch with hostname: %q", hi.ServerName) } - // Return a shallow copy of the cert so the caller can append to its - // Certificate field. - certCopy := new(tls.Certificate) - *certCopy = *m.cert - certCopy.Certificate = certCopy.Certificate[:len(certCopy.Certificate):len(certCopy.Certificate)] - return certCopy, nil + // Return a shallow copy of the cert with a capacity-clamped chain + // so callers can never mutate the manager's long-lived certificate. + certCopy := *m.cert + certCopy.Certificate = slices.Clip(certCopy.Certificate) + return &certCopy, nil } func (m *manualCertManager) HTTPHandler(fallback http.Handler) http.Handler { diff --git a/cmd/derper/cert_test.go b/cmd/derper/cert_test.go index e111ed76b..1b359385a 100644 --- a/cmd/derper/cert_test.go +++ b/cmd/derper/cert_test.go @@ -91,7 +91,7 @@ func TestCertIP(t *testing.T) { t.Fatalf("Error closing key.pem: %v", err) } - cp, err := certProviderByCertMode("manual", dir, hostname, "", "", "") + cp, err := certProviderByCertMode("manual", dir, hostname, false, "", "", "") if err != nil { t.Fatal(err) } @@ -174,25 +174,25 @@ func TestGCPCertMode(t *testing.T) { dir := t.TempDir() // Missing EAB credentials - _, err := certProviderByCertMode("gcp", dir, "test.example.com", "", "", "test@example.com") + _, err := certProviderByCertMode("gcp", dir, "test.example.com", false, "", "", "test@example.com") if err == nil { t.Fatal("expected error when EAB credentials are missing") } // Missing email - _, err = certProviderByCertMode("gcp", dir, "test.example.com", "kid", "dGVzdC1rZXk", "") + _, err = certProviderByCertMode("gcp", dir, "test.example.com", false, "kid", "dGVzdC1rZXk", "") if err == nil { t.Fatal("expected error when email is missing") } // Invalid base64 - _, err = certProviderByCertMode("gcp", dir, "test.example.com", "kid", "not-valid!", "test@example.com") + _, err = certProviderByCertMode("gcp", dir, "test.example.com", false, "kid", "not-valid!", "test@example.com") if err == nil { t.Fatal("expected error for invalid base64") } // Valid base64url (no padding) - cp, err := certProviderByCertMode("gcp", dir, "test.example.com", "kid", "dGVzdC1rZXk", "test@example.com") + cp, err := certProviderByCertMode("gcp", dir, "test.example.com", false, "kid", "dGVzdC1rZXk", "test@example.com") if err != nil { t.Fatalf("base64url: %v", err) } @@ -201,7 +201,7 @@ func TestGCPCertMode(t *testing.T) { } // Valid standard base64 (with padding, gcloud format) - cp, err = certProviderByCertMode("gcp", dir, "test.example.com", "kid", "dGVzdC1rZXk=", "test@example.com") + cp, err = certProviderByCertMode("gcp", dir, "test.example.com", false, "kid", "dGVzdC1rZXk=", "test@example.com") if err != nil { t.Fatalf("base64: %v", err) } diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index ec59c7264..865dbab42 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -6,10 +6,9 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa github.com/axiomhq/hyperloglog from tailscale.com/derp/derpserver github.com/beorn7/perks/quantile from github.com/prometheus/client_golang/prometheus 💣 github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus - github.com/coder/websocket from tailscale.com/cmd/derper+ + github.com/coder/websocket from tailscale.com/derp/derpserver+ github.com/coder/websocket/internal/errd from github.com/coder/websocket github.com/coder/websocket/internal/util from github.com/coder/websocket - github.com/coder/websocket/internal/xsync from github.com/coder/websocket github.com/creachadair/msync/throttle from github.com/tailscale/setec/client/setec W 💣 github.com/dblohm7/wingoes from tailscale.com/util/winutil github.com/dgryski/go-metro from github.com/axiomhq/hyperloglog @@ -20,6 +19,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/v1 from tailscale.com/net/routecheck+ + 💣 github.com/go4org/hashtriemap from tailscale.com/derp/derpserver github.com/golang/groupcache/lru from tailscale.com/net/dnscache github.com/hdevalence/ed25519consensus from tailscale.com/tka L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon @@ -90,6 +91,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa tailscale.com/envknob from tailscale.com/client/local+ tailscale.com/feature from tailscale.com/tsweb+ tailscale.com/feature/buildfeatures from tailscale.com/feature+ + tailscale.com/feature/serviceclientprefs/serviceclient from tailscale.com/client/local tailscale.com/health from tailscale.com/net/tlsdial+ tailscale.com/hostinfo from tailscale.com/net/netmon+ tailscale.com/ipn from tailscale.com/client/local @@ -106,19 +108,23 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa 💣 tailscale.com/net/netns from tailscale.com/derp/derphttp tailscale.com/net/netutil from tailscale.com/client/local tailscale.com/net/netx from tailscale.com/net/dnscache+ + tailscale.com/net/routecheck from tailscale.com/client/local + tailscale.com/net/routecheck/peernode from tailscale.com/net/routecheck tailscale.com/net/sockstats from tailscale.com/derp/derphttp tailscale.com/net/stun from tailscale.com/net/stunserver tailscale.com/net/stunserver from tailscale.com/cmd/derper L tailscale.com/net/tcpinfo from tailscale.com/derp/derpserver tailscale.com/net/tlsdial from tailscale.com/derp/derphttp tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/net/routecheck tailscale.com/net/tsaddr from tailscale.com/ipn+ tailscale.com/net/udprelay/status from tailscale.com/client/local - tailscale.com/net/wsconn from tailscale.com/cmd/derper + tailscale.com/net/wsconn from tailscale.com/derp/derpserver tailscale.com/paths from tailscale.com/client/local 💣 tailscale.com/safesocket from tailscale.com/client/local tailscale.com/syncs from tailscale.com/cmd/derper+ tailscale.com/tailcfg from tailscale.com/client/local+ + tailscale.com/tempfork/acme from tailscale.com/cmd/derper tailscale.com/tka from tailscale.com/client/local+ tailscale.com/tsconst from tailscale.com/net/netmon+ tailscale.com/tstime from tailscale.com/derp+ @@ -134,7 +140,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa tailscale.com/types/key from tailscale.com/client/local+ tailscale.com/types/lazy from tailscale.com/version+ tailscale.com/types/logger from tailscale.com/cmd/derper+ - tailscale.com/types/netmap from tailscale.com/ipn + tailscale.com/types/netmap from tailscale.com/ipn+ tailscale.com/types/opt from tailscale.com/envknob+ tailscale.com/types/persist from tailscale.com/ipn+ tailscale.com/types/preftype from tailscale.com/ipn @@ -163,7 +169,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa tailscale.com/util/syspolicy/pkey from tailscale.com/ipn+ tailscale.com/util/syspolicy/policyclient from tailscale.com/ipn tailscale.com/util/syspolicy/ptype from tailscale.com/util/syspolicy/policyclient+ - tailscale.com/util/syspolicy/setting from tailscale.com/client/local + tailscale.com/util/syspolicy/setting from tailscale.com/client/local+ tailscale.com/util/testenv from tailscale.com/net/bakedroots+ tailscale.com/util/usermetric from tailscale.com/health tailscale.com/util/vizerror from tailscale.com/tailcfg+ @@ -243,22 +249,22 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ - crypto/internal/fips140 from crypto/internal/fips140/aes+ + crypto/internal/fips140 from crypto/fips140+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ crypto/internal/fips140/alias from crypto/cipher+ crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ - crypto/internal/fips140/check from crypto/internal/fips140/aes+ - crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/check from crypto/fips140+ + crypto/internal/fips140/drbg from crypto/hpke+ crypto/internal/fips140/ecdh from crypto/ecdh crypto/internal/fips140/ecdsa from crypto/ecdsa crypto/internal/fips140/ed25519 from crypto/ed25519 crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 crypto/internal/fips140/edwards25519/field from crypto/ecdh+ - crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ + crypto/internal/fips140/hkdf from crypto/hkdf+ crypto/internal/fips140/hmac from crypto/hmac+ crypto/internal/fips140/mlkem from crypto/mlkem - crypto/internal/fips140/nistec from crypto/elliptic+ + crypto/internal/fips140/nistec from crypto/ecdsa+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ @@ -309,8 +315,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa go/token from google.golang.org/protobuf/internal/strs hash from crypto+ hash/crc32 from compress/gzip+ - hash/fnv from google.golang.org/protobuf/internal/detrand - hash/maphash from go4.org/mem + hash/fnv from google.golang.org/protobuf/internal/detrand+ + hash/maphash from go4.org/mem+ html from net/http/pprof+ html/template from tailscale.com/cmd/derper+ internal/abi from crypto/x509/internal/macos+ @@ -324,13 +330,13 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa internal/filepathlite from os+ internal/fmtsort from fmt+ internal/goarch from crypto/internal/fips140deps/cpu+ - internal/godebug from crypto/internal/fips140deps/godebug+ + internal/godebug from crypto/ed25519+ internal/godebugs from internal/godebug+ internal/goexperiment from net/http/pprof+ internal/goos from crypto/x509+ internal/msan from internal/runtime/maps+ internal/nettrace from net+ - internal/oserror from io/fs+ + internal/oserror from internal/syscall/windows+ internal/poll from net+ internal/profile from net/http/pprof internal/profilerecord from runtime+ @@ -340,9 +346,9 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa internal/runtime/atomic from internal/runtime/exithook+ L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime - internal/runtime/gc from runtime+ + internal/runtime/gc from internal/runtime/gc/scan+ internal/runtime/gc/scan from runtime - internal/runtime/maps from reflect+ + internal/runtime/maps from hash/maphash+ internal/runtime/math from internal/runtime/maps+ internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ @@ -356,7 +362,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa internal/synctest from sync internal/syscall/execenv from os+ LD internal/syscall/unix from crypto/internal/sysrand+ - W internal/syscall/windows from crypto/internal/sysrand+ + W internal/syscall/windows from crypto/internal/fips140deps/time+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 429aff361..bae4d9f01 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -62,10 +62,11 @@ var ( configPath = flag.String("c", "", "config file path") certMode = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt, gcp") certDir = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store ACME (e.g. LetsEncrypt) certs, if addr's port is :443") - hostname = flag.String("hostname", "derp.tailscale.com", "TLS host name for certs, if addr's port is :443. When --certmode=manual, this can be an IP address to avoid SNI checks") + hostname = flag.String("hostname", "derp.tailscale.com", "TLS host name for certs, if addr's port is :443. It can be an IP address when --certmode=manual (to avoid SNI checks) or when --acme-ip-certs is set (to run an IP-only server with no hostname cert)") acmeEABKid = flag.String("acme-eab-kid", "", "ACME External Account Binding (EAB) Key ID (required for --certmode=gcp)") acmeEABKey = flag.String("acme-eab-key", "", "ACME External Account Binding (EAB) HMAC key, base64-encoded (required for --certmode=gcp)") acmeEmail = flag.String("acme-email", "", "ACME account contact email address (required for --certmode=gcp, optional for letsencrypt)") + acmeIPCerts = flag.Bool("acme-ip-certs", false, "whether to serve LetsEncrypt certs for the server's IP addresses: when a client connects by IP address (sending no TLS SNI, or an IP address SNI matching the connection's destination IP), get and serve a LetsEncrypt cert for that IP, using the short-lived (~6 day) ACME certificate profile. This works for both IPv4 and IPv6 with no per-address configuration. It requires --certmode=letsencrypt and the ACME server must be able to reach port 80 at each such IP for the HTTP-01 challenge.") runSTUN = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.") runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.") flagHome = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to") @@ -87,8 +88,7 @@ var ( acceptConnLimit = flag.Float64("accept-connection-limit", math.Inf(+1), "rate limit for accepting new connection") acceptConnBurst = flag.Int("accept-connection-burst", math.MaxInt, "burst limit for accepting new connection") - perClientRateLimit = flag.Uint("per-client-rate-limit", 0, "per-client receive rate limit in bytes/sec; 0 means unlimited. Mesh peers are exempt.") - perClientRateBurst = flag.Uint("per-client-rate-burst", 0, "per-client receive rate burst in bytes; 0 defaults to 2x the rate limit (only relevant when using nonzero --per-client-rate-limit)") + rateConfigPath = flag.String("rate-config", "", "if non-empty, path to JSON rate limit config file. Rate limiting is experimental and subject to change. Configuration is reloaded on SIGHUP.") // tcpKeepAlive is intentionally long, to reduce battery cost. There is an L7 keepalive on a higher frequency schedule. tcpKeepAlive = flag.Duration("tcp-keepalive-time", 10*time.Minute, "TCP keepalive time") @@ -195,12 +195,11 @@ func main() { s.SetVerifyClientURL(*verifyClientURL) s.SetVerifyClientURLFailOpen(*verifyFailOpen) s.SetTCPWriteTimeout(*tcpWriteTimeout) - if *perClientRateLimit > 0 { - burst := *perClientRateBurst - if burst < 1 { - burst = *perClientRateLimit * 2 + if *rateConfigPath != "" { + if err := s.LoadAndApplyRateConfig(*rateConfigPath); err != nil { + log.Fatalf("derper: loading rate config: %v", err) } - s.SetPerClientRateLimit(*perClientRateLimit, burst) + go watchRateConfig(ctx, s, *rateConfigPath) } var meshKey string @@ -254,7 +253,7 @@ func main() { if err := startMesh(s); err != nil { log.Fatalf("startMesh: %v", err) } - expvar.Publish("derp", s.ExpVar()) + expvar.Publish("derp", s.ExpVar(*rateConfigPath != "")) handleHome, ok := getHomeHandler(*flagHome) if !ok { @@ -264,7 +263,7 @@ func main() { mux := http.NewServeMux() if *runDERP { derpHandler := derpserver.Handler(s) - derpHandler = addWebSocketSupport(s, derpHandler) + derpHandler = derpserver.AddWebSocketSupport(s, derpHandler) mux.Handle("/derp", derpHandler) } else { mux.Handle("/derp", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -351,20 +350,12 @@ func main() { if serveTLS { log.Printf("derper: serving on %s with TLS", *addr) var certManager certProvider - certManager, err = certProviderByCertMode(*certMode, *certDir, *hostname, *acmeEABKid, *acmeEABKey, *acmeEmail) + certManager, err = certProviderByCertMode(*certMode, *certDir, *hostname, *acmeIPCerts, *acmeEABKid, *acmeEABKey, *acmeEmail) if err != nil { log.Fatalf("derper: can not start cert provider: %v", err) } httpsrv.TLSConfig = certManager.TLSConfig() - getCert := httpsrv.TLSConfig.GetCertificate - httpsrv.TLSConfig.GetCertificate = func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { - cert, err := getCert(hi) - if err != nil { - return nil, err - } - cert.Certificate = append(cert.Certificate, s.MetaCert()) - return cert, nil - } + s.ModifyTLSConfigToAddMetaCert(httpsrv.TLSConfig) // Disable TLS 1.0 and 1.1, which are obsolete and have security issues. httpsrv.TLSConfig.MinVersion = tls.VersionTLS12 httpsrv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -436,6 +427,27 @@ func main() { } } +// watchRateConfig listens for SIGHUP signals and reloads the rate config +// file on each signal, applying it to the server. It returns when ctx is done. +func watchRateConfig(ctx context.Context, s *derpserver.Server, path string) { + sighup := make(chan os.Signal, 1) + signal.Notify(sighup, syscall.SIGHUP) + defer signal.Stop(sighup) + for { + select { + case <-ctx.Done(): + return + case <-sighup: + log.Printf("derper: received SIGHUP, reloading rate config from %s", path) + if err := s.LoadAndApplyRateConfig(path); err != nil { + log.Printf("derper: rate config reload failed: %v", err) + continue + } + log.Printf("derper: rate config reloaded successfully") + } + } +} + var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`) func prodAutocertHostPolicy(_ context.Context, host string) error { diff --git a/cmd/derper/ipcert.go b/cmd/derper/ipcert.go new file mode 100644 index 000000000..b3b4192de --- /dev/null +++ b/cmd/derper/ipcert.go @@ -0,0 +1,497 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/netip" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "tailscale.com/atomicfile" + "tailscale.com/tailcfg" + "tailscale.com/tempfork/acme" +) + +// shortlivedProfile is the ACME certificate profile required by +// LetsEncrypt for IP address certificates. Certificates issued under +// it are valid for about six days. +// See https://letsencrypt.org/docs/profiles/. +const shortlivedProfile = "shortlived" + +// ipCertManager is a certProvider that obtains and renews LetsEncrypt +// TLS certificates for the server's IP addresses on demand, using the +// short-lived ACME certificate profile and the HTTP-01 challenge +// served on the derper's plaintext HTTP port. +// +// Clients connecting to an IP address usually send no SNI, so the +// requested IP address is taken from the TCP connection's local +// address. That works for however many IPv4 and IPv6 addresses the +// server has, with no configuration. Clients that do send an IP +// address in the SNI get a certificate only if it matches the +// connection's local address, so a client can never make us request a +// certificate for an address that isn't ours. +// +// Connections with a DNS name in the SNI are passed through to the +// optional next provider (the regular autocert manager for the +// --hostname certificate), if any. +type ipCertManager struct { + certDir string + email string // optional ACME account contact + client *acme.Client + next certProvider // provider for DNS hostname connections, or nil + nextTLS *tls.Config // next.TLSConfig(), or nil + + mu sync.Mutex + certs map[netip.Addr]*ipCertEntry + tokens map[string]string // HTTP-01 challenge URL path => response body +} + +// ipCertEntry is the issuance state for one IP address. +// All fields are guarded by ipCertManager.mu. +type ipCertEntry struct { + cert *tls.Certificate // current cert with Leaf set, or nil if not yet issued + + flight chan struct{} // non-nil while an issuance is running; closed when it finishes + flightErr error // result of the last finished issuance + nextAttempt time.Time // earliest time of the next issuance attempt, after a failure + retryDelay time.Duration // backoff to apply after the next failure +} + +// newIPCertManager returns an ipCertManager storing its ACME account +// key and issued certificates in certdir. +// +// If directoryURL is empty, the LetsEncrypt production directory is +// used; tests point it at a fake ACME server. If next is non-nil, +// connections with a DNS name in the SNI are served by it. +func newIPCertManager(certdir, email, directoryURL string, next certProvider) (*ipCertManager, error) { + if err := os.MkdirAll(certdir, 0700); err != nil { + return nil, err + } + accountKey, err := loadOrCreateAccountKey(filepath.Join(certdir, "acme-account.key")) + if err != nil { + return nil, fmt.Errorf("ACME account key: %w", err) + } + m := &ipCertManager{ + certDir: certdir, + email: email, + client: &acme.Client{ + Key: accountKey, + DirectoryURL: directoryURL, + UserAgent: "tailscale-derper", + }, + next: next, + certs: make(map[netip.Addr]*ipCertEntry), + tokens: make(map[string]string), + } + if next != nil { + m.nextTLS = next.TLSConfig() + } + go m.renewLoop() + return m, nil +} + +func loadOrCreateAccountKey(path string) (*ecdsa.PrivateKey, error) { + if pemBytes, err := os.ReadFile(path); err == nil { + block, _ := pem.Decode(pemBytes) + if block == nil { + return nil, fmt.Errorf("invalid PEM in %s", path) + } + return x509.ParseECPrivateKey(block.Bytes) + } else if !os.IsNotExist(err) { + return nil, err + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + der, err := x509.MarshalECPrivateKey(key) + if err != nil { + return nil, err + } + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) + if err := atomicfile.WriteFile(path, pemBytes, 0600); err != nil { + return nil, err + } + return key, nil +} + +// certPaths returns the cert and key file paths for ip in m.certDir. +// Colons in IPv6 addresses are replaced with dots to keep the names +// filesystem-safe. +func (m *ipCertManager) certPaths(ip netip.Addr) (crtPath, keyPath string) { + base := strings.ReplaceAll(ip.String(), ":", ".") + return filepath.Join(m.certDir, base+".crt"), filepath.Join(m.certDir, base+".key") +} + +func (m *ipCertManager) TLSConfig() *tls.Config { + var conf *tls.Config + if m.nextTLS != nil { + conf = m.nextTLS.Clone() + } else { + conf = &tls.Config{ + NextProtos: []string{ + "http/1.1", + }, + } + } + conf.GetCertificate = m.getCertificate + return conf +} + +// connLocalIP returns the local (server side) IP address of the +// connection that sent the ClientHello. +func connLocalIP(hi *tls.ClientHelloInfo) (netip.Addr, bool) { + if hi.Conn == nil { + return netip.Addr{}, false + } + ta, ok := hi.Conn.LocalAddr().(*net.TCPAddr) + if !ok { + return netip.Addr{}, false + } + ip := ta.AddrPort().Addr().Unmap() + return ip, ip.IsValid() +} + +func (m *ipCertManager) getCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { + connIP, connIPOK := connLocalIP(hi) + if hi.ServerName != "" { + sniIP, err := netip.ParseAddr(hi.ServerName) + if err != nil { + // The SNI is a DNS name; let the hostname provider handle it. + if m.nextTLS != nil && m.nextTLS.GetCertificate != nil { + return m.nextTLS.GetCertificate(hi) + } + return nil, fmt.Errorf("no certificate for hostname %q; this server only serves IP address certificates", hi.ServerName) + } + if !connIPOK || sniIP.Unmap() != connIP { + return nil, fmt.Errorf("requested certificate for IP %v does not match the connection's IP address", sniIP) + } + } + if !connIPOK { + return nil, errors.New("unable to determine the connection's local IP address") + } + ctx := hi.Context() + if ctx == nil { + ctx = context.Background() + } + return m.certForIP(ctx, connIP) +} + +// certForIP returns the current certificate for ip, obtaining one +// first if there is no unexpired certificate for it. Concurrent +// callers for the same IP share a single issuance. +func (m *ipCertManager) certForIP(ctx context.Context, ip netip.Addr) (*tls.Certificate, error) { + m.mu.Lock() + e := m.entryLocked(ip) + if e.cert != nil && time.Now().Before(e.cert.Leaf.NotAfter) { + defer m.mu.Unlock() + return clipCert(e.cert), nil + } + if e.flight == nil && time.Now().Before(e.nextAttempt) { + m.mu.Unlock() + return nil, fmt.Errorf("cert issuance for %v failed recently; next attempt no earlier than %v", ip, e.nextAttempt.Format(time.RFC3339)) + } + flight := m.startFlightLocked(ip, e) + m.mu.Unlock() + + select { + case <-flight: + case <-ctx.Done(): + return nil, ctx.Err() + } + + m.mu.Lock() + defer m.mu.Unlock() + if e.cert == nil { + return nil, e.flightErr + } + return clipCert(e.cert), nil +} + +func (m *ipCertManager) entryLocked(ip netip.Addr) *ipCertEntry { + e, ok := m.certs[ip] + if !ok { + e = &ipCertEntry{} + m.certs[ip] = e + } + return e +} + +// clipCert returns a shallow copy of cert with a capacity-clamped +// chain so callers can never mutate the manager's long-lived +// certificate. +func clipCert(cert *tls.Certificate) *tls.Certificate { + certCopy := *cert + certCopy.Certificate = slices.Clip(certCopy.Certificate) + return &certCopy +} + +// startFlightLocked starts an issuance for ip if none is running and +// returns a channel that is closed when it finishes. +func (m *ipCertManager) startFlightLocked(ip netip.Addr, e *ipCertEntry) chan struct{} { + if e.flight != nil { + return e.flight + } + done := make(chan struct{}) + e.flight = done + go func() { + err := m.issue(ip) + m.mu.Lock() + defer m.mu.Unlock() + e.flight = nil + e.flightErr = err + if err != nil { + if e.retryDelay == 0 { + e.retryDelay = time.Minute + } + e.nextAttempt = time.Now().Add(e.retryDelay) + e.retryDelay = min(e.retryDelay*2, 30*time.Minute) + log.Printf("derper: acme: getting cert for %v: %v (next attempt in %v)", ip, err, time.Until(e.nextAttempt).Round(time.Second)) + } else { + e.retryDelay = 0 + e.nextAttempt = time.Time{} + } + close(done) + }() + return done +} + +// issue obtains a certificate for ip, preferring a still-fresh one +// cached on disk over a new ACME order. +func (m *ipCertManager) issue(ip netip.Addr) error { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + if cert, err := m.loadCachedCert(ip); err == nil && !certNeedsRenewal(cert.Leaf) { + m.mu.Lock() + m.entryLocked(ip).cert = cert + m.mu.Unlock() + log.Printf("derper: acme: loaded cached cert for %v (expires %v)", ip, cert.Leaf.NotAfter) + return nil + } + return m.obtainCert(ctx, ip) +} + +// loadCachedCert loads a previously issued certificate for ip from +// disk, if present and still valid. +func (m *ipCertManager) loadCachedCert(ip netip.Addr) (*tls.Certificate, error) { + crtPath, keyPath := m.certPaths(ip) + cert, err := tls.LoadX509KeyPair(crtPath, keyPath) + if err != nil { + return nil, err + } + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, err + } + now := time.Now() + if now.Before(leaf.NotBefore) || now.After(leaf.NotAfter) { + return nil, fmt.Errorf("cached cert is expired or not yet valid (NotAfter %v)", leaf.NotAfter) + } + if err := leaf.VerifyHostname(ip.String()); err != nil { + return nil, err + } + cert.Leaf = leaf + return &cert, nil +} + +// HTTPHandler returns a handler serving HTTP-01 challenge responses on +// the derper's plaintext HTTP port, sending all other requests to the +// next provider's handler, if any, and otherwise to fallback. +func (m *ipCertManager) HTTPHandler(fallback http.Handler) http.Handler { + if m.next != nil { + fallback = m.next.HTTPHandler(fallback) + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, "/.well-known/acme-challenge/") { + fallback.ServeHTTP(w, r) + return + } + m.mu.Lock() + response, ok := m.tokens[r.URL.Path] + m.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain") + io.WriteString(w, response) + }) +} + +func (m *ipCertManager) setToken(path, response string) { + m.mu.Lock() + defer m.mu.Unlock() + m.tokens[path] = response +} + +func (m *ipCertManager) deleteToken(path string) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.tokens, path) +} + +// certNeedsRenewal reports whether leaf has less than a third of its +// lifetime remaining. LetsEncrypt short-lived certs are valid for +// about six days, so renewal happens roughly every four. +func certNeedsRenewal(leaf *x509.Certificate) bool { + total := leaf.NotAfter.Sub(leaf.NotBefore) + return time.Until(leaf.NotAfter) < total/3 +} + +// renewLoop runs for the lifetime of the process, renewing each issued +// certificate as it approaches expiry. +func (m *ipCertManager) renewLoop() { + for { + time.Sleep(time.Hour) + m.mu.Lock() + now := time.Now() + for ip, e := range m.certs { + if e.cert != nil && certNeedsRenewal(e.cert.Leaf) && e.flight == nil && now.After(e.nextAttempt) { + m.startFlightLocked(ip, e) + } + } + m.mu.Unlock() + } +} + +// obtainCert does one ACME issuance flow for ip: it registers the +// account if needed, orders a short-lived profile certificate for the +// IP address identifier, fulfills the HTTP-01 challenges, and installs +// and caches the issued certificate. +func (m *ipCertManager) obtainCert(ctx context.Context, ip netip.Addr) error { + ipStr := ip.String() + + var contact []string + if m.email != "" { + contact = []string{"mailto:" + m.email} + } + _, err := m.client.Register(ctx, &acme.Account{Contact: contact}, acme.AcceptTOS) + if err != nil && !errors.Is(err, acme.ErrAccountAlreadyExists) { + return fmt.Errorf("register: %w", err) + } + + order, err := m.client.AuthorizeOrder(ctx, acme.IPIDs(ipStr), acme.WithOrderProfile(shortlivedProfile)) + if err != nil { + return fmt.Errorf("new order: %w", err) + } + for _, authzURL := range order.AuthzURLs { + if err := m.fulfillAuthz(ctx, authzURL); err != nil { + return err + } + } + order, err = m.client.WaitOrder(ctx, order.URI) + if err != nil { + return fmt.Errorf("waiting for order: %w", err) + } + + certKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return err + } + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{ + IPAddresses: []net.IP{ip.AsSlice()}, + }, certKey) + if err != nil { + return err + } + der, _, err := m.client.CreateOrderCert(ctx, order.FinalizeURL, csr, true) + if err != nil { + return fmt.Errorf("finalizing order: %w", err) + } + leaf, err := x509.ParseCertificate(der[0]) + if err != nil { + return fmt.Errorf("parsing issued cert: %w", err) + } + if err := leaf.VerifyHostname(ipStr); err != nil { + return fmt.Errorf("issued cert: %w", err) + } + + keyDER, err := x509.MarshalECPrivateKey(certKey) + if err != nil { + return err + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + var chainPEM []byte + for _, b := range der { + chainPEM = append(chainPEM, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: b})...) + } + crtPath, keyPath := m.certPaths(ip) + if err := atomicfile.WriteFile(keyPath, keyPEM, 0600); err != nil { + return err + } + if err := atomicfile.WriteFile(crtPath, chainPEM, 0644); err != nil { + return err + } + + m.mu.Lock() + m.entryLocked(ip).cert = &tls.Certificate{ + Certificate: der, + PrivateKey: certKey, + Leaf: leaf, + } + m.mu.Unlock() + + dn := &tailcfg.DERPNode{ + Name: "custom", + RegionID: 900, + HostName: ipStr, + } + dnJSON, _ := json.Marshal(dn) + log.Printf("derper: acme: got cert for %v (expires %v). Configure it in DERPMap using (https://tailscale.com/s/custom-derp):\n %s", ip, leaf.NotAfter, dnJSON) + return nil +} + +// fulfillAuthz completes the HTTP-01 challenge for one authorization, +// if it is still pending. +func (m *ipCertManager) fulfillAuthz(ctx context.Context, authzURL string) error { + authz, err := m.client.GetAuthorization(ctx, authzURL) + if err != nil { + return fmt.Errorf("getting authorization: %w", err) + } + if authz.Status != acme.StatusPending { + return nil + } + var challenge *acme.Challenge + for _, c := range authz.Challenges { + if c.Type == "http-01" { + challenge = c + break + } + } + if challenge == nil { + return errors.New("authorization offers no http-01 challenge") + } + response, err := m.client.HTTP01ChallengeResponse(challenge.Token) + if err != nil { + return err + } + path := m.client.HTTP01ChallengePath(challenge.Token) + m.setToken(path, response) + defer m.deleteToken(path) + if _, err := m.client.Accept(ctx, challenge); err != nil { + return fmt.Errorf("accepting challenge: %w", err) + } + if _, err := m.client.WaitAuthorization(ctx, authz.URI); err != nil { + return fmt.Errorf("waiting for authorization: %w", err) + } + return nil +} diff --git a/cmd/derper/ipcert_test.go b/cmd/derper/ipcert_test.go new file mode 100644 index 000000000..b3deb3fdd --- /dev/null +++ b/cmd/derper/ipcert_test.go @@ -0,0 +1,486 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// fakeIPACME is a minimal fake ACME (RFC 8555) certificate authority +// for testing ipCertManager. It implements just enough of the protocol +// for http-01 order flows with IP address identifiers and the +// "shortlived" profile: one order at a time, no JWS signature +// verification, no nonce tracking. +type fakeIPACME struct { + t *testing.T + srv *httptest.Server + challengeBase string // base URL at which http-01 challenges are fetched + + caKey *ecdsa.PrivateKey + caCert *x509.Certificate + + mu sync.Mutex + orders int // number of orders created + gotProfile string // profile of the last order + gotIDType string // identifier type of the last order + gotIDValue string // identifier value of the last order + authzStatus string // "pending" or "valid" + orderStatus string // "pending", "ready", or "valid" + token string + certPEM []byte +} + +func newFakeIPACME(t *testing.T) *fakeIPACME { + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + caTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "fake IP ACME root"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + caCert, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + f := &fakeIPACME{ + t: t, + caKey: caKey, + caCert: caCert, + } + f.srv = httptest.NewServer(http.HandlerFunc(f.serveHTTP)) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeIPACME) directoryURL() string { return f.srv.URL + "/directory" } + +func (f *fakeIPACME) numOrders() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.orders +} + +func (f *fakeIPACME) serveHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Replay-Nonce", "test-nonce") + writeJSON := func(v any) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) + } + orderJSON := func() any { + return map[string]any{ + "status": f.orderStatus, + "identifiers": []map[string]string{{"type": f.gotIDType, "value": f.gotIDValue}}, + "authorizations": []string{f.srv.URL + "/authz/1"}, + "finalize": f.srv.URL + "/finalize/1", + "certificate": f.srv.URL + "/cert/1", + } + } + switch { + case r.URL.Path == "/directory": + writeJSON(map[string]any{ + "newNonce": f.srv.URL + "/new-nonce", + "newAccount": f.srv.URL + "/new-account", + "newOrder": f.srv.URL + "/new-order", + "revokeCert": f.srv.URL + "/revoke-cert", + "keyChange": f.srv.URL + "/key-change", + "meta": map[string]any{ + "profiles": map[string]string{ + "classic": "the default profile", + shortlivedProfile: "six day certificates", + }, + }, + }) + case r.URL.Path == "/new-nonce": + // The Replay-Nonce header was already set above. + case r.URL.Path == "/new-account": + w.Header().Set("Location", f.srv.URL+"/account/1") + w.WriteHeader(http.StatusCreated) + writeJSON(map[string]any{"status": "valid"}) + case r.URL.Path == "/new-order": + var req struct { + Identifiers []struct{ Type, Value string } `json:"identifiers"` + Profile string `json:"profile"` + } + if err := decodeJWSPayload(r, &req); err != nil { + f.t.Errorf("new-order payload: %v", err) + } + f.mu.Lock() + f.orders++ + f.gotProfile = req.Profile + if len(req.Identifiers) == 1 { + f.gotIDType = req.Identifiers[0].Type + f.gotIDValue = req.Identifiers[0].Value + } + f.authzStatus = "pending" + f.orderStatus = "pending" + f.token = fmt.Sprintf("tok-%d", f.orders) + f.mu.Unlock() + w.Header().Set("Location", f.srv.URL+"/order/1") + w.WriteHeader(http.StatusCreated) + writeJSON(orderJSON()) + case r.URL.Path == "/authz/1": + f.mu.Lock() + defer f.mu.Unlock() + writeJSON(map[string]any{ + "status": f.authzStatus, + "identifier": map[string]string{"type": f.gotIDType, "value": f.gotIDValue}, + "challenges": []map[string]string{{ + "type": "http-01", + "url": f.srv.URL + "/chal/1", + "token": f.token, + "status": f.authzStatus, + }}, + }) + case r.URL.Path == "/chal/1": + if err := f.validateChallenge(); err != nil { + f.t.Errorf("challenge validation: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + f.mu.Lock() + f.authzStatus = "valid" + f.orderStatus = "ready" + f.mu.Unlock() + writeJSON(map[string]any{"type": "http-01", "status": "valid", "token": f.token}) + case r.URL.Path == "/order/1": + f.mu.Lock() + defer f.mu.Unlock() + w.Header().Set("Location", f.srv.URL+"/order/1") + writeJSON(orderJSON()) + case r.URL.Path == "/finalize/1": + var req struct { + CSR string `json:"csr"` + } + if err := decodeJWSPayload(r, &req); err != nil { + f.t.Errorf("finalize payload: %v", err) + } + if err := f.issueCert(req.CSR); err != nil { + f.t.Errorf("issuing cert: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + f.mu.Lock() + f.orderStatus = "valid" + f.mu.Unlock() + w.Header().Set("Location", f.srv.URL+"/order/1") + writeJSON(orderJSON()) + case r.URL.Path == "/cert/1": + f.mu.Lock() + defer f.mu.Unlock() + w.Header().Set("Content-Type", "application/pem-certificate-chain") + w.Write(f.certPEM) + default: + f.t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + http.NotFound(w, r) + } +} + +// validateChallenge fetches the http-01 challenge response from the +// server under test, standing in for the CA dialing port 80 at the IP +// address being validated. +func (f *fakeIPACME) validateChallenge() error { + f.mu.Lock() + token := f.token + base := f.challengeBase + f.mu.Unlock() + res, err := http.Get(base + "/.well-known/acme-challenge/" + token) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("status %d", res.StatusCode) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + if !strings.HasPrefix(string(body), token+".") { + return fmt.Errorf("challenge response %q does not start with %q", body, token+".") + } + return nil +} + +// issueCert signs a certificate for the CSR (base64url DER), valid for +// six days like a LetsEncrypt shortlived profile certificate. +func (f *fakeIPACME) issueCert(csrB64 string) error { + csrDER, err := base64.RawURLEncoding.DecodeString(csrB64) + if err != nil { + return err + } + csr, err := x509.ParseCertificateRequest(csrDER) + if err != nil { + return err + } + if len(csr.IPAddresses) != 1 { + return fmt.Errorf("CSR has %d IP addresses; want 1", len(csr.IPAddresses)) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + IPAddresses: csr.IPAddresses, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(6 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, f.caCert, csr.PublicKey, f.caKey) + if err != nil { + return err + } + var buf []byte + buf = append(buf, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) + buf = append(buf, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: f.caCert.Raw})...) + f.mu.Lock() + f.certPEM = buf + f.mu.Unlock() + return nil +} + +// decodeJWSPayload decodes the payload of a JWS-encoded ACME request +// without verifying its signature. +func decodeJWSPayload(r *http.Request, v any) error { + var req struct{ Payload string } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return err + } + b, err := base64.RawURLEncoding.DecodeString(req.Payload) + if err != nil { + return err + } + return json.Unmarshal(b, v) +} + +// ipConn is a stub net.Conn whose LocalAddr is the given IP, standing +// in for the accepted TLS connection whose destination address decides +// which certificate to serve. +type ipConn struct { + net.Conn + local net.Addr +} + +func (c ipConn) LocalAddr() net.Addr { return c.local } + +// helloFor returns a ClientHelloInfo as ipCertManager.getCertificate +// would see it for a connection to localIP with the given SNI value. +func helloFor(t *testing.T, localIP, sni string) *tls.ClientHelloInfo { + t.Helper() + ip := net.ParseIP(localIP) + if ip == nil { + t.Fatalf("bad IP %q", localIP) + } + return &tls.ClientHelloInfo{ + ServerName: sni, + Conn: ipConn{local: &net.TCPAddr{IP: ip, Port: 443}}, + } +} + +// stubCertProvider is a certProvider returning a fixed certificate, +// standing in for the autocert manager handling DNS hostname +// connections. +type stubCertProvider struct { + cert *tls.Certificate +} + +func (p *stubCertProvider) TLSConfig() *tls.Config { + return &tls.Config{ + GetCertificate: func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { + return p.cert, nil + }, + } +} + +func (p *stubCertProvider) HTTPHandler(fallback http.Handler) http.Handler { return fallback } + +// TestIPCertManager exercises the on-demand issuance flow of +// ipCertManager against a fake ACME CA: certs are ordered for whatever +// IP address a connection arrives on (IPv4 and IPv6), with the +// shortlived profile, answering the http-01 challenge, and reusing the +// on-disk cache. +func TestIPCertManager(t *testing.T) { + const ip4 = "203.0.113.7" + const ip6 = "2001:db8::7" + dir := t.TempDir() + ca := newFakeIPACME(t) + + m, err := newIPCertManager(dir, "test@example.com", ca.directoryURL(), nil) + if err != nil { + t.Fatal(err) + } + + // Serve the manager's HTTP-01 challenge handler like derper's port + // 80 listener does. + challengeSrv := httptest.NewServer(m.HTTPHandler(http.NotFoundHandler())) + defer challengeSrv.Close() + ca.challengeBase = challengeSrv.URL + + // A connection to the IPv4 address with no SNI mints a cert for it. + cert, err := m.getCertificate(helloFor(t, ip4, "")) + if err != nil { + t.Fatal(err) + } + if err := cert.Leaf.VerifyHostname(ip4); err != nil { + t.Errorf("issued cert not valid for %v: %v", ip4, err) + } + if got, want := ca.gotProfile, shortlivedProfile; got != want { + t.Errorf("order profile = %q; want %q", got, want) + } + if ca.gotIDType != "ip" || ca.gotIDValue != ip4 { + t.Errorf("order identifier = %q %q; want %q %q", ca.gotIDType, ca.gotIDValue, "ip", ip4) + } + if n := ca.numOrders(); n != 1 { + t.Errorf("orders created = %d; want 1", n) + } + + // A connection to the IPv6 address mints a second, separate cert. + cert6, err := m.getCertificate(helloFor(t, ip6, "")) + if err != nil { + t.Fatal(err) + } + if err := cert6.Leaf.VerifyHostname(ip6); err != nil { + t.Errorf("issued cert not valid for %v: %v", ip6, err) + } + if ca.gotIDType != "ip" || ca.gotIDValue != ip6 { + t.Errorf("order identifier = %q %q; want %q %q", ca.gotIDType, ca.gotIDValue, "ip", ip6) + } + if n := ca.numOrders(); n != 2 { + t.Errorf("orders created = %d; want 2", n) + } + + // An SNI containing the connection's own IP address is served from + // the cache. + if _, err := m.getCertificate(helloFor(t, ip4, ip4)); err != nil { + t.Errorf("getCertificate with matching IP SNI: %v", err) + } + if n := ca.numOrders(); n != 2 { + t.Errorf("orders created after cached hit = %d; want 2", n) + } + + // An SNI naming some other IP address is rejected. + if _, err := m.getCertificate(helloFor(t, ip4, "203.0.113.8")); err == nil { + t.Error("getCertificate with mismatched IP SNI succeeded; want error") + } + + // A DNS name SNI has no provider to go to here. + if _, err := m.getCertificate(helloFor(t, ip4, "derp.example.com")); err == nil { + t.Error("getCertificate with DNS SNI and no next provider succeeded; want error") + } + + // A second manager over the same cert directory must use the + // on-disk cache rather than creating more orders. + m2, err := newIPCertManager(dir, "test@example.com", ca.directoryURL(), nil) + if err != nil { + t.Fatal(err) + } + cachedCert, err := m2.getCertificate(helloFor(t, ip4, "")) + if err != nil { + t.Fatal(err) + } + if err := cachedCert.Leaf.VerifyHostname(ip4); err != nil { + t.Errorf("cached cert not valid for %v: %v", ip4, err) + } + if n := ca.numOrders(); n != 2 { + t.Errorf("orders created after cache reuse = %d; want 2", n) + } + + // Non-challenge requests go to the fallback handler. + rec := httptest.NewRecorder() + m.HTTPHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "fallback") + })).ServeHTTP(rec, httptest.NewRequest("GET", "/other", nil)) + if got := rec.Body.String(); got != "fallback" { + t.Errorf("fallback body = %q; want %q", got, "fallback") + } +} + +// TestIPCertManagerNextProvider verifies that connections with a DNS +// name in the SNI are passed through to the next provider. +func TestIPCertManagerNextProvider(t *testing.T) { + dir := t.TempDir() + ca := newFakeIPACME(t) + stubCert := &tls.Certificate{} + m, err := newIPCertManager(dir, "", ca.directoryURL(), &stubCertProvider{cert: stubCert}) + if err != nil { + t.Fatal(err) + } + got, err := m.getCertificate(helloFor(t, "203.0.113.7", "derp.example.com")) + if err != nil { + t.Fatal(err) + } + if got != stubCert { + t.Errorf("DNS SNI returned %p; want the next provider's cert %p", got, stubCert) + } + if n := ca.numOrders(); n != 0 { + t.Errorf("orders created = %d; want 0", n) + } +} + +// TestCertModeIPCertsGating verifies the flag validation around +// --acme-ip-certs and IP address hostnames. +func TestCertModeIPCertsGating(t *testing.T) { + tests := []struct { + name string + mode string + host string + ipCerts bool + wantErr string // or empty to expect success + }{ + {"letsencrypt_ip_no_flag", "letsencrypt", "1.2.3.4", false, "--acme-ip-certs"}, + {"gcp_ip", "gcp", "1.2.3.4", false, "--certmode=gcp requires --hostname to be a DNS name"}, + {"gcp_flag", "gcp", "1.2.3.4", true, "--acme-ip-certs requires --certmode=letsencrypt"}, + {"manual_flag", "manual", "1.2.3.4", true, "--acme-ip-certs requires --certmode=letsencrypt"}, + {"letsencrypt_ip_flag", "letsencrypt", "1.2.3.4", true, ""}, + {"letsencrypt_hostname_flag", "letsencrypt", "derp.example.com", true, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cp, err := certProviderByCertMode(tt.mode, t.TempDir(), tt.host, tt.ipCerts, "", "", "") + if tt.wantErr == "" { + if err != nil { + t.Fatalf("certProviderByCertMode(%q, %q, ipCerts=%v) = %v; want success", tt.mode, tt.host, tt.ipCerts, err) + } + m, ok := cp.(*ipCertManager) + if !ok { + t.Fatalf("provider type = %T; want *ipCertManager", cp) + } + wantNext := net.ParseIP(tt.host) == nil + if gotNext := m.next != nil; gotNext != wantNext { + t.Errorf("has next provider = %v; want %v", gotNext, wantNext) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("certProviderByCertMode(%q, %q, ipCerts=%v) error = %v; want contains %q", + tt.mode, tt.host, tt.ipCerts, err, tt.wantErr) + } + }) + } +} diff --git a/cmd/fbstatus/fbstatus.go b/cmd/fbstatus/fbstatus.go new file mode 100644 index 000000000..af17e7944 --- /dev/null +++ b/cmd/fbstatus/fbstatus.go @@ -0,0 +1,821 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +// fbstatus is a Linux framebuffer status display for the Tailscale +// appliance. It draws the Tailscale logo, the tailscaled backend state, +// the device's tailnet IP addresses, and (when the device needs to be +// logged in) a QR code containing the login URL so a user can enroll +// the appliance into a tailnet by pointing their phone camera at the +// screen. +// +// fbstatus accesses the framebuffer via the Linux UAPI in +// include/uapi/linux/fb.h: FBIOGET_VSCREENINFO and FBIOGET_FSCREENINFO +// ioctls plus an mmap of /dev/fb0. Only 32-bit truecolor framebuffers +// (the Raspberry Pi default) are supported. +package main + +import ( + "bytes" + "context" + _ "embed" + "encoding/binary" + "flag" + "fmt" + "image" + "image/color" + "image/draw" + "image/png" + "log" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + "unsafe" + + "github.com/skip2/go-qrcode" + xdraw "golang.org/x/image/draw" + "golang.org/x/image/font" + "golang.org/x/image/font/basicfont" + "golang.org/x/image/math/fixed" + "golang.org/x/sys/unix" + "tailscale.com/client/local" + "tailscale.com/ipn" + "tailscale.com/util/cloudenv" +) + +//go:embed tailscale.png +var tailscalePNG []byte + +// Linux framebuffer ioctl numbers, from include/uapi/linux/fb.h. +const ( + fbioGetVScreenInfo = 0x4600 + fbioGetFScreenInfo = 0x4602 +) + +// Linux VT ioctl numbers and KD_* modes, from include/uapi/linux/kd.h +// and include/uapi/linux/vt.h. +const ( + kdSetMode = 0x4B3A + kdGraphics = 1 + kdText = 0 + vtActivate = 0x5606 + vtWaitActive = 0x5607 +) + +// Byte offsets into the raw fb_var_screeninfo struct returned by +// FBIOGET_VSCREENINFO. All fields we read are little-endian uint32. +const ( + vsOffXres = 0 + vsOffYres = 4 + vsOffBitsPerPixel = 24 + vsOffRedOffset = 32 // start of struct fb_bitfield red + vsOffGreenOffset = 44 // start of struct fb_bitfield green + vsOffBlueOffset = 56 // start of struct fb_bitfield blue +) + +// Byte offsets into the raw fb_fix_screeninfo struct returned by +// FBIOGET_FSCREENINFO. Layout assumes a 64-bit kernel (the gokrazy +// appliance targets — arm64/amd64 — are both 64-bit). smem_start and +// mmio_start are "unsigned long", which is 8 bytes on 64-bit. +const ( + fsOffSmemLen = 24 + fsOffLineLength = 48 +) + +var flagFB = flag.String("fb", "/dev/fb0", "framebuffer device to draw to") + +// noFramebufferReason reports whether this host lacks a usable Linux +// framebuffer, along with a short human-readable explanation. If the +// framebuffer device is missing, or we're running on a cloud (currently +// only AWS) whose instances don't expose one, we return true. +func noFramebufferReason(fbPath string) (string, bool) { + if cloudenv.Get() == cloudenv.AWS { + return "running on AWS (no framebuffer)", true + } + if _, err := os.Stat(fbPath); err != nil { + return fmt.Sprintf("no framebuffer at %s: %v", fbPath, err), true + } + return "", false +} + +func main() { + flag.Parse() + log.SetFlags(log.LstdFlags | log.Lmicroseconds) + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + // Bail out early on cloud VMs that don't ship a framebuffer. We + // still kick off breakglass once DHCP succeeds (otherwise the + // appliance is unreachable — breakglass declares DontStartOnBoot + // and only runs when fbstatus pokes the supervisor), then exit + // 125 so the gokrazy supervisor stops respawning us. See + // https://gokrazy.org/development/process-interface/. + if reason, ok := noFramebufferReason(*flagFB); ok { + log.Printf("%s; starting breakglass after DHCP then exiting 125", reason) + startBreakglassAfterDHCP(&uiState{}) + log.Printf("breakglass started; exiting 125 so gokrazy won't respawn fbstatus") + os.Exit(125) + } + + if restore, err := claimVTGraphics(); err != nil { + log.Printf("could not put VT into graphics mode (fbcon may overdraw): %v", err) + } else { + defer restore() + } + + fb, err := openFramebuffer(*flagFB) + if err != nil { + return fmt.Errorf("open framebuffer: %w", err) + } + defer fb.Close() + log.Printf("framebuffer %s: %dx%d, %d bpp, line=%d, RGB offsets %d/%d/%d", + *flagFB, fb.width, fb.height, fb.bpp, fb.lineLength, + fb.redShift, fb.greenShift, fb.blueShift) + + logo, err := png.Decode(bytes.NewReader(tailscalePNG)) + if err != nil { + return fmt.Errorf("decoding embedded logo: %w", err) + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + var lc local.Client + st := &uiState{fb: fb, logo: logo} + st.updateLAN() + st.render() + + go st.pollLAN(ctx) + go startBreakglassAfterDHCP(st) + go watchKeyboardForConsole(ctx, st) + + for ctx.Err() == nil { + if err := watchBusOnce(ctx, &lc, st); err != nil && ctx.Err() == nil { + log.Printf("ipn watch: %v; retrying in 2s", err) + select { + case <-ctx.Done(): + case <-time.After(2 * time.Second): + } + } + } + return nil +} + +func watchBusOnce(ctx context.Context, lc *local.Client, st *uiState) error { + w, err := lc.WatchIPNBus(ctx, + ipn.NotifyInitialState|ipn.NotifyInitialPrefs|ipn.NotifyInitialStatus) + if err != nil { + return err + } + defer w.Close() + + loginRequested := false + + for ctx.Err() == nil { + n, err := w.Next() + if err != nil { + return err + } + if n.State != nil { + st.state = *n.State + // On a fresh appliance, tailscaled enters NeedsLogin but + // does not generate a login URL until someone asks. Trigger + // an interactive login so the control server sends us a URL + // (and thus a QR code appears on the display). + if *n.State == ipn.NeedsLogin && !loginRequested { + loginRequested = true + go func() { + if err := lc.StartLoginInteractive(ctx); err != nil { + log.Printf("StartLoginInteractive: %v", err) + } + }() + } + } + if n.BrowseToURL != nil { + st.loginURL = *n.BrowseToURL + } + if n.InitialStatus != nil { + st.ips = append(st.ips[:0], n.InitialStatus.TailscaleIPs...) + } + if n.SelfChange != nil { + st.ips = st.ips[:0] + for _, p := range n.SelfChange.Addresses { + st.ips = append(st.ips, p.Addr()) + } + } + st.render() + } + return ctx.Err() +} + +// updateLAN scans network interfaces for a non-loopback interface with a +// hardware address, updating st.lanIP and st.lanMAC. Shows the MAC even +// if DHCP hasn't assigned an IP yet. +func (st *uiState) updateLAN() { + ifaces, err := net.Interfaces() + if err != nil { + return + } + var bestMAC string + var bestIP string + for _, iface := range ifaces { + if iface.Flags&net.FlagLoopback != 0 { + continue + } + if len(iface.HardwareAddr) == 0 { + continue + } + if bestMAC == "" { + bestMAC = iface.HardwareAddr.String() + } + if iface.Flags&net.FlagUp == 0 { + continue + } + // Prefer the first UP interface with a MAC. + if bestMAC != iface.HardwareAddr.String() && bestIP == "" { + bestMAC = iface.HardwareAddr.String() + } + addrs, err := iface.Addrs() + if err != nil { + continue + } + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && ipnet.IP.To4() != nil { + bestMAC = iface.HardwareAddr.String() + bestIP = ipnet.IP.String() + } + } + } + st.lanMAC = bestMAC + st.lanIP = bestIP +} + +// startBreakglassAfterDHCP waits until a LAN IP is assigned (meaning DHCP +// succeeded), then restarts breakglass. This ensures breakglass sees the +// real LAN address in PrivateInterfaceAddrs and binds to it, rather than +// only binding to 127.0.0.1. +func startBreakglassAfterDHCP(st *uiState) { + for { + st.updateLAN() + if st.lanIP != "" { + break + } + time.Sleep(time.Second) + } + startBreakglass() +} + +// startBreakglass asks the gokrazy init HTTP API (over its unix socket) to +// restart the breakglass service so it actually runs. By default breakglass +// calls DontStartOnBoot and exits on the first launch attempt; this poke +// tells the supervisor to try again (without GOKRAZY_FIRST_START=1). +func startBreakglass() { + const sock = "/run/gokrazy-http.sock" + hc := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", sock) + }, + }, + } + form := url.Values{ + "path": {"/user/breakglass"}, + "xsrftoken": {"1"}, + } + req, err := http.NewRequest("POST", "http://gokrazy/restart", strings.NewReader(form.Encode())) + if err != nil { + log.Printf("startBreakglass: %v", err) + return + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.AddCookie(&http.Cookie{Name: "gokrazy_xsrf", Value: "1"}) + resp, err := hc.Do(req) + if err != nil { + log.Printf("startBreakglass: %v", err) + return + } + resp.Body.Close() + if resp.StatusCode < 300 || resp.StatusCode == http.StatusSeeOther { + log.Printf("startBreakglass: restarted (status %s)", resp.Status) + } else { + log.Printf("startBreakglass: unexpected status %s", resp.Status) + } +} + +// watchKeyboardForConsole monitors keyboard input devices for VT-switching +// accelerators. +// - Ctrl-Alt-F2 (or plain Esc — easier to type in NoVNC where the +// Ctrl-Alt-Fn sequence doesn't always propagate) switches to VT2, a +// text-mode busybox shell. When that shell exits, we switch back +// automatically. +// - Ctrl-Alt-F1 switches back to VT1 (fbstatus graphics mode). +// +// This mirrors standard Linux VT switching conventions plus a NoVNC- +// friendly shortcut. +func watchKeyboardForConsole(ctx context.Context, st *uiState) { + kbdPath := findKeyboard() + if kbdPath == "" { + log.Printf("no keyboard found for VT switching") + return + } + kbd, err := os.Open(kbdPath) + if err != nil { + log.Printf("open keyboard %s: %v", kbdPath, err) + return + } + defer kbd.Close() + + ttyFile, err := os.OpenFile("/dev/tty0", os.O_RDWR, 0) + if err != nil { + log.Printf("open /dev/tty0 for VT switch: %v", err) + return + } + defer ttyFile.Close() + ttyFd := int(ttyFile.Fd()) + + log.Printf("watching %s for Ctrl-Alt-F1/F2 and Esc (VT switching)", kbdPath) + + // Linux input_event has the same layout on both arm64 and amd64 + // (24 bytes: two uint64 timestamps + uint16 type + uint16 code + + // int32 value), so this parser handles both the Pi and Proxmox VM. + const evSize = 24 + const evKey = 1 // EV_KEY + const keyEsc = 1 // KEY_ESC + const keyF1 = 59 // KEY_F1 + const keyF2 = 60 // KEY_F2 + const keyLeftCtrl = 29 + const keyLeftAlt = 56 + const keyRightCtrl = 97 + const keyRightAlt = 100 + const keyPress = 1 + + buf := make([]byte, evSize) + var ctrlHeld, altHeld bool + + switchToFbstatus := func() { + st.paused.Store(false) + syscall.Syscall(syscall.SYS_IOCTL, uintptr(ttyFd), vtActivate, 1) + syscall.Syscall(syscall.SYS_IOCTL, uintptr(ttyFd), vtWaitActive, 1) + ioctlSetInt(ttyFile, kdSetMode, kdGraphics) + st.render() + } + switchToShell := func(reason string) { + st.paused.Store(true) + ioctlSetInt(ttyFile, kdSetMode, kdText) + syscall.Syscall(syscall.SYS_IOCTL, uintptr(ttyFd), vtActivate, 2) + syscall.Syscall(syscall.SYS_IOCTL, uintptr(ttyFd), vtWaitActive, 2) + go ensureShellOnVT2(switchToFbstatus) + log.Printf("%s: switched to text console", reason) + } + + for ctx.Err() == nil { + n, err := kbd.Read(buf) + if err != nil || n < evSize { + continue + } + evType := binary.LittleEndian.Uint16(buf[16:18]) + evCode := binary.LittleEndian.Uint16(buf[18:20]) + evValue := int32(binary.LittleEndian.Uint32(buf[20:24])) + + if evType != evKey { + continue + } + + pressed := evValue == keyPress + released := evValue == 0 + + switch evCode { + case keyLeftCtrl, keyRightCtrl: + if pressed { + ctrlHeld = true + } else if released { + ctrlHeld = false + } + case keyLeftAlt, keyRightAlt: + if pressed { + altHeld = true + } else if released { + altHeld = false + } + case keyEsc: + if pressed && !ctrlHeld && !altHeld { + // Bare Esc: NoVNC-friendly shortcut to the shell. + switchToShell("Esc") + } + case keyF1: + if pressed && ctrlHeld && altHeld { + switchToFbstatus() + log.Printf("Ctrl-Alt-F1: switched to fbstatus") + } + case keyF2: + if pressed && ctrlHeld && altHeld { + switchToShell("Ctrl-Alt-F2") + } + } + } +} + +// ensureShellOnVT2 spawns a busybox ash shell on /dev/tty2 if one isn't +// already running. The shell gets the VT2 tty as its controlling terminal +// so keyboard input on VT2 goes to it. When the shell exits, onExit is +// called (typically to switch back to VT1 / fbstatus graphics mode). +var shellOnVT2Running atomic.Bool + +func ensureShellOnVT2(onExit func()) { + if !shellOnVT2Running.CompareAndSwap(false, true) { + return + } + go func() { + defer shellOnVT2Running.Store(false) + defer func() { + if onExit != nil { + onExit() + } + }() + shell := "/tmp/serial-busybox/ash" + if _, err := os.Stat(shell); err != nil { + log.Printf("no shell at %s for VT2", shell) + return + } + tty, err := os.OpenFile("/dev/tty2", os.O_RDWR, 0) + if err != nil { + log.Printf("open /dev/tty2: %v", err) + return + } + defer tty.Close() + cmd := exec.Command(shell) + cmd.Stdin = tty + cmd.Stdout = tty + cmd.Stderr = tty + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + Setctty: true, + Ctty: 0, // index into cmd's file descriptors (stdin = tty) + } + cmd.Env = append(os.Environ(), "TERM=linux", "HOME=/tmp", "PATH=/tmp/serial-busybox:/user:/gokrazy") + log.Printf("starting shell on VT2") + if err := cmd.Run(); err != nil { + log.Printf("shell on VT2 exited: %v", err) + } + }() +} + +// findKeyboard looks for a keyboard among /dev/input/event* devices by +// checking that the device's key capability bitmap has KEY_ESC set. The +// alternative "any non-zero key bitmap" check picks up the ACPI power +// button (which advertises KEY_POWER but no Esc) and misses the real +// keyboard on amd64 Proxmox VMs, where the AT keyboard is event1 but +// event0 is Power Button. +func findKeyboard() string { + matches, _ := filepath.Glob("/dev/input/event*") + for _, path := range matches { + name := filepath.Base(path) + capData, err := os.ReadFile("/sys/class/input/" + name + "/device/capabilities/key") + if err != nil { + continue + } + fields := strings.Fields(strings.TrimSpace(string(capData))) + if len(fields) == 0 { + continue + } + // The kernel prints capability bitmaps as space-separated 64-bit + // hex chunks, most-significant chunk first. The last chunk holds + // bits 0..63. KEY_ESC = 1, so its bit-mask is 1<<1 == 0x2. + low, err := strconv.ParseUint(fields[len(fields)-1], 16, 64) + if err != nil { + continue + } + const keyEscBit = 1 << 1 + if low&keyEscBit != 0 { + return path + } + } + return "" +} + +// pollLAN periodically refreshes LAN info and re-renders. +func (st *uiState) pollLAN(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-time.After(5 * time.Second): + st.updateLAN() + st.render() + } + } +} + +// uiState is the most-recently-known view of the appliance state that +// gets rendered to the framebuffer on each notify. +type uiState struct { + fb *framebuffer + logo image.Image + + state ipn.State + loginURL string + ips []netip.Addr + + lanIP string // LAN IPv4 address (from DHCP) + lanMAC string // MAC address of the primary interface + + paused atomic.Bool // when true, render() is a no-op (VT switched away) +} + +var ( + bgColor = color.RGBA{0x10, 0x12, 0x20, 0xff} // near-black slate + fgColor = color.RGBA{0xff, 0xff, 0xff, 0xff} + dimColor = color.RGBA{0xa0, 0xa6, 0xb8, 0xff} + stateOK = color.RGBA{0x4a, 0xc8, 0x82, 0xff} // green for Running + stateWait = color.RGBA{0xf0, 0xc8, 0x60, 0xff} // amber for NeedsLogin/Starting +) + +// render composes the current state into an in-memory image and blits +// it to the framebuffer. +func (st *uiState) render() { + if st.paused.Load() { + return + } + w, h := st.fb.width, st.fb.height + img := image.NewRGBA(image.Rect(0, 0, w, h)) + draw.Draw(img, img.Bounds(), &image.Uniform{C: bgColor}, image.Point{}, draw.Src) + + shortSide := min(w, h) + + // Logo, scaled to ~25% of the shorter dimension, centered + // horizontally near the top. + logoSize := shortSide / 4 + logoRect := image.Rect(0, 0, logoSize, logoSize).Add(image.Point{ + X: (w - logoSize) / 2, + Y: shortSide / 16, + }) + xdraw.ApproxBiLinear.Scale(img, logoRect, st.logo, st.logo.Bounds(), xdraw.Over, nil) + + lineH := basicfont.Face7x13.Metrics().Height.Ceil() + textTop := logoRect.Max.Y + shortSide/24 + + // Hide the state line when the QR code is visible (the "Scan to + // enroll" label is clear enough context). + showState := !(st.state == ipn.NeedsLogin && st.loginURL != "") + if showState { + stateColor := dimColor + switch st.state { + case ipn.Running: + stateColor = stateOK + case ipn.NeedsLogin, ipn.Starting, ipn.NoState: + stateColor = stateWait + } + drawCenteredScaled(img, fmt.Sprintf("State: %s", stateLabel(st.state)), + stateColor, w/2, textTop, 3) + } + + y := textTop + 3*lineH + shortSide/40 + + if len(st.ips) > 0 { + drawCenteredScaled(img, "Tailscale IPs:", dimColor, w/2, y, 2) + y += 2 * lineH + for _, a := range st.ips { + drawCenteredScaled(img, a.String(), fgColor, w/2, y, 2) + y += 2*lineH + 4 + } + } + + // QR code with the login URL when enrollment is needed. + if st.state == ipn.NeedsLogin && st.loginURL != "" { + qrSize := shortSide / 2 + q, err := qrcode.New(st.loginURL, qrcode.Medium) + if err != nil { + log.Printf("qr encode %q: %v", st.loginURL, err) + } else { + q.DisableBorder = false + qrImg := q.Image(qrSize) + qrRect := qrImg.Bounds().Add(image.Point{ + X: (w - qrSize) / 2, + Y: h - qrSize - shortSide/16, + }) + draw.Draw(img, qrRect, qrImg, qrImg.Bounds().Min, draw.Src) + drawCenteredScaled(img, "Scan to enroll this device", + fgColor, w/2, qrRect.Min.Y-lineH*2-8, 2) + } + } + + // LAN status pinned to the bottom-left corner. + { + lanY := h - lineH - 4 + var lanText string + if st.lanIP != "" { + lanText = "LAN IP: " + st.lanIP + } else if st.lanMAC != "" { + lanText = "Waiting for DHCP (" + st.lanMAC + ")" + } + if lanText != "" { + face := basicfont.Face7x13 + textW := font.MeasureString(face, lanText).Ceil() + small := image.NewRGBA(image.Rect(0, 0, textW, lineH)) + d := font.Drawer{ + Dst: small, + Src: &image.Uniform{C: dimColor}, + Face: face, + Dot: fixed.P(0, face.Metrics().Ascent.Ceil()), + } + d.DrawString(lanText) + dstRect := image.Rect(4, lanY, 4+textW, lanY+lineH) + draw.Draw(img, dstRect, small, image.Point{}, draw.Over) + } + } + + st.fb.blit(img) +} + +// drawCenteredScaled draws s with basicfont.Face7x13 at the given +// integer pixel scale, centered horizontally on x at top y, in col. +func drawCenteredScaled(dst *image.RGBA, s string, col color.Color, x, y, scale int) { + if s == "" { + return + } + face := basicfont.Face7x13 + width := font.MeasureString(face, s).Ceil() + height := face.Metrics().Height.Ceil() + + small := image.NewRGBA(image.Rect(0, 0, width, height)) + d := font.Drawer{ + Dst: small, + Src: &image.Uniform{C: col}, + Face: face, + Dot: fixed.P(0, face.Metrics().Ascent.Ceil()), + } + d.DrawString(s) + + scaledW, scaledH := width*scale, height*scale + dstRect := image.Rect(0, 0, scaledW, scaledH).Add(image.Point{ + X: x - scaledW/2, + Y: y, + }) + xdraw.NearestNeighbor.Scale(dst, dstRect, small, small.Bounds(), xdraw.Over, nil) +} + +func stateLabel(s ipn.State) string { + switch s { + case ipn.NoState, ipn.Starting: + return "starting" + case ipn.NeedsLogin: + return "needs login" + case ipn.NeedsMachineAuth: + return "needs machine auth" + case ipn.Stopped: + return "stopped" + case ipn.Running: + return "running" + } + return strings.ToLower(s.String()) +} + +// framebuffer is an mmap'd Linux framebuffer device. +type framebuffer struct { + f *os.File + mem []byte + width int + height int + bpp int + lineLength int + + // Bit offsets into a 32-bit pixel for each channel, from the + // fb_bitfield values returned by FBIOGET_VSCREENINFO. + redShift uint32 + greenShift uint32 + blueShift uint32 +} + +// openFramebuffer opens path, queries dimensions and pixel format via +// the FBIOGET_* ioctls, and mmaps the framebuffer memory. +// +// Only 32-bits-per-pixel framebuffers are supported. Raspberry Pi 3/4/5 +// default to that. +func openFramebuffer(path string) (*framebuffer, error) { + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return nil, err + } + var ( + vbuf [160]byte // fb_var_screeninfo + fbuf [80]byte // fb_fix_screeninfo + ) + if err := ioctlGet(f, fbioGetVScreenInfo, vbuf[:]); err != nil { + f.Close() + return nil, fmt.Errorf("FBIOGET_VSCREENINFO: %w", err) + } + if err := ioctlGet(f, fbioGetFScreenInfo, fbuf[:]); err != nil { + f.Close() + return nil, fmt.Errorf("FBIOGET_FSCREENINFO: %w", err) + } + + fb := &framebuffer{ + f: f, + width: int(binary.LittleEndian.Uint32(vbuf[vsOffXres:])), + height: int(binary.LittleEndian.Uint32(vbuf[vsOffYres:])), + bpp: int(binary.LittleEndian.Uint32(vbuf[vsOffBitsPerPixel:])), + lineLength: int(binary.LittleEndian.Uint32(fbuf[fsOffLineLength:])), + redShift: binary.LittleEndian.Uint32(vbuf[vsOffRedOffset:]), + greenShift: binary.LittleEndian.Uint32(vbuf[vsOffGreenOffset:]), + blueShift: binary.LittleEndian.Uint32(vbuf[vsOffBlueOffset:]), + } + if fb.bpp != 32 { + f.Close() + return nil, fmt.Errorf("unsupported framebuffer bpp %d (only 32 is supported)", fb.bpp) + } + + memLen := int(binary.LittleEndian.Uint32(fbuf[fsOffSmemLen:])) + mem, err := unix.Mmap(int(f.Fd()), 0, memLen, + unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) + if err != nil { + f.Close() + return nil, fmt.Errorf("mmap %s: %w", path, err) + } + fb.mem = mem + return fb, nil +} + +func (fb *framebuffer) Close() error { + if fb.mem != nil { + unix.Munmap(fb.mem) + fb.mem = nil + } + return fb.f.Close() +} + +// blit copies img into the mapped framebuffer, packing each +// image.RGBA pixel into the framebuffer's per-channel bit layout. +func (fb *framebuffer) blit(img *image.RGBA) { + srcStride := img.Stride + for y := 0; y < fb.height; y++ { + srcRow := img.Pix[y*srcStride : y*srcStride+fb.width*4] + dstRow := fb.mem[y*fb.lineLength:] + for x := 0; x < fb.width; x++ { + r := uint32(srcRow[x*4+0]) + g := uint32(srcRow[x*4+1]) + b := uint32(srcRow[x*4+2]) + px := r< 0 { - return vals[0] - } - for _, nodeIP := range who.Node.Addresses { - if nodeIP.Addr().Is4() && nodeIP.IsSingleIP() { - return nodeIP.Addr().String() - } - } - for _, nodeIP := range who.Node.Addresses { - if nodeIP.IsSingleIP() { - return nodeIP.Addr().String() - } - } - return "" -} - -func root(w http.ResponseWriter, r *http.Request) { - if r.TLS == nil && *httpsAddr != "" { - host := r.Host - if strings.Contains(r.Host, "100.101.102.103") || - strings.Contains(r.Host, "hello.ipn.dev") { - host = "hello.ts.net" - } - http.Redirect(w, r, "https://"+host, http.StatusFound) - return - } - if r.RequestURI != "/" { - http.Redirect(w, r, "/", http.StatusFound) - return - } - if r.TLS != nil && *httpsAddr != "" && strings.Contains(r.Host, "hello.ipn.dev") { - http.Redirect(w, r, "https://hello.ts.net", http.StatusFound) - return - } - tmpl, err := getTmpl() - if err != nil { - w.Header().Set("Content-Type", "text/plain") - http.Error(w, "template error: "+err.Error(), 500) - return - } - - who, err := localClient.WhoIs(r.Context(), r.RemoteAddr) - var data tmplData - if err != nil { - if devMode() { - log.Printf("warning: using fake data in dev mode due to whois lookup error: %v", err) - data = tmplData{ - DisplayName: "Taily Scalerson", - LoginName: "taily@scaler.son", - ProfilePicURL: "https://placekitten.com/200/200", - MachineName: "scaled", - MachineOS: "Linux", - IP: "100.1.2.3", - } - } else { - log.Printf("whois(%q) error: %v", r.RemoteAddr, err) - http.Error(w, "Your Tailscale works, but we failed to look you up.", 500) - return - } - } else { - data = tmplData{ - DisplayName: who.UserProfile.DisplayName, - LoginName: who.UserProfile.LoginName, - ProfilePicURL: who.UserProfile.ProfilePicURL, - MachineName: firstLabel(who.Node.ComputedName), - MachineOS: who.Node.Hostinfo.OS(), - IP: tailscaleIP(who), - } - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - tmpl.Execute(w, data) -} - -// firstLabel s up until the first period, if any. -func firstLabel(s string) string { - s, _, _ = strings.Cut(s, ".") - return s + log.Fatal(s.Run()) } diff --git a/cmd/hello/hello.tmpl.html b/cmd/hello/hello.tmpl.html deleted file mode 100644 index 3ecd1b58a..000000000 --- a/cmd/hello/hello.tmpl.html +++ /dev/null @@ -1,438 +0,0 @@ - - - - - - Hello from Tailscale - - - - -
- -
-

You're connected over Tailscale!

-

This device is signed in as…

-
-
-
- - - -
-
-
-
- {{ with .DisplayName }} -

{{.}}

- {{ end }} -
{{.LoginName}}
-
-
-
-
- - - - - - -

{{.MachineName}}

-
-
{{.IP}}
-
-
- -
- - diff --git a/cmd/hello/helloserver/hello.tmpl.html b/cmd/hello/helloserver/hello.tmpl.html new file mode 100644 index 000000000..0f74d116f --- /dev/null +++ b/cmd/hello/helloserver/hello.tmpl.html @@ -0,0 +1,71 @@ + + + + + + Hello from Tailscale + + + + +
+ +
+

You're connected over Tailscale!

+

This device is signed in as…

+
+
+
+ + + +
+
+
+ Profile picture +
+
+ {{ with .DisplayName }} +

{{.}}

+ {{ end }} +
{{.LoginName}}
+
+
+
+
+ + + + + + +

{{.MachineName}}

+
+
{{.IP}}
+
+
+ +
+ + diff --git a/cmd/hello/helloserver/helloserver.go b/cmd/hello/helloserver/helloserver.go new file mode 100644 index 000000000..41e7dbce2 --- /dev/null +++ b/cmd/hello/helloserver/helloserver.go @@ -0,0 +1,157 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package helloserver implements the HTTP server behind hello.ts.net. +package helloserver + +import ( + "crypto/tls" + "embed" + "html/template" + "log" + "net/http" + "strings" + "time" + + "tailscale.com/client/local" + "tailscale.com/client/tailscale/apitype" + "tailscale.com/tailcfg" +) + +//go:embed hello.tmpl.html +var embeddedTemplate string + +//go:embed static/* +var staticFiles embed.FS + +var staticHandler = http.FileServerFS(staticFiles) + +var tmpl = template.Must(template.New("home").Parse(embeddedTemplate)) + +// Server is an HTTP server for hello.ts.net. +// +// The zero value is not valid; populate at least one of HTTPAddr or HTTPSAddr +// before calling Run. +type Server struct { + // HTTPAddr is the address to run an HTTP server on, or empty for none. + HTTPAddr string + + // HTTPSAddr is the address to run an HTTPS server on, or empty for none. + HTTPSAddr string + + // LocalClient is used to look up the identity of incoming requests and + // to obtain TLS certificates. If nil, the zero value of local.Client is + // used. + LocalClient *local.Client +} + +func (s *Server) localClient() *local.Client { + if s.LocalClient != nil { + return s.LocalClient + } + return &local.Client{} +} + +// Run starts the configured HTTP and HTTPS servers and blocks until one of +// them returns an error. +func (s *Server) Run() error { + errc := make(chan error, 1) + if s.HTTPAddr != "" { + log.Printf("running HTTP server on %s", s.HTTPAddr) + go func() { + errc <- http.ListenAndServe(s.HTTPAddr, s) + }() + } + if s.HTTPSAddr != "" { + log.Printf("running HTTPS server on %s", s.HTTPSAddr) + go func() { + hs := &http.Server{ + Addr: s.HTTPSAddr, + Handler: s, + TLSConfig: &tls.Config{ + GetCertificate: s.localClient().GetCertificate, + }, + IdleTimeout: 30 * time.Second, + ReadHeaderTimeout: 20 * time.Second, + MaxHeaderBytes: 10 << 10, + } + errc <- hs.ListenAndServeTLS("", "") + }() + } + return <-errc +} + +type tmplData struct { + DisplayName string // "Foo Barberson" + LoginName string // "foo@bar.com" + ProfilePicURL string // "https://..." + MachineName string // "imac5k" + MachineOS string // "Linux" + IP string // "100.2.3.4" +} + +func tailscaleIP(who *apitype.WhoIsResponse) string { + if who == nil { + return "" + } + vals, err := tailcfg.UnmarshalNodeCapJSON[string](who.Node.CapMap, tailcfg.NodeAttrNativeIPV4) + if err == nil && len(vals) > 0 { + return vals[0] + } + for _, nodeIP := range who.Node.Addresses { + if nodeIP.Addr().Is4() && nodeIP.IsSingleIP() { + return nodeIP.Addr().String() + } + } + for _, nodeIP := range who.Node.Addresses { + if nodeIP.IsSingleIP() { + return nodeIP.Addr().String() + } + } + return "" +} + +// ServeHTTP implements http.Handler. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.TLS == nil && s.HTTPSAddr != "" { + host := r.Host + if strings.Contains(r.Host, "100.101.102.103") { + host = "hello.ts.net" + } + http.Redirect(w, r, "https://"+host, http.StatusFound) + return + } + + if strings.HasPrefix(r.RequestURI, "/static/") { + staticHandler.ServeHTTP(w, r) + return + } + + if r.RequestURI != "/" { + http.Redirect(w, r, "/", http.StatusFound) + return + } + + who, err := s.localClient().WhoIs(r.Context(), r.RemoteAddr) + if err != nil { + log.Printf("whois(%q) error: %v", r.RemoteAddr, err) + http.Error(w, "Your Tailscale works, but we failed to look you up.", 500) + return + } + data := tmplData{ + DisplayName: who.UserProfile.DisplayName, + LoginName: who.UserProfile.LoginName, + ProfilePicURL: who.UserProfile.ProfilePicURL, + MachineName: firstLabel(who.Node.ComputedName), + MachineOS: who.Node.Hostinfo.OS(), + IP: tailscaleIP(who), + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl.Execute(w, data) +} + +// firstLabel returns s up until the first period, if any. +func firstLabel(s string) string { + s, _, _ = strings.Cut(s, ".") + return s +} diff --git a/cmd/hello/helloserver/static/script.js b/cmd/hello/helloserver/static/script.js new file mode 100644 index 000000000..db9bcd0f3 --- /dev/null +++ b/cmd/hello/helloserver/static/script.js @@ -0,0 +1,12 @@ +(function () { + var lastSeen = localStorage.getItem("lastSeen"); + if (!lastSeen) { + document.body.classList.add("animate"); + window.addEventListener("load", function () { + setTimeout(function () { + document.body.classList.add("animating"); + localStorage.setItem("lastSeen", Date.now()); + }, 100); + }); + } +})(); diff --git a/cmd/hello/helloserver/static/style.css b/cmd/hello/helloserver/static/style.css new file mode 100644 index 000000000..8ad55edc6 --- /dev/null +++ b/cmd/hello/helloserver/static/style.css @@ -0,0 +1,366 @@ +html, +body { + margin: 0; + padding: 0; +} + +body { + font-family: Inter, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + font-size: 100%; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +html, +body, +main { + height: 100%; +} + +*, +::before, +::after { + box-sizing: border-box; + border-width: 0; + border-style: solid; + border-color: #dad6d5; +} + +h1, +h2, +h3, +h4, +h5, +h6 { + margin: 0; + font-size: 1rem; + font-weight: inherit; +} + +a { + color: inherit; +} + +p { + margin: 0; +} + +main { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + max-width: 24rem; + width: 95%; + margin-left: auto; + margin-right: auto; +} + +.p-2 { + padding: 0.5rem; +} + +.p-4 { + padding: 1rem; +} + +.px-2 { + padding-left: 0.5rem; + padding-right: 0.5rem; +} + +.pl-3 { + padding-left: 0.75rem; +} + +.pr-3 { + padding-right: 0.75rem; +} + +.pt-4 { + padding-top: 1rem; +} + +.mr-2 { + margin-right: 0.5rem; +; +} + +.mb-1 { + margin-bottom: 0.25rem; +} + +.mb-2 { + margin-bottom: 0.5rem; +} + +.mb-4 { + margin-bottom: 1rem; +} + +.mb-6 { + margin-bottom: 1.5rem; +} + +.mb-8 { + margin-bottom: 2rem; +} + +.mb-12 { + margin-bottom: 3rem; +} + +.width-full { + width: 100%; +} + +.min-width-0 { + min-width: 0; +} + +.rounded-lg { + border-radius: 0.5rem; +} + +.relative { + position: relative; +} + +.flex { + display: flex; +} + +.justify-between { + justify-content: space-between; +} + +.items-center { + align-items: center; +} + +.border { + border-width: 1px; +} + +.border-t-1 { + border-top-width: 1px; +} + +.border-gray-100 { + border-color: #f7f5f4; +} + +.border-gray-200 { + border-color: #eeebea; +} + +.border-gray-300 { + border-color: #dad6d5; +} + +.bg-white { + background-color: white; +} + +.bg-gray-0 { + background-color: #faf9f8; +} + +.bg-gray-100 { + background-color: #f7f5f4; +} + +.text-green-600 { + color: #0d4b3b; +} + +.text-blue-600 { + color: #3f5db3; +} + +.hover\:text-blue-800:hover { + color: #253570; +} + +.text-gray-600 { + color: #444342; +} + +.text-gray-700 { + color: #2e2d2d; +} + +.text-gray-800 { + color: #232222; +} + +.text-center { + text-align: center; +} + +.text-sm { + font-size: 0.875rem; +} + +.font-title { + font-size: 1.25rem; + letter-spacing: -0.025em; +} + +.font-semibold { + font-weight: 600; +} + +.font-medium { + font-weight: 500; +} + +.font-regular { + font-weight: 400; +} + +.truncate { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.overflow-hidden { + overflow: hidden; +} + +.profile-pic { + width: 2.5rem; + height: 2.5rem; + background-size: cover; + margin-right: 0.5rem; + flex-shrink: 0; +} + +.profile-pic-img { + width: 100%; + height: 100%; + object-fit: cover; + display: block; + border-radius: 9999px; +} + +.panel { + box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04); +} + +.animate .panel { + transform: translateY(10%); + box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.0), 0 10px 10px -5px rgba(0, 0, 0, 0.0); + transition: transform 1200ms ease, opacity 1200ms ease, box-shadow 1200ms ease; +} + +.animate .panel-interior { + opacity: 0.0; + transition: opacity 1200ms ease; +} + +.animate .logo { + transform: translateY(2rem); + opacity: 0.0; + transition: transform 1200ms ease, opacity 1200ms ease; +} + +.animate .header-title { + transform: translateY(1.6rem); + opacity: 0.0; + transition: transform 1200ms ease, opacity 1200ms ease; +} + +.animate .header-text { + transform: translateY(1.2rem); + opacity: 0.0; + transition: transform 1200ms ease, opacity 1200ms ease; +} + +.animate .footer { + transform: translateY(-0.5rem); + opacity: 0.0; + transition: transform 1200ms ease, opacity 1200ms ease; +} + +.animating .panel { + transform: translateY(0); + opacity: 1.0; + box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04); +} + +.animating .panel-interior { + opacity: 1.0; +} + +.animating .spinner { + opacity: 0.0; +} + +.animating .logo, +.animating .header-title, +.animating .header-text, +.animating .footer { + transform: translateY(0); + opacity: 1.0; +} + +.spinner { + display: inline-flex; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + align-items: center; + transition: opacity 200ms ease; +} + +.spinner span { + display: inline-block; + background-color: currentColor; + border-radius: 9999px; + animation-name: loading-dots-blink; + animation-duration: 1.4s; + animation-iteration-count: infinite; + animation-fill-mode: both; + width: 0.35em; + height: 0.35em; + margin: 0 0.15em; +} + +.spinner span:nth-child(2) { + animation-delay: 200ms; +} + +.spinner span:nth-child(3) { + animation-delay: 400ms; +} + +.spinner { + display: none; +} + +.animate .spinner { + display: inline-flex; +} + +@keyframes loading-dots-blink { + 0% { + opacity: 0.2; + } + 20% { + opacity: 1; + } + 100% { + opacity: 0.2; + } +} + +@media (prefers-reduced-motion) { + * { + animation-duration: 0ms !important; + transition-duration: 0ms !important; + transition-delay: 0ms !important; + } +} diff --git a/cmd/k8s-nameserver/main.go b/cmd/k8s-nameserver/main.go index 1b219fb1a..1e6a1dea9 100644 --- a/cmd/k8s-nameserver/main.go +++ b/cmd/k8s-nameserver/main.go @@ -29,7 +29,7 @@ import ( const ( // tsNetDomain is the domain that this DNS nameserver has registered a handler for. tsNetDomain = "ts.net" - // addr is the the address that the UDP and TCP listeners will listen on. + // addr is the address that the UDP and TCP listeners will listen on. addr = ":1053" // defaultTTL is the default TTL for DNS records in seconds. // Set to 0 to disable caching. Can be increased when usage patterns are better understood. diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go index ac2f73577..37260c7a0 100644 --- a/cmd/k8s-operator/api-server-proxy-pg.go +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -436,14 +436,16 @@ func exclusiveOwnerAnnotations(pg *tsapi.ProxyGroup, operatorID string, svc *tai } if svc == nil { c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} - json, err := json.Marshal(c) + data, err := json.Marshal(c) if err != nil { - return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service's owner annotation contents: %w, please report this", err) + return nil, fmt.Errorf("failed to marshal Tailscale Service's owner annotation contents: %w", err) } + return map[string]string{ - ownerAnnotation: string(json), + ownerAnnotation: string(data), }, nil } + o, err := parseOwnerAnnotation(svc) if err != nil { return nil, err @@ -451,15 +453,19 @@ func exclusiveOwnerAnnotations(pg *tsapi.ProxyGroup, operatorID string, svc *tai if o == nil || len(o.OwnerRefs) == 0 { return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name) } + if len(o.OwnerRefs) > 1 || o.OwnerRefs[0].OperatorID != operatorID { return nil, fmt.Errorf("Tailscale Service %s is already owned by other operator(s) and cannot be shared across multiple clusters; configure a difference Service name to continue", svc.Name) } + if o.OwnerRefs[0].Resource == nil { return nil, fmt.Errorf("Tailscale Service %s exists, but does not reference an owning resource; not proceeding as this is likely a Service already owned by an Ingress", svc.Name) } + if o.OwnerRefs[0].Resource.Kind != "ProxyGroup" || o.OwnerRefs[0].Resource.UID != string(pg.UID) { return nil, fmt.Errorf("Tailscale Service %s is already owned by another resource: %#v; configure a difference Service name to continue", svc.Name, o.OwnerRefs[0].Resource) } + if o.OwnerRefs[0].Resource.Name != pg.Name { // ProxyGroup name can be updated in place. o.OwnerRefs[0].Resource.Name = pg.Name diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index 0c2d32482..323dc7b86 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -29,6 +29,8 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" + "tailscale.com/net/netutil" + "tailscale.com/net/tsaddr" "tailscale.com/tstime" "tailscale.com/util/clientmetric" "tailscale.com/util/set" @@ -356,6 +358,11 @@ func validateRoutes(routes tsapi.Routes) error { if pfx.Masked() != pfx { errs = append(errs, fmt.Errorf("route %s has non-address bits set; expected %s", pfx, pfx.Masked())) } + if tsaddr.IsViaPrefix(pfx) { + if err := netutil.ValidateViaPrefix(pfx); err != nil { + errs = append(errs, err) + } + } } return errors.Join(errs...) } diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 69e8e287d..b366dac38 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -145,6 +145,22 @@ func TestConnector(t *testing.T) { expectReconciled(t, cr, "", "test") expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) + // Set an invalid 4via6 route (site ID too large). + mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { + conn.Spec.SubnetRouter.AdvertiseRoutes = []tsapi.Route{"fd7a:115c:a1e0:b1a:1:0:a2c:0/116"} + }) + expectReconciled(t, cr, "", "test") + // STS should still have the previous valid route, unchanged. + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) + + // Set a valid 4via6 route. + mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { + conn.Spec.SubnetRouter.AdvertiseRoutes = []tsapi.Route{"fd7a:115c:a1e0:b1a:0:1:a2c:0/116"} + }) + opts.subnetRoutes = "fd7a:115c:a1e0:b1a:0:1:a2c:0/116" + expectReconciled(t, cr, "", "test") + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) + // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { t.Fatalf("error deleting Connector: %v", err) diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 2b6884683..51222b6b0 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -6,84 +6,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ W 💣 github.com/alexbrainman/sspi from github.com/alexbrainman/sspi/internal/common+ W github.com/alexbrainman/sspi/internal/common from github.com/alexbrainman/sspi/negotiate W 💣 github.com/alexbrainman/sspi/negotiate from tailscale.com/net/tshttpproxy - github.com/aws/aws-sdk-go-v2/aws from github.com/aws/aws-sdk-go-v2/aws/defaults+ - github.com/aws/aws-sdk-go-v2/aws/defaults from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/aws-sdk-go-v2/aws/middleware from github.com/aws/aws-sdk-go-v2/aws/retry+ - github.com/aws/aws-sdk-go-v2/aws/protocol/query from github.com/aws/aws-sdk-go-v2/service/sts - github.com/aws/aws-sdk-go-v2/aws/protocol/restjson from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/aws-sdk-go-v2/aws/protocol/xml from github.com/aws/aws-sdk-go-v2/service/sts - github.com/aws/aws-sdk-go-v2/aws/ratelimit from github.com/aws/aws-sdk-go-v2/aws/retry - github.com/aws/aws-sdk-go-v2/aws/retry from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client+ - github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 from github.com/aws/aws-sdk-go-v2/aws/signer/v4 - github.com/aws/aws-sdk-go-v2/aws/signer/v4 from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+ - github.com/aws/aws-sdk-go-v2/aws/transport/http from github.com/aws/aws-sdk-go-v2/config+ - github.com/aws/aws-sdk-go-v2/config from tailscale.com/wif - github.com/aws/aws-sdk-go-v2/credentials from github.com/aws/aws-sdk-go-v2/config - github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds from github.com/aws/aws-sdk-go-v2/config - github.com/aws/aws-sdk-go-v2/credentials/endpointcreds from github.com/aws/aws-sdk-go-v2/config - github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds - github.com/aws/aws-sdk-go-v2/credentials/processcreds from github.com/aws/aws-sdk-go-v2/config - github.com/aws/aws-sdk-go-v2/credentials/ssocreds from github.com/aws/aws-sdk-go-v2/config - github.com/aws/aws-sdk-go-v2/credentials/stscreds from github.com/aws/aws-sdk-go-v2/config - github.com/aws/aws-sdk-go-v2/feature/ec2/imds from github.com/aws/aws-sdk-go-v2/config+ - github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds - github.com/aws/aws-sdk-go-v2/internal/auth from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ - github.com/aws/aws-sdk-go-v2/internal/auth/smithy from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/aws-sdk-go-v2/internal/configsources from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/aws-sdk-go-v2/internal/context from github.com/aws/aws-sdk-go-v2/aws/retry+ - github.com/aws/aws-sdk-go-v2/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 from github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints+ - github.com/aws/aws-sdk-go-v2/internal/ini from github.com/aws/aws-sdk-go-v2/config - github.com/aws/aws-sdk-go-v2/internal/middleware from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/aws-sdk-go-v2/internal/rand from github.com/aws/aws-sdk-go-v2/aws+ - github.com/aws/aws-sdk-go-v2/internal/sdk from github.com/aws/aws-sdk-go-v2/aws+ - github.com/aws/aws-sdk-go-v2/internal/sdkio from github.com/aws/aws-sdk-go-v2/credentials/processcreds - github.com/aws/aws-sdk-go-v2/internal/shareddefaults from github.com/aws/aws-sdk-go-v2/config+ - github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 - github.com/aws/aws-sdk-go-v2/internal/sync/singleflight from github.com/aws/aws-sdk-go-v2/aws - github.com/aws/aws-sdk-go-v2/internal/timeconv from github.com/aws/aws-sdk-go-v2/aws/retry - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url from github.com/aws/aws-sdk-go-v2/service/sts - github.com/aws/aws-sdk-go-v2/service/sso from github.com/aws/aws-sdk-go-v2/config+ - github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sso - github.com/aws/aws-sdk-go-v2/service/sso/types from github.com/aws/aws-sdk-go-v2/service/sso - github.com/aws/aws-sdk-go-v2/service/ssooidc from github.com/aws/aws-sdk-go-v2/config+ - github.com/aws/aws-sdk-go-v2/service/ssooidc/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssooidc - github.com/aws/aws-sdk-go-v2/service/ssooidc/types from github.com/aws/aws-sdk-go-v2/service/ssooidc - github.com/aws/aws-sdk-go-v2/service/sts from github.com/aws/aws-sdk-go-v2/config+ - github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sts - github.com/aws/aws-sdk-go-v2/service/sts/types from github.com/aws/aws-sdk-go-v2/credentials/stscreds+ - github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ - github.com/aws/smithy-go/auth from github.com/aws/aws-sdk-go-v2/internal/auth+ - github.com/aws/smithy-go/auth/bearer from github.com/aws/aws-sdk-go-v2/aws+ - github.com/aws/smithy-go/context from github.com/aws/smithy-go/auth/bearer - github.com/aws/smithy-go/document from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/smithy-go/encoding from github.com/aws/smithy-go/encoding/json+ - github.com/aws/smithy-go/encoding/httpbinding from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ - github.com/aws/smithy-go/encoding/json from github.com/aws/aws-sdk-go-v2/service/ssooidc - github.com/aws/smithy-go/encoding/xml from github.com/aws/aws-sdk-go-v2/service/sts - github.com/aws/smithy-go/endpoints from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/smithy-go/endpoints/private/rulesfn from github.com/aws/aws-sdk-go-v2/service/sts - github.com/aws/smithy-go/internal/sync/singleflight from github.com/aws/smithy-go/auth/bearer - github.com/aws/smithy-go/io from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+ - github.com/aws/smithy-go/logging from github.com/aws/aws-sdk-go-v2/aws+ - github.com/aws/smithy-go/metrics from github.com/aws/aws-sdk-go-v2/aws/retry+ - github.com/aws/smithy-go/middleware from github.com/aws/aws-sdk-go-v2/aws+ - github.com/aws/smithy-go/private/requestcompression from github.com/aws/aws-sdk-go-v2/config - github.com/aws/smithy-go/ptr from github.com/aws/aws-sdk-go-v2/aws+ - github.com/aws/smithy-go/rand from github.com/aws/aws-sdk-go-v2/aws/middleware - github.com/aws/smithy-go/time from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/smithy-go/tracing from github.com/aws/aws-sdk-go-v2/aws/middleware+ - github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws+ - github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http github.com/beorn7/perks/quantile from github.com/prometheus/client_golang/prometheus github.com/blang/semver/v4 from k8s.io/component-base/metrics 💣 github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus+ github.com/coder/websocket from tailscale.com/util/eventbus github.com/coder/websocket/internal/errd from github.com/coder/websocket github.com/coder/websocket/internal/util from github.com/coder/websocket - github.com/coder/websocket/internal/xsync from github.com/coder/websocket github.com/creachadair/msync/trigger from tailscale.com/logtail 💣 github.com/davecgh/go-spew/spew from k8s.io/apimachinery/pkg/util/dump W 💣 github.com/dblohm7/wingoes from tailscale.com/net/tshttpproxy+ @@ -113,6 +41,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json/jsontext+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json/jsontext+ github.com/go-json-experiment/json/jsontext from tailscale.com/logtail+ + github.com/go-json-experiment/json/v1 from tailscale.com/net/routecheck+ github.com/go-logr/logr from github.com/go-logr/logr/slogr+ github.com/go-logr/logr/slogr from github.com/go-logr/zapr github.com/go-logr/zapr from sigs.k8s.io/controller-runtime/pkg/log/zap+ @@ -130,7 +59,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/google/gnostic-models/jsonschema from github.com/google/gnostic-models/compiler github.com/google/gnostic-models/openapiv2 from k8s.io/client-go/discovery+ github.com/google/gnostic-models/openapiv3 from k8s.io/kube-openapi/pkg/handler3+ - github.com/google/uuid from github.com/prometheus-community/pro-bing+ + github.com/google/uuid from k8s.io/apimachinery/pkg/util/uuid+ github.com/hdevalence/ed25519consensus from tailscale.com/tka github.com/huin/goupnp from github.com/huin/goupnp/dcps/internetgateway2+ github.com/huin/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper @@ -164,7 +93,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/pires/go-proxyproto from tailscale.com/ipn/ipnlocal+ github.com/pkg/errors from github.com/evanphx/json-patch/v5+ github.com/pmezard/go-difflib/difflib from k8s.io/apimachinery/pkg/util/diff - D github.com/prometheus-community/pro-bing from tailscale.com/wgengine/netstack github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil from github.com/prometheus/client_golang/prometheus/promhttp github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil/header from github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil 💣 github.com/prometheus/client_golang/prometheus from github.com/prometheus/client_golang/prometheus/collectors+ @@ -180,7 +108,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ LD github.com/prometheus/procfs/internal/util from github.com/prometheus/procfs L 💣 github.com/safchain/ethtool from tailscale.com/net/netkernelconf github.com/spf13/pflag from k8s.io/client-go/tools/clientcmd+ - W 💣 github.com/tailscale/certstore from tailscale.com/control/controlclient + DW 💣 github.com/tailscale/certstore from tailscale.com/control/controlclient W 💣 github.com/tailscale/go-winio from tailscale.com/safesocket W 💣 github.com/tailscale/go-winio/internal/fs from github.com/tailscale/go-winio W 💣 github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio @@ -802,22 +730,23 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/envknob from tailscale.com/client/local+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/ipn/ipnext+ + tailscale.com/feature/acme from tailscale.com/tsnet tailscale.com/feature/buildfeatures from tailscale.com/wgengine/magicsock+ tailscale.com/feature/c2n from tailscale.com/tsnet tailscale.com/feature/condlite/expvar from tailscale.com/wgengine/magicsock - tailscale.com/feature/condregister/identityfederation from tailscale.com/tsnet + tailscale.com/feature/condregister/netlog from tailscale.com/tsnet tailscale.com/feature/condregister/oauthkey from tailscale.com/tsnet tailscale.com/feature/condregister/portmapper from tailscale.com/tsnet tailscale.com/feature/condregister/useproxy from tailscale.com/tsnet - tailscale.com/feature/identityfederation from tailscale.com/feature/condregister/identityfederation + tailscale.com/feature/netlog from tailscale.com/feature/condregister/netlog tailscale.com/feature/oauthkey from tailscale.com/feature/condregister/oauthkey tailscale.com/feature/portmapper from tailscale.com/feature/condregister/portmapper - tailscale.com/feature/syspolicy from tailscale.com/logpolicy + tailscale.com/feature/serviceclientprefs/serviceclient from tailscale.com/client/local tailscale.com/feature/useproxy from tailscale.com/feature/condregister/useproxy tailscale.com/health from tailscale.com/control/controlclient+ tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal tailscale.com/hostinfo from tailscale.com/client/web+ - tailscale.com/internal/client/tailscale from tailscale.com/feature/identityfederation+ + tailscale.com/internal/client/tailscale from tailscale.com/feature/oauthkey+ tailscale.com/ipn from tailscale.com/client/local+ tailscale.com/ipn/conffile from tailscale.com/ipn/ipnlocal+ 💣 tailscale.com/ipn/ipnauth from tailscale.com/ipn/ipnlocal+ @@ -826,16 +755,18 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/ipn/ipnlocal/netmapcache from tailscale.com/ipn/ipnlocal tailscale.com/ipn/ipnstate from tailscale.com/client/local+ tailscale.com/ipn/localapi from tailscale.com/tsnet - tailscale.com/ipn/store from tailscale.com/ipn/ipnlocal+ + tailscale.com/ipn/store from tailscale.com/ipn/store/kubestore+ tailscale.com/ipn/store/kubestore from tailscale.com/cmd/k8s-operator tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+ tailscale.com/k8s-operator from tailscale.com/cmd/k8s-operator+ tailscale.com/k8s-operator/api-proxy from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/apis from tailscale.com/k8s-operator/apis/v1alpha1 tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+ - tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet + tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet+ + tailscale.com/k8s-operator/reconciler/peerrelay from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/proxygrouppolicy from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator/reconciler/tailscaled from tailscale.com/k8s-operator/reconciler/peerrelay tailscale.com/k8s-operator/sessionrecording from tailscale.com/k8s-operator/api-proxy tailscale.com/k8s-operator/sessionrecording/spdy from tailscale.com/k8s-operator/sessionrecording tailscale.com/k8s-operator/sessionrecording/tsrecorder from tailscale.com/k8s-operator/sessionrecording+ @@ -856,7 +787,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/metrics from tailscale.com/tsweb+ tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+ 💣 tailscale.com/net/batching from tailscale.com/wgengine/magicsock - tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+ tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+ tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ tailscale.com/net/dns/resolvconffile from tailscale.com/cmd/k8s-operator+ @@ -867,7 +797,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ tailscale.com/net/memnet from tailscale.com/tsnet tailscale.com/net/netaddr from tailscale.com/ipn+ - tailscale.com/net/netcheck from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/netcheck from tailscale.com/wgengine/magicsock tailscale.com/net/neterror from tailscale.com/net/dns/resolver+ tailscale.com/net/netkernelconf from tailscale.com/ipn/ipnlocal tailscale.com/net/netknob from tailscale.com/logpolicy+ @@ -881,12 +811,16 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/net/portmapper from tailscale.com/feature/portmapper tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ tailscale.com/net/proxymux from tailscale.com/tsnet + tailscale.com/net/routecheck from tailscale.com/client/local+ + tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ 💣 tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/tsnet tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/ipn/localapi+ tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/ipn/ipnlocal+ tailscale.com/net/tsaddr from tailscale.com/client/web+ tailscale.com/net/tsdial from tailscale.com/control/controlclient+ 💣 tailscale.com/net/tshttpproxy from tailscale.com/feature/useproxy @@ -900,7 +834,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/sessionrecording from tailscale.com/k8s-operator/sessionrecording+ tailscale.com/syncs from tailscale.com/control/controlknobs+ tailscale.com/tailcfg from tailscale.com/client/local+ - tailscale.com/tempfork/acme from tailscale.com/ipn/ipnlocal + tailscale.com/tempfork/acme from tailscale.com/feature/acme tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock tailscale.com/tempfork/httprec from tailscale.com/feature/c2n tailscale.com/tka from tailscale.com/client/local+ @@ -910,7 +844,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/tstime from tailscale.com/cmd/k8s-operator+ tailscale.com/tstime/mono from tailscale.com/net/tstun+ tailscale.com/tstime/rate from tailscale.com/wgengine/filter - tailscale.com/tsweb from tailscale.com/util/eventbus + tailscale.com/tsweb from tailscale.com/util/eventbus+ tailscale.com/tsweb/varz from tailscale.com/util/usermetric+ tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+ tailscale.com/types/bools from tailscale.com/tsnet+ @@ -922,7 +856,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/types/lazy from tailscale.com/ipn/ipnlocal+ tailscale.com/types/logger from tailscale.com/appc+ tailscale.com/types/logid from tailscale.com/ipn/ipnlocal+ - tailscale.com/types/mapx from tailscale.com/ipn/ipnext + tailscale.com/types/mapx from tailscale.com/ipn/ipnext+ tailscale.com/types/netlogfunc from tailscale.com/net/tstun+ tailscale.com/types/netlogtype from tailscale.com/wgengine/netlog tailscale.com/types/netmap from tailscale.com/control/controlclient+ @@ -944,6 +878,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ LW tailscale.com/util/cmpver from tailscale.com/net/dns+ tailscale.com/util/ctxkey from tailscale.com/client/tailscale/apitype+ 💣 tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting + tailscale.com/util/def from tailscale.com/ipn/localapi L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/appc+ tailscale.com/util/eventbus from tailscale.com/tsd+ @@ -966,16 +901,15 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/util/set from tailscale.com/cmd/k8s-operator+ tailscale.com/util/singleflight from tailscale.com/control/controlclient+ tailscale.com/util/slicesx from tailscale.com/appc+ - tailscale.com/util/syspolicy from tailscale.com/feature/syspolicy tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy/setting+ tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy/internal/metrics+ tailscale.com/util/syspolicy/internal/metrics from tailscale.com/util/syspolicy/source tailscale.com/util/syspolicy/pkey from tailscale.com/control/controlclient+ tailscale.com/util/syspolicy/policyclient from tailscale.com/control/controlclient+ - tailscale.com/util/syspolicy/ptype from tailscale.com/util/syspolicy+ - tailscale.com/util/syspolicy/rsop from tailscale.com/util/syspolicy+ - tailscale.com/util/syspolicy/setting from tailscale.com/util/syspolicy+ - tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy+ + tailscale.com/util/syspolicy/ptype from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/syspolicy/rsop from tailscale.com/ipn/localapi + tailscale.com/util/syspolicy/setting from tailscale.com/client/local+ + tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy/rsop tailscale.com/util/testenv from tailscale.com/control/controlclient+ tailscale.com/util/truncate from tailscale.com/logtail tailscale.com/util/usermetric from tailscale.com/health+ @@ -992,15 +926,13 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ 💣 tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/netlog from tailscale.com/wgengine + tailscale.com/wgengine/netlog from tailscale.com/feature/netlog tailscale.com/wgengine/netstack from tailscale.com/tsnet tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ tailscale.com/wgengine/router from tailscale.com/ipn/ipnlocal+ tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal 💣 tailscale.com/wgengine/wgint from tailscale.com/wgengine+ tailscale.com/wgengine/wglog from tailscale.com/wgengine - tailscale.com/wif from tailscale.com/feature/identityfederation golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ @@ -1023,19 +955,20 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/net/http/httpproxy from tailscale.com/net/tshttpproxy golang.org/x/net/http2 from k8s.io/apimachinery/pkg/util/net+ golang.org/x/net/http2/hpack from golang.org/x/net/http2+ - golang.org/x/net/icmp from github.com/prometheus-community/pro-bing+ + golang.org/x/net/icmp from tailscale.com/net/ping golang.org/x/net/idna from golang.org/x/net/http/httpguts+ golang.org/x/net/internal/httpcommon from golang.org/x/net/http2 + golang.org/x/net/internal/httpsfv from golang.org/x/net/http2 golang.org/x/net/internal/iana from golang.org/x/net/icmp+ - golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ golang.org/x/net/internal/socks from golang.org/x/net/proxy - golang.org/x/net/ipv4 from github.com/prometheus-community/pro-bing+ - golang.org/x/net/ipv6 from github.com/prometheus-community/pro-bing+ + golang.org/x/net/ipv4 from github.com/tailscale/wireguard-go/conn+ + golang.org/x/net/ipv6 from github.com/tailscale/wireguard-go/conn+ golang.org/x/net/proxy from tailscale.com/net/netns D golang.org/x/net/route from tailscale.com/net/netmon+ golang.org/x/net/websocket from tailscale.com/k8s-operator/sessionrecording/ws golang.org/x/oauth2 from golang.org/x/oauth2/clientcredentials+ - golang.org/x/oauth2/clientcredentials from tailscale.com/cmd/k8s-operator+ + golang.org/x/oauth2/clientcredentials from tailscale.com/client/tailscale/v2+ golang.org/x/oauth2/internal from golang.org/x/oauth2+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ golang.org/x/sys/cpu from github.com/tailscale/certstore+ @@ -1092,22 +1025,22 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ - crypto/internal/fips140 from crypto/internal/fips140/aes+ + crypto/internal/fips140 from crypto/fips140+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ crypto/internal/fips140/alias from crypto/cipher+ crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ - crypto/internal/fips140/check from crypto/internal/fips140/aes+ - crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/check from crypto/fips140+ + crypto/internal/fips140/drbg from crypto/hpke+ crypto/internal/fips140/ecdh from crypto/ecdh crypto/internal/fips140/ecdsa from crypto/ecdsa crypto/internal/fips140/ed25519 from crypto/ed25519 crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 crypto/internal/fips140/edwards25519/field from crypto/ecdh+ - crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ + crypto/internal/fips140/hkdf from crypto/hkdf+ crypto/internal/fips140/hmac from crypto/hmac+ crypto/internal/fips140/mlkem from crypto/mlkem - crypto/internal/fips140/nistec from crypto/elliptic+ + crypto/internal/fips140/nistec from crypto/ecdsa+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ @@ -1137,7 +1070,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/sha3 from crypto/internal/fips140hash+ crypto/sha512 from crypto/ecdsa+ crypto/subtle from crypto/cipher+ - crypto/tls from github.com/prometheus-community/pro-bing+ + crypto/tls from github.com/prometheus/client_golang/prometheus/promhttp+ crypto/tls/internal/fips140tls from crypto/tls crypto/x509 from crypto/tls+ D crypto/x509/internal/macos from crypto/x509 @@ -1172,7 +1105,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ hash from compress/zlib+ hash/adler32 from compress/zlib hash/crc32 from compress/gzip+ - hash/fnv from google.golang.org/protobuf/internal/detrand + hash/fnv from google.golang.org/protobuf/internal/detrand+ hash/maphash from go4.org/mem html from html/template+ html/template from tailscale.com/util/eventbus @@ -1187,14 +1120,14 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ internal/filepathlite from os+ internal/fmtsort from fmt+ internal/goarch from crypto/internal/fips140deps/cpu+ - internal/godebug from crypto/internal/fips140deps/godebug+ + internal/godebug from crypto/ed25519+ internal/godebugs from internal/godebug+ internal/goexperiment from net/http/pprof+ internal/goos from crypto/x509+ internal/lazyregexp from go/doc internal/msan from internal/runtime/maps+ internal/nettrace from net+ - internal/oserror from io/fs+ + internal/oserror from internal/syscall/windows+ internal/poll from net+ internal/profile from net/http/pprof internal/profilerecord from runtime+ @@ -1204,9 +1137,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ internal/runtime/atomic from internal/runtime/exithook+ L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime - internal/runtime/gc from runtime+ + internal/runtime/gc from internal/runtime/gc/scan+ internal/runtime/gc/scan from runtime - internal/runtime/maps from reflect+ + internal/runtime/maps from hash/maphash+ internal/runtime/math from internal/runtime/maps+ internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ @@ -1220,7 +1153,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ internal/synctest from sync internal/syscall/execenv from os+ LD internal/syscall/unix from crypto/internal/sysrand+ - W internal/syscall/windows from crypto/internal/sysrand+ + W internal/syscall/windows from crypto/internal/fips140deps/time+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os @@ -1228,7 +1161,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ internal/unsafeheader from internal/reflectlite+ io from bufio+ io/fs from crypto/x509+ - io/ioutil from github.com/godbus/dbus/v5+ + io/ioutil from github.com/google/gnostic-models/compiler+ iter from go/ast+ log from expvar+ log/internal from log+ @@ -1246,7 +1179,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ mime/quotedprintable from mime/multipart net from crypto/tls+ net/http from expvar+ - net/http/httptrace from github.com/prometheus-community/pro-bing+ + net/http/httptrace from github.com/prometheus/client_golang/prometheus/promhttp+ net/http/httputil from tailscale.com/client/web+ net/http/internal from net/http+ net/http/internal/ascii from net/http+ @@ -1265,7 +1198,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ regexp from github.com/davecgh/go-spew/spew+ regexp/syntax from regexp runtime from crypto/internal/fips140+ - runtime/debug from github.com/coder/websocket/internal/xsync+ + runtime/debug from github.com/klauspost/compress/zstd+ runtime/metrics from github.com/prometheus/client_golang/prometheus+ runtime/pprof from net/http/pprof+ runtime/trace from net/http/pprof diff --git a/cmd/k8s-operator/deploy/chart/templates/.gitignore b/cmd/k8s-operator/deploy/chart/templates/.gitignore index 185ea9e2b..ccdea8702 100644 --- a/cmd/k8s-operator/deploy/chart/templates/.gitignore +++ b/cmd/k8s-operator/deploy/chart/templates/.gitignore @@ -10,3 +10,4 @@ /recorder.yaml /tailnet.yaml /proxygrouppolicy.yaml +/peerrelay.yaml diff --git a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml index 0c0cb64cb..d8c3803d5 100644 --- a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml @@ -6,6 +6,9 @@ kind: Deployment metadata: name: operator namespace: {{ .Release.Namespace }} + {{- if .Values.annotations }} + annotations: {{- toYaml .Values.annotations | nindent 4 }} + {{- end }} spec: replicas: 1 strategy: @@ -78,6 +81,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPERATOR_SERVICE_ACCOUNT_NAME + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName - name: OPERATOR_LOGIN_SERVER value: {{ .Values.loginServer }} - name: OPERATOR_INGRESS_CLASS_NAME @@ -117,6 +124,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: OPERATOR_SHARED_ACME_ACCOUNT_KEY + value: {{ .Values.operatorConfig.sharedACMEAccountKey | quote }} {{- with .Values.operatorConfig.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} @@ -146,3 +155,6 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.operatorConfig.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 4d59b4aad..847109c40 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -40,6 +40,9 @@ rules: - apiGroups: ["tailscale.com"] resources: ["tailnets", "tailnets/status"] verbs: ["get", "list", "watch", "update"] +- apiGroups: ["tailscale.com"] + resources: ["peerrelays", "peerrelays/status"] + verbs: ["get", "list", "watch", "update"] - apiGroups: ["tailscale.com"] resources: ["proxygrouppolicies", "proxygrouppolicies/status"] verbs: ["get", "list", "watch", "update"] @@ -76,6 +79,10 @@ rules: - apiGroups: [""] resources: ["secrets", "serviceaccounts", "configmaps"] verbs: ["create","delete","deletecollection","get","list","patch","update","watch"] +- apiGroups: [""] + resources: ["serviceaccounts/token"] + resourceNames: ["operator"] + verbs: ["create"] - apiGroups: [""] resources: ["pods"] verbs: ["get","list","watch", "update"] diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index 8517d77aa..6cb0628dd 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -62,6 +62,9 @@ operatorConfig: resources: {} + # Specifies annotations for deployment + annotations: {} + podAnnotations: {} podLabels: {} @@ -72,6 +75,8 @@ operatorConfig: affinity: {} + priorityClassName: "" + podSecurityContext: {} securityContext: {} @@ -82,6 +87,13 @@ operatorConfig: # - name: EXTRA_VAR2 # value: "value2" + # Default for the tailscale.com/share-acme-account annotation on new + # ProxyGroups. When true, the operator provisions a shared per-tailnet + # ACME account key Secret and configures proxies to use it, preserving + # Let's Encrypt's ARI "replaces" renewal exemption across pod restarts + # and ProxyGroup recreation. See #18251. + sharedACMEAccountKey: false + # In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here ingressClass: # Allows for customization of the ingress class name used by the operator to identify ingresses to reconcile. This does diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml index a819aa651..4d6422ede 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml @@ -104,6 +104,884 @@ spec: description: Pod configuration. type: object properties: + affinity: + description: If specified, applies affinity rules to the pods deployed by the DNSConfig resource. + type: object + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + type: array + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + type: object + required: + - preference + - weight + properties: + preference: + description: A node selector term, associated with the corresponding weight. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + type: integer + format: int32 + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + type: object + required: + - nodeSelectorTerms + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + type: array + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + x-kubernetes-map-type: atomic + x-kubernetes-list-type: atomic + x-kubernetes-map-type: atomic + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + type: integer + format: int32 + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + x-kubernetes-list-type: atomic + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + type: integer + format: int32 + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + x-kubernetes-list-type: atomic + nodeSelector: + description: If specified, applies node selector rules to the pods deployed by the DNSConfig resource. + type: object + additionalProperties: + type: string tolerations: description: If specified, applies tolerations to the pods deployed by the DNSConfig resource. type: array diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_peerrelays.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_peerrelays.yaml new file mode 100644 index 000000000..7b8247b7d --- /dev/null +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_peerrelays.yaml @@ -0,0 +1,264 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: peerrelays.tailscale.com +spec: + group: tailscale.com + names: + kind: PeerRelay + listKind: PeerRelayList + plural: peerrelays + shortNames: + - pr + singular: peerrelay + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Status of the deployed PeerRelay resources. + jsonPath: .status.conditions[?(@.type == "PeerRelayReady")].reason + name: Status + type: string + - description: Public addresses the peer relay replicas are reachable on. + jsonPath: .status.endpoints[*].address + name: Endpoints + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - metadata + - spec + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired state of the PeerRelay. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + type: object + properties: + aws: + description: |- + AWS contains configuration for pinning each replica to a specific AWS Elastic IP and subnet. Only meaningful + when running on EKS with the AWS Load Balancer Controller. When set, the per-replica values override any + aws-load-balancer-eip-allocations or aws-load-balancer-subnets values supplied via spec.service.annotations. + type: object + required: + - elasticIPs + properties: + elasticIPs: + description: |- + ElasticIPs pins each replica to a specific AWS EIP allocation and subnet. Only meaningful when Network Load + Balancers are provisioned by the AWS Load Balancer Controller. ElasticIPs supplies one allocation-subnet pair + per replica: replica N uses ElasticIPs[N]. The list must be at least as long as spec.replicas so every replica + has a distinct EIP; extra entries are permitted so that scale-up doesn't immediately trip validation. + + When set, the reconciler stamps + service.beta.kubernetes.io/aws-load-balancer-eip-allocations and + service.beta.kubernetes.io/aws-load-balancer-subnets on each per-replica Service, overriding any values in + spec.service.annotations. + type: array + minItems: 1 + items: + description: PeerRelayAWSElasticIP pairs an EIP allocation with the subnet in the same AZ. + type: object + required: + - allocationID + - subnetID + properties: + allocationID: + description: |- + AllocationID is the AWS EIP allocation ID (e.g. eipalloc-0123abcd) whose public IP this replica is reachable + on. Stamped as service.beta.kubernetes.io/aws-load-balancer-eip-allocations on the replica's Service. + type: string + pattern: ^eipalloc-[0-9a-f]+$ + subnetID: + description: |- + SubnetID is the AWS subnet in the same availability zone as AllocationID (e.g. subnet-0123abcd). Stamped as + service.beta.kubernetes.io/aws-load-balancer-subnets on the replica's Service so the NLB is provisioned in + the same AZ as the EIP. + type: string + pattern: ^subnet-[0-9a-f]+$ + x-kubernetes-list-type: atomic + hostnamePrefix: + description: |- + HostnamePrefix specifies the hostname prefix for each + replica. Each device will have the integer number + from its StatefulSet pod appended to this prefix to form the full hostname. + HostnamePrefix can contain lower case letters, numbers and dashes, it + must not start with a dash and must be between 1 and 62 characters long. + type: string + pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + proxyClass: + description: |- + ProxyClass is the name of the ProxyClass custom resource that + contains configuration options that should be applied to the + resources created for this PeerRelay. If unset, the operator will + create resources with the default configuration. + type: string + replicas: + description: |- + Replicas specifies how many devices to create. Set this to enable + high availability for peer relays. + https://tailscale.com/kb/1115/high-availability. Defaults to 1. + type: integer + format: int32 + default: 1 + minimum: 0 + service: + description: Service contains configuration values to modify the LoadBalancer service used to expose the peer relay. + type: object + properties: + annotations: + description: |- + Annotations to apply to the LoadBalancer service. Any annotations that conflict with those used by known + cloud providers to ensure IP addresses rather than DNS names are ignored. + type: object + additionalProperties: + type: string + tags: + description: |- + Tags that the Tailscale node will be tagged with. + Defaults to [tag:k8s]. + To autoapprove the device defined by a PeerRelay, + you can configure Tailscale ACLs to give these tags the necessary + permissions. + See https://tailscale.com/kb/1337/acl-syntax#autoapprovers. + If you specify custom tags here, you must also make the operator an owner of these tags. + See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + Tags cannot be changed once a PeerRelay node has been created. + Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + type: array + items: + type: string + pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + tailnet: + description: |- + Tailnet specifies the tailnet this PeerRelay should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: PeerRelay tailnet is immutable + x-kubernetes-validations: + - rule: '!has(self.aws) || !has(self.aws.elasticIPs) || self.aws.elasticIPs.size() >= self.replicas' + message: spec.aws.elasticIPs must contain at least one entry per replica + status: + description: |- + Status describes the status of the PeerRelay. This is set + and managed by the Tailscale operator. + type: object + properties: + conditions: + type: array + items: + description: Condition contains details for one aspect of the current state of this API Resource. + type: object + required: + - lastTransitionTime + - message + - reason + - status + - type + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + type: string + format: date-time + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + type: string + maxLength: 32768 + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + type: integer + format: int64 + minimum: 0 + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + type: string + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + description: status of the condition, one of True, False, Unknown. + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + endpoints: + description: |- + Endpoints lists the public address:port pairs each peer relay replica is reachable on. There is one entry + per replica whose LoadBalancer Service has been assigned a public address; entries appear as the underlying + cloud provisions each Service. + type: array + items: + type: object + required: + - address + - port + - replica + properties: + address: + description: |- + Address is the public IP or hostname the cloud has allocated for this replica's LoadBalancer Service. + Peers reach this relay by connecting to Address:Port over UDP. + type: string + port: + description: Port is the UDP port the peer relay listens on. + type: integer + format: int32 + replica: + description: Replica is the zero-based index of the peer relay replica this endpoint targets. + type: integer + format: int32 + x-kubernetes-list-map-keys: + - replica + x-kubernetes-list-type: map + served: true + storage: true + subresources: + status: {} diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml index 200d83943..5e6dd5880 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml @@ -58,15 +58,18 @@ spec: - credentials properties: credentials: - description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet. + description: Denotes the location of the credentials to use for authenticating with this Tailnet. type: object required: - secretName properties: secretName: description: |- - The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and - "client_secret". + The name of the secret containing the credentials used to authenticate with this Tailnet. The secret must always + contain a "client_id" field. To authenticate with a static OAuth client, also set "client_secret". To authenticate + via workload identity federation, set "audience" to the audience value expected by the Tailscale OAuth + client; the operator will mint a ServiceAccount token for itself with that audience and exchange it for an API + token. "client_secret" and "audience" are mutually exclusive. type: string loginUrl: description: URL of the control plane to be used by all resources managed by the operator using this Tailnet. diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 597641bde..03502be4c 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -442,6 +442,884 @@ spec: pod: description: Pod configuration. properties: + affinity: + description: If specified, applies affinity rules to the pods deployed by the DNSConfig resource. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + nodeSelector: + additionalProperties: + type: string + description: If specified, applies node selector rules to the pods deployed by the DNSConfig resource. + type: object tolerations: description: If specified, applies tolerations to the pods deployed by the DNSConfig resource. items: @@ -585,6 +1463,271 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: peerrelays.tailscale.com +spec: + group: tailscale.com + names: + kind: PeerRelay + listKind: PeerRelayList + plural: peerrelays + shortNames: + - pr + singular: peerrelay + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Status of the deployed PeerRelay resources. + jsonPath: .status.conditions[?(@.type == "PeerRelayReady")].reason + name: Status + type: string + - description: Public addresses the peer relay replicas are reachable on. + jsonPath: .status.endpoints[*].address + name: Endpoints + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired state of the PeerRelay. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + aws: + description: |- + AWS contains configuration for pinning each replica to a specific AWS Elastic IP and subnet. Only meaningful + when running on EKS with the AWS Load Balancer Controller. When set, the per-replica values override any + aws-load-balancer-eip-allocations or aws-load-balancer-subnets values supplied via spec.service.annotations. + properties: + elasticIPs: + description: |- + ElasticIPs pins each replica to a specific AWS EIP allocation and subnet. Only meaningful when Network Load + Balancers are provisioned by the AWS Load Balancer Controller. ElasticIPs supplies one allocation-subnet pair + per replica: replica N uses ElasticIPs[N]. The list must be at least as long as spec.replicas so every replica + has a distinct EIP; extra entries are permitted so that scale-up doesn't immediately trip validation. + + When set, the reconciler stamps + service.beta.kubernetes.io/aws-load-balancer-eip-allocations and + service.beta.kubernetes.io/aws-load-balancer-subnets on each per-replica Service, overriding any values in + spec.service.annotations. + items: + description: PeerRelayAWSElasticIP pairs an EIP allocation with the subnet in the same AZ. + properties: + allocationID: + description: |- + AllocationID is the AWS EIP allocation ID (e.g. eipalloc-0123abcd) whose public IP this replica is reachable + on. Stamped as service.beta.kubernetes.io/aws-load-balancer-eip-allocations on the replica's Service. + pattern: ^eipalloc-[0-9a-f]+$ + type: string + subnetID: + description: |- + SubnetID is the AWS subnet in the same availability zone as AllocationID (e.g. subnet-0123abcd). Stamped as + service.beta.kubernetes.io/aws-load-balancer-subnets on the replica's Service so the NLB is provisioned in + the same AZ as the EIP. + pattern: ^subnet-[0-9a-f]+$ + type: string + required: + - allocationID + - subnetID + type: object + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - elasticIPs + type: object + hostnamePrefix: + description: |- + HostnamePrefix specifies the hostname prefix for each + replica. Each device will have the integer number + from its StatefulSet pod appended to this prefix to form the full hostname. + HostnamePrefix can contain lower case letters, numbers and dashes, it + must not start with a dash and must be between 1 and 62 characters long. + pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + type: string + proxyClass: + description: |- + ProxyClass is the name of the ProxyClass custom resource that + contains configuration options that should be applied to the + resources created for this PeerRelay. If unset, the operator will + create resources with the default configuration. + type: string + replicas: + default: 1 + description: |- + Replicas specifies how many devices to create. Set this to enable + high availability for peer relays. + https://tailscale.com/kb/1115/high-availability. Defaults to 1. + format: int32 + minimum: 0 + type: integer + service: + description: Service contains configuration values to modify the LoadBalancer service used to expose the peer relay. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations to apply to the LoadBalancer service. Any annotations that conflict with those used by known + cloud providers to ensure IP addresses rather than DNS names are ignored. + type: object + type: object + tags: + description: |- + Tags that the Tailscale node will be tagged with. + Defaults to [tag:k8s]. + To autoapprove the device defined by a PeerRelay, + you can configure Tailscale ACLs to give these tags the necessary + permissions. + See https://tailscale.com/kb/1337/acl-syntax#autoapprovers. + If you specify custom tags here, you must also make the operator an owner of these tags. + See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + Tags cannot be changed once a PeerRelay node has been created. + Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + items: + pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + type: string + type: array + tailnet: + description: |- + Tailnet specifies the tailnet this PeerRelay should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - message: PeerRelay tailnet is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: spec.aws.elasticIPs must contain at least one entry per replica + rule: '!has(self.aws) || !has(self.aws.elasticIPs) || self.aws.elasticIPs.size() >= self.replicas' + status: + description: |- + Status describes the status of the PeerRelay. This is set + and managed by the Tailscale operator. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + endpoints: + description: |- + Endpoints lists the public address:port pairs each peer relay replica is reachable on. There is one entry + per replica whose LoadBalancer Service has been assigned a public address; entries appear as the underlying + cloud provisions each Service. + items: + properties: + address: + description: |- + Address is the public IP or hostname the cloud has allocated for this replica's LoadBalancer Service. + Peers reach this relay by connecting to Address:Port over UDP. + type: string + port: + description: Port is the UDP port the peer relay listens on. + format: int32 + type: integer + replica: + description: Replica is the zero-based index of the peer relay replica this endpoint targets. + format: int32 + type: integer + required: + - address + - port + - replica + type: object + type: array + x-kubernetes-list-map-keys: + - replica + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.17.0 @@ -5273,12 +6416,15 @@ spec: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: credentials: - description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet. + description: Denotes the location of the credentials to use for authenticating with this Tailnet. properties: secretName: description: |- - The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and - "client_secret". + The name of the secret containing the credentials used to authenticate with this Tailnet. The secret must always + contain a "client_id" field. To authenticate with a static OAuth client, also set "client_secret". To authenticate + via workload identity federation, set "audience" to the audience value expected by the Tailscale OAuth + client; the operator will mint a ServiceAccount token for itself with that audience and exchange it for an API + token. "client_secret" and "audience" are mutually exclusive. type: string required: - secretName @@ -5454,6 +6600,16 @@ rules: - list - watch - update + - apiGroups: + - tailscale.com + resources: + - peerrelays + - peerrelays/status + verbs: + - get + - list + - watch + - update - apiGroups: - tailscale.com resources: @@ -5531,6 +6687,14 @@ rules: - patch - update - watch + - apiGroups: + - "" + resourceNames: + - operator + resources: + - serviceaccounts/token + verbs: + - create - apiGroups: - "" resources: @@ -5682,6 +6846,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPERATOR_SERVICE_ACCOUNT_NAME + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName - name: OPERATOR_LOGIN_SERVER value: null - name: OPERATOR_INGRESS_CLASS_NAME @@ -5706,6 +6874,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: OPERATOR_SHARED_ACME_ACCOUNT_KEY + value: "false" image: tailscale/k8s-operator:stable imagePullPolicy: Always name: operator diff --git a/cmd/k8s-operator/dnsrecords.go b/cmd/k8s-operator/dnsrecords.go index e75bcd4c2..762223df8 100644 --- a/cmd/k8s-operator/dnsrecords.go +++ b/cmd/k8s-operator/dnsrecords.go @@ -22,6 +22,7 @@ import ( "k8s.io/utils/net" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/util/mak" @@ -106,6 +107,7 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile. if err := dnsRR.maybeProvision(ctx, proxySvc, logger); err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { logger.Infof("optimistic lock error, retrying: %s", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } else { return reconcile.Result{}, err } @@ -281,19 +283,26 @@ func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, proxySv if err := dnsRR.Get(ctx, parentName, ing); err != nil { return "", err } + if len(ing.Status.LoadBalancer.Ingress) == 0 { return "", nil } + return ing.Status.LoadBalancer.Ingress[0].Hostname, nil } + if isManagedByType(proxySvc, serviceTypeSvc) { - svc := new(corev1.Service) - if err := dnsRR.Get(ctx, parentName, svc); apierrors.IsNotFound(err) { - logger.Infof("[unexpected] parent Service for egress proxy %s not found", proxySvc.Name) + var svc corev1.Service + + err := dnsRR.Get(ctx, parentName, &svc) + switch { + case apierrors.IsNotFound(err): + logger.Warnf("parent Service for egress proxy %q not found", proxySvc.Name) return "", nil - } else if err != nil { + case err != nil: return "", err } + return svc.Annotations[AnnotationTailnetTargetFQDN], nil } return "", nil @@ -303,28 +312,31 @@ func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, proxySv // ConfigMap. At this point the in-cluster ts.net nameserver is expected to be // successfully created together with the ConfigMap. func (dnsRR *dnsRecordsReconciler) updateDNSConfig(ctx context.Context, update func(*operatorutils.Records)) error { - cm := &corev1.ConfigMap{} - err := dnsRR.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, cm) - if apierrors.IsNotFound(err) { - dnsRR.logger.Info("[unexpected] dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an issue and attach operator logs.") + var cm corev1.ConfigMap + err := dnsRR.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, &cm) + switch { + case apierrors.IsNotFound(err): + dnsRR.logger.Warn("dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an issue and attach operator logs.") return nil + case err != nil: + return fmt.Errorf("failed to retrieve dnsrecords ConfigMap: %w", err) } - if err != nil { - return fmt.Errorf("error retrieving dnsrecords ConfigMap: %w", err) - } + dnsRecords := operatorutils.Records{Version: operatorutils.Alpha1Version, IP4: map[string][]string{}} if cm.Data != nil && cm.Data[operatorutils.DNSRecordsCMKey] != "" { - if err := json.Unmarshal([]byte(cm.Data[operatorutils.DNSRecordsCMKey]), &dnsRecords); err != nil { + if err = json.Unmarshal([]byte(cm.Data[operatorutils.DNSRecordsCMKey]), &dnsRecords); err != nil { return err } } + update(&dnsRecords) dnsRecordsBs, err := json.Marshal(dnsRecords) if err != nil { return fmt.Errorf("error marshalling DNS records: %w", err) } + mak.Set(&cm.Data, operatorutils.DNSRecordsCMKey, string(dnsRecordsBs)) - return dnsRR.Update(ctx, cm) + return dnsRR.Update(ctx, &cm) } // isSvcForFQDNEgressProxy returns true if the Service is a headless Service diff --git a/cmd/k8s-operator/dnsrecords_test.go b/cmd/k8s-operator/dnsrecords_test.go index c6c5ee029..a2a612351 100644 --- a/cmd/k8s-operator/dnsrecords_test.go +++ b/cmd/k8s-operator/dnsrecords_test.go @@ -8,6 +8,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "testing" @@ -21,6 +22,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -290,6 +293,88 @@ func TestDNSRecordsReconcilerErrorCases(t *testing.T) { } } +func TestDNSRecordsReconcilerOptimisticLockError(t *testing.T) { + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + funcs := interceptor.Funcs{ + Update: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + return errors.New(optimisticLockErrorMsg) + }, + } + + dnsCfg := &tsapi.DNSConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + TypeMeta: metav1.TypeMeta{Kind: "DNSConfig"}, + Spec: tsapi.DNSConfigSpec{Nameserver: &tsapi.Nameserver{}}, + } + dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{ + Type: string(tsapi.NameserverReady), + Status: metav1.ConditionTrue, + }) + + egressSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "lock-service", + Namespace: "default", + Annotations: map[string]string{ + AnnotationTailnetTargetFQDN: "lock-service.example.ts.net", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "unused", + }, + } + + proxyGroupEgressSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ts-proxygroup-egress-abcd1", + Namespace: "tailscale", + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentName: "lock-service", + LabelParentNamespace: "default", + LabelParentType: "svc", + labelProxyGroup: "test-proxy-group", + labelSvcType: typeEgress, + }, + }, + } + + f := fake.NewClientBuilder(). + WithInterceptorFuncs(funcs). + WithScheme(tsapi.GlobalScheme). + WithObjects(dnsCfg, proxyGroupEgressSvc, egressSvc). + WithStatusSubresource(dnsCfg). + Build() + + dnsRR := &dnsRecordsReconciler{ + Client: f, + tsNamespace: "tailscale", + logger: zl.Sugar(), + } + + namespacedName := types.NamespacedName{ + Namespace: proxyGroupEgressSvc.GetNamespace(), + Name: proxyGroupEgressSvc.GetName(), + } + + res, err := dnsRR.Reconcile(t.Context(), reconcile.Request{ + NamespacedName: namespacedName, + }) + + if err != nil { + t.Errorf("expected requeueAfter in result, got error: %s", err) + } + + if res.RequeueAfter == 0 { + t.Errorf("exptected requeueAfter in result to be > 0, got %d", res.RequeueAfter) + } +} + func TestDNSRecordsReconcilerDualStack(t *testing.T) { // Test dual-stack (IPv4 and IPv6) scenarios zl, err := zap.NewDevelopment() diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go index 4eb813a77..bef24ca5a 100644 --- a/cmd/k8s-operator/e2e/ingress_test.go +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -17,9 +17,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "tailscale.com/client/tailscale/v2" kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" + "tailscale.com/tsnet" "tailscale.com/tstest" "tailscale.com/util/httpm" ) @@ -31,12 +33,12 @@ func TestL3Ingress(t *testing.T) { } // Apply nginx - createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) // Apply service to expose it as ingress - name := generateName("test-ingress") svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: generateName("test-ingress"), Namespace: ns, Annotations: map[string]string{ "tailscale.com/expose": "true", @@ -44,7 +46,7 @@ func TestL3Ingress(t *testing.T) { }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ - "app.kubernetes.io/name": "nginx", + "app.kubernetes.io/name": nginx.Name, }, Ports: []corev1.ServicePort{ { @@ -58,7 +60,7 @@ func TestL3Ingress(t *testing.T) { createAndCleanup(t, kubeClient, svc) if err := tstest.WaitFor(time.Minute, func() error { - maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, name)} + maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, svc.Name)} if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { return err } @@ -79,7 +81,7 @@ func TestL3Ingress(t *testing.T) { if err := kubeClient.List(t.Context(), &secrets, client.InNamespace("tailscale"), client.MatchingLabels{ - "tailscale.com/parent-resource": name, + "tailscale.com/parent-resource": svc.Name, "tailscale.com/parent-resource-ns": ns, }, ); err != nil { @@ -109,33 +111,34 @@ func TestL3HAIngress(t *testing.T) { } // Apply nginx. - createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) // Create an ingress ProxyGroup. - createAndCleanup(t, kubeClient, &tsapi.ProxyGroup{ + pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ - Name: "ingress", + Name: generateName("ingress"), }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, - }) + } + createAndCleanup(t, kubeClient, pg) // Apply a Service to expose nginx via the ProxyGroup. - name := generateName("test-ingress") svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: generateName("test-ingress"), Namespace: ns, Annotations: map[string]string{ - "tailscale.com/proxy-group": "ingress", + "tailscale.com/proxy-group": pg.Name, }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: new("tailscale"), Selector: map[string]string{ - "app.kubernetes.io/name": "nginx", + "app.kubernetes.io/name": nginx.Name, }, Ports: []corev1.ServicePort{ { @@ -150,12 +153,12 @@ func TestL3HAIngress(t *testing.T) { var svcIPv4 string forceReconcile := triggerReconcile(t, - client.ObjectKey{Namespace: ns, Name: name}, + client.ObjectKey{Namespace: ns, Name: svc.Name}, &corev1.Service{}, 30*time.Second) // Wait for Service to be ready if err := tstest.WaitFor(5*time.Minute, func() error { - maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, name)} + maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, svc.Name)} forceReconcile() if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { return err @@ -186,15 +189,16 @@ func TestL7Ingress(t *testing.T) { } // Apply nginx Deployment and Service. - createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) createAndCleanup(t, kubeClient, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "nginx", + Name: nginx.Name, Namespace: ns, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ - "app.kubernetes.io/name": "nginx", + "app.kubernetes.io/name": nginx.Name, }, Ports: []corev1.ServicePort{ { @@ -206,13 +210,12 @@ func TestL7Ingress(t *testing.T) { }) // Apply Ingress to expose nginx. - name := generateName("test-ingress") - ingress := l7Ingress(ns, name, map[string]string{}) + ingress := l7Ingress(ns, nginx.Name, map[string]string{}) createAndCleanup(t, kubeClient, ingress) t.Log("Waiting for the Ingress to be ready...") - hostname, err := waitForIngressHostname(t, ns, name) + hostname, err := waitForIngressHostname(t, ns, ingress.Name) if err != nil { t.Fatalf("error waiting for Ingress hostname: %v", err) } @@ -228,15 +231,16 @@ func TestL7HAIngress(t *testing.T) { } // Apply nginx Deployment and Service. - createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) createAndCleanup(t, kubeClient, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "nginx", + Name: nginx.Name, Namespace: ns, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ - "app.kubernetes.io/name": "nginx", + "app.kubernetes.io/name": nginx.Name, }, Ports: []corev1.ServicePort{ { @@ -248,23 +252,23 @@ func TestL7HAIngress(t *testing.T) { }) // Create ProxyGroup that the Ingress will reference. - createAndCleanup(t, kubeClient, &tsapi.ProxyGroup{ + pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ - Name: "ingress", + Name: generateName("ingress"), }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, - }) + } + createAndCleanup(t, kubeClient, pg) // Apply Ingress to expose nginx. - name := generateName("test-ingress") - ingress := l7Ingress(ns, name, map[string]string{"tailscale.com/proxy-group": "ingress"}) + ingress := l7Ingress(ns, nginx.Name, map[string]string{"tailscale.com/proxy-group": pg.Name}) createAndCleanup(t, kubeClient, ingress) t.Log("Waiting for the Ingress to be ready...") - hostname, err := waitForIngressHostname(t, ns, name) + hostname, err := waitForIngressHostname(t, ns, ingress.Name) if err != nil { t.Fatalf("error waiting for Ingress hostname: %v", err) } @@ -274,7 +278,88 @@ func TestL7HAIngress(t *testing.T) { } } -func l7Ingress(namespace, name string, annotations map[string]string) *networkingv1.Ingress { +func TestL7HAIngressMultiTailnet(t *testing.T) { + if tnClient == nil || secondTNClient == nil { + t.Skip("TestL7HAIngressMultiTailnet requires a working tailnet client for a first and second tailnet") + } + + // Apply nginx Deployment and Service. + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: nginx.Name, + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": nginx.Name, + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + // Create Ingress ProxyGroup for each Tailnet. + firstTailnetPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName("first-tailnet"), + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + } + createAndCleanup(t, kubeClient, firstTailnetPG) + secondTailnetPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName("second-tailnet"), + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + Tailnet: "second-tailnet", + }, + } + createAndCleanup(t, kubeClient, secondTailnetPG) + + if err := verifyProxyGroupTailnet(t, firstTailnetPG, tnClient); err != nil { + t.Fatalf("verifying ProxyGroup %s is registered to the correct tailnet: %v", firstTailnetPG.Name, err) + } + if err := verifyProxyGroupTailnet(t, secondTailnetPG, secondTNClient); err != nil { + t.Fatalf("verifying ProxyGroup %s is registered to the correct tailnet: %v", secondTailnetPG.Name, err) + } + + // Apply Ingress to expose nginx. + ingress := l7Ingress(ns, nginx.Name, map[string]string{ + "tailscale.com/proxy-group": secondTailnetPG.Name, + }) + createAndCleanup(t, kubeClient, ingress) + + // Check that the tailscale (VIP) Service has been created in the expected Tailnet. + svcName := "svc:" + ingress.Name + if err := tstest.WaitFor(3*time.Minute, func() error { + _, err := secondTSClient.VIPServices().Get(t.Context(), svcName) + if tailscale.IsNotFound(err) { + return fmt.Errorf("Tailscale service %q not yet in expected tailnet", svcName) + } + return err + }); err != nil { + t.Fatalf("Tailscale service %q never appeared in expected tailnet: %v", svcName, err) + } + hostname, err := waitForIngressHostname(t, ns, ingress.Name) + if err != nil { + t.Fatalf("error waiting for Ingress hostname: %v", err) + } + if err := testIngressIsReachable(t, newHTTPClient(secondTNClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { + t.Fatal(err) + } +} + +func l7Ingress(namespace, svc string, annotations map[string]string) *networkingv1.Ingress { + name := generateName("test-ingress") ingress := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -296,7 +381,7 @@ func l7Ingress(namespace, name string, annotations map[string]string) *networkin PathType: new(networkingv1.PathTypePrefix), Backend: networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ - Name: "nginx", + Name: svc, Port: networkingv1.ServiceBackendPort{ Number: 80, }, @@ -313,26 +398,27 @@ func l7Ingress(namespace, name string, annotations map[string]string) *networkin return ingress } -func nginxDeployment(namespace, name string) *appsv1.Deployment { +func nginxDeployment(namespace string) *appsv1.Deployment { + name := generateName("nginx") return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: map[string]string{ - "app.kubernetes.io/name": "nginx", + "app.kubernetes.io/name": name, }, }, Spec: appsv1.DeploymentSpec{ Replicas: new(int32(1)), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app.kubernetes.io/name": "nginx", + "app.kubernetes.io/name": name, }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app.kubernetes.io/name": "nginx", + "app.kubernetes.io/name": name, }, }, Spec: corev1.PodSpec{ @@ -406,6 +492,56 @@ func testIngressIsReachable(t *testing.T, httpClient *http.Client, url string) e return nil } +// verifyProxyGroupTailnet verifies that a ProxyGroup is registered to the correct tailnet. +// This is done by getting the expected tailnet domain for the tailnet client, +// and comparing this with the actual device fqdn in the ProxyGroup state secret. +func verifyProxyGroupTailnet(t *testing.T, pg *tsapi.ProxyGroup, cl *tsnet.Server) error { + t.Helper() + // Determine the expected tailnet Magic DNS Name. + lc, err := cl.LocalClient() + if err != nil { + return err + } + status, err := lc.Status(t.Context()) + if err != nil { + return err + } + _, expectedTailnet, ok := strings.Cut(strings.TrimSuffix(status.Self.DNSName, "."), ".") + if !ok { + return fmt.Errorf("unexpected DNSName format %q", status.Self.DNSName) + } + // Read the device FQDN from the first state secret for the ProxyGroup, + // and verify that this matches the expected tailnet. + if err := tstest.WaitFor(3*time.Minute, func() error { + var secrets corev1.SecretList + if err := kubeClient.List(t.Context(), &secrets, + client.InNamespace("tailscale"), + client.MatchingLabels{ + kubetypes.LabelSecretType: kubetypes.LabelSecretTypeState, + "tailscale.com/parent-resource-type": "proxygroup", + "tailscale.com/parent-resource": pg.Name, + }, + ); err != nil { + return err + } + if len(secrets.Items) == 0 { + return fmt.Errorf("no state secrets found for ProxyGroup %q yet", pg.Name) + } + fqdn := strings.TrimSuffix(string(secrets.Items[0].Data[kubetypes.KeyDeviceFQDN]), ".") + _, tailnet, ok := strings.Cut(fqdn, ".") + if !ok { + return fmt.Errorf("ProxyGroup %q: device FQDN %q has no domain yet", pg.Name, fqdn) + } + if tailnet != expectedTailnet { + return fmt.Errorf("ProxyGroup %q on wrong tailnet: got domain %q, want %q", pg.Name, tailnet, expectedTailnet) + } + return nil + }); err != nil { + return fmt.Errorf("ProxyGroup %q not on expected tailnet: %v", pg.Name, err) + } + return nil +} + func waitForIngressHostname(t *testing.T, namespace, name string) (string, error) { t.Helper() var hostname string diff --git a/cmd/k8s-operator/e2e/main_test.go b/cmd/k8s-operator/e2e/main_test.go index 02f614014..9eab9e301 100644 --- a/cmd/k8s-operator/e2e/main_test.go +++ b/cmd/k8s-operator/e2e/main_test.go @@ -54,7 +54,7 @@ func createAndCleanup(t *testing.T, cl client.Client, obj client.Object) { t.Cleanup(func() { // Use context.Background() for cleanup, as t.Context() is cancelled // just before cleanup functions are called. - if err = cl.Delete(context.Background(), obj); err != nil { + if err := cl.Delete(context.Background(), obj); err != nil { t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err) } }) @@ -69,7 +69,7 @@ func createAndCleanupErr(t *testing.T, cl client.Client, obj client.Object) erro } t.Cleanup(func() { - if err = cl.Delete(context.Background(), obj); err != nil { + if err := cl.Delete(context.Background(), obj); err != nil { t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err) } }) diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index 55d4652d8..0d4ca80ad 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -4,6 +4,7 @@ package e2e import ( + "bytes" "context" "crypto/rand" "crypto/tls" @@ -39,6 +40,7 @@ import ( "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/storage/driver" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" @@ -69,10 +71,13 @@ const ( ) var ( - tsClient *tailscale.Client // For API calls to control. - tnClient *tsnet.Server // For testing real tailnet traffic. - restCfg *rest.Config // For constructing a client-go client if necessary. - kubeClient client.WithWatch // For k8s API calls. + tsClient *tailscale.Client // For API calls to control. + tnClient *tsnet.Server // For testing real tailnet traffic on first tailnet. + secondTSClient *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet). + secondTNClient *tsnet.Server // For testing real tailnet traffic on second tailnet. + restCfg *rest.Config // For constructing a client-go client if necessary. + kubeClient client.WithWatch // For k8s API calls. + clusterLoginServer string //go:embed certs/pebble.minica.crt pebbleMiniCACert []byte @@ -157,11 +162,11 @@ func runTests(m *testing.M) (int, error) { } var ( - clusterLoginServer string // Login server from cluster Pod point of view. - clientID, clientSecret string // OAuth client for the operator to use. + clientID, clientSecret string // OAuth client for the first tailnet (for the operator to use). caPaths []string // Extra CA cert file paths to add to images. - certsDir = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images. + certsDir = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images. + secondClientID, secondClientSecret string // OAuth client for the second tailnet (for the operator to use). ) if *fDevcontrol { // Deploy pebble and get its certs. @@ -279,7 +284,7 @@ func runTests(m *testing.M) (int, error) { return 0, fmt.Errorf("failed to set policy file: %w", err) } - logger.Infof("ACLs configured") + logger.Info("ACLs configured for first tailnet") key, err := tsClient.Keys().CreateOAuthClient(ctx, tailscale.CreateOAuthClientRequest{ Scopes: []string{"auth_keys", "devices:core", "services"}, @@ -287,36 +292,77 @@ func runTests(m *testing.M) (int, error) { Description: "k8s-operator client for e2e tests", }) if err != nil { - return 0, fmt.Errorf("failed to marshal OAuth client creation request: %w", err) + return 0, fmt.Errorf("failed to create OAuth client for first tailnet: %w", err) } - clientID = key.ID clientSecret = key.Key + + logger.Info("OAuth credentials set for first tailnet") + + // Create second tailnet. The bootstrap credentials returned have 'all' permissions- + // they are used for administrative actions and to create a separately scoped + // Oauth client for the k8s operator. + bootstrapClient, err := createTailnet(ctx, tsClient) + if err != nil { + return 0, fmt.Errorf("failed to create second tailnet: %w", err) + } + + // Set HTTPS on second tailnet. + err = bootstrapClient.TailnetSettings().Update(ctx, tailscale.UpdateTailnetSettingsRequest{HTTPSEnabled: new(true)}) + if err != nil { + return 0, fmt.Errorf("failed to configure https for second tailnet: %w", err) + } + logger.Info("HTTPS settings configured for second tailnet") + + // Set ACLs for second tailnet. + if err = bootstrapClient.PolicyFile().Set(ctx, string(requiredACLs), ""); err != nil { + return 0, fmt.Errorf("failed to set policy file: %w", err) + } + + logger.Info("ACLs configured for second tailnet") + + // Create an OAuth client for the second tailnet to be used + // by the k8s-operator. + secondKey, err := bootstrapClient.Keys().CreateOAuthClient(ctx, tailscale.CreateOAuthClientRequest{ + Scopes: []string{"auth_keys", "devices:core", "services"}, + Tags: []string{"tag:k8s-operator"}, + Description: "k8s-operator client for e2e tests", + }) + if err != nil { + return 0, fmt.Errorf("failed to create OAuth client for second tailnet: %w", err) + } + secondClientID = secondKey.ID + secondClientSecret = secondKey.Key + + secondTSClient, err = tailscaleClientFromSecret(ctx, "http://localhost:31544", secondClientID, secondClientSecret) + if err != nil { + return 0, fmt.Errorf("failed to set up second tailnet client: %w", err) + } + } else { clientSecret = os.Getenv("TS_API_CLIENT_SECRET") if clientSecret == "" { return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") } - // Format is "tskey-client--". - parts := strings.Split(clientSecret, "-") - if len(parts) != 4 { - return 0, fmt.Errorf("TS_API_CLIENT_SECRET is not valid") - } - clientID = parts[2] - credentials := clientcredentials.Config{ - ClientID: clientID, - ClientSecret: clientSecret, - TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL), - Scopes: []string{"auth_keys"}, - } - tk, err := credentials.Token(ctx) + clientID, err = clientIDFromSecret(clientSecret) if err != nil { - return 0, fmt.Errorf("failed to get OAuth token: %w", err) + return 0, fmt.Errorf("failed to get client id from secret: %w", err) } - // An access token will last for an hour which is plenty of time for - // the tests to run. No need for token refresh logic. - tsClient = &tailscale.Client{ - APIKey: tk.AccessToken, + tsClient, err = tailscaleClientFromSecret(ctx, ipn.DefaultControlURL, clientID, clientSecret) + if err != nil { + return 0, fmt.Errorf("failed to set up first tailnet client: %w", err) + } + secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET") + if secondClientSecret == "" { + return 0, fmt.Errorf("must use --devcontrol or set SECOND_TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") + } + secondClientID, err = clientIDFromSecret(secondClientSecret) + if err != nil { + return 0, fmt.Errorf("failed to get client id from secret: %w", err) + } + secondTSClient, err = tailscaleClientFromSecret(ctx, ipn.DefaultControlURL, secondClientID, secondClientSecret) + if err != nil { + return 0, fmt.Errorf("failed to set up second tailnet client: %w", err) } } @@ -446,10 +492,16 @@ func runTests(m *testing.M) (int, error) { authKey, err := tsClient.Keys().CreateAuthKey(ctx, tailscale.CreateKeyRequest{Capabilities: caps}) if err != nil { - return 0, err + return 0, fmt.Errorf("failed to create auth key for first tailnet: %w", err) } defer tsClient.Keys().Delete(context.Background(), authKey.ID) + secondAuthKey, err := secondTSClient.Keys().CreateAuthKey(ctx, tailscale.CreateKeyRequest{Capabilities: caps}) + if err != nil { + return 0, fmt.Errorf("failed to create auth key for second tailnet: %w", err) + } + defer secondTSClient.Keys().Delete(context.Background(), secondAuthKey.ID) + tnClient = &tsnet.Server{ ControlURL: tsClient.BaseURL.String(), Hostname: "test-proxy", @@ -463,9 +515,64 @@ func runTests(m *testing.M) (int, error) { } defer tnClient.Close() + secondTNClient = &tsnet.Server{ + ControlURL: secondTSClient.BaseURL.String(), + Hostname: "test-proxy", + Ephemeral: true, + Store: &mem.Store{}, + AuthKey: secondAuthKey.Key, + } + _, err = secondTNClient.Up(ctx) + if err != nil { + return 0, err + } + defer secondTNClient.Close() + + // Create the tailnet Secret in the tailscale namespace. + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet-credentials", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_id": []byte(secondClientID), + "client_secret": []byte(secondClientSecret), + }, + } + if err := createOrUpdate(ctx, kubeClient, secret); err != nil { + return 0, fmt.Errorf("failed to create second-tailnet-credentials Secret: %w", err) + } + defer kubeClient.Delete(context.Background(), secret) + + // Create the Tailnet resource. + tn := &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet", + }, + Spec: tsapi.TailnetSpec{ + LoginURL: clusterLoginServer, + Credentials: tsapi.TailnetCredentials{ + SecretName: "second-tailnet-credentials", + }, + }, + } + if err := createOrUpdate(ctx, kubeClient, tn); err != nil { + return 0, fmt.Errorf("failed to create second-tailnet Tailnet: %w", err) + } + defer kubeClient.Delete(context.Background(), tn) + return m.Run(), nil } +func clientIDFromSecret(clientSecret string) (string, error) { + // Format is "tskey-client--". + parts := strings.Split(clientSecret, "-") + if len(parts) != 4 { + return "", fmt.Errorf("secret is not valid") + } + return parts[2], nil +} + func upgraderOrInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc { hist := action.NewHistory(cfg) hist.Max = 1 @@ -724,3 +831,65 @@ func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts return nil } + +func createOrUpdate(ctx context.Context, cl client.Client, obj client.Object) error { + if err := cl.Create(ctx, obj); err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + return cl.Update(ctx, obj) + } + return nil +} + +// createTailnet creates a new tailnet and returns a tailscale.Client +// authenticated against it using the bootstrap credentials included in the +// creation response. +func createTailnet(ctx context.Context, tsClient *tailscale.Client) (*tailscale.Client, error) { + tailnetName := fmt.Sprintf("second-tailnet-%d", time.Now().Unix()) + body, err := json.Marshal(map[string]any{"displayName": tailnetName}) + if err != nil { + return nil, fmt.Errorf("failed to marshal tailnet creation request: %w", err) + } + // TODO(beckypauley): change to use a method on tailscale.Client once this is available. + req, _ := http.NewRequestWithContext(ctx, "POST", tsClient.BaseURL.String()+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body)) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tsClient.APIKey)) + resp, err := tsClient.HTTP.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to create tailnet: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b)) + } + var result struct { + OauthClient struct { + ID string `json:"id"` + Secret string `json:"secret"` + } `json:"oauthClient"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return tailscaleClientFromSecret(ctx, tsClient.BaseURL.String(), result.OauthClient.ID, result.OauthClient.Secret) +} + +// tailscaleClientFromSecret exchanges OAuth client credentials for an access token and +// returns a tailscale.Client configured to use it. The token is valid for +// one hour, which is sufficient for the tests to run. No need for refresh logic. +func tailscaleClientFromSecret(ctx context.Context, baseURL, clientID, clientSecret string) (*tailscale.Client, error) { + cfg := clientcredentials.Config{ + ClientID: clientID, + ClientSecret: clientSecret, + TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", baseURL), + } + tk, err := cfg.Token(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get OAuth token for client %q: %w", clientID, err) + } + return &tailscale.Client{ + APIKey: tk.AccessToken, + BaseURL: must.Get(url.Parse(baseURL)), + }, nil +} diff --git a/cmd/k8s-operator/egress-eps.go b/cmd/k8s-operator/egress-eps.go index a248ed888..1f02cfab9 100644 --- a/cmd/k8s-operator/egress-eps.go +++ b/cmd/k8s-operator/egress-eps.go @@ -20,6 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/kube/egressservices" ) @@ -90,9 +91,10 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ lg.Debugf("No egress config found, likely because ProxyGroup has not been created") return res, nil } - cfg, ok := (*cfgs)[tailnetSvc] + + cfg, ok := cfgs[tailnetSvc] if !ok { - lg.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc) + lg.Warnf("configuration for tailnet service %q not found", tailnetSvc) return res, nil } @@ -104,16 +106,19 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ } newEndpoints := make([]discoveryv1.Endpoint, 0) for _, pod := range podList.Items { - ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, lg) + ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, eps.AddressType, lg) if err != nil { return res, fmt.Errorf("error verifying if Pod is ready to route traffic: %w", err) } if !ready { continue // maybe next time } - podIP, err := podIPv4(&pod) // we currently only support IPv4 + podIP, err := podIPForFamily(&pod, eps.AddressType) if err != nil { - return res, fmt.Errorf("error determining IPv4 address for Pod: %w", err) + return res, fmt.Errorf("error determining Pod IP for %s EndpointSlice: %w", eps.AddressType, err) + } + if podIP == "" { + continue // Pod doesn't have an IP for this address family } newEndpoints = append(newEndpoints, discoveryv1.Endpoint{ Hostname: (*string)(&pod.UID), @@ -129,21 +134,25 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ // run a cleanup for deleted Pods etc. eps.Endpoints = newEndpoints if !reflect.DeepEqual(eps, oldEps) { - lg.Infof("Updating EndpointSlice to ensure traffic is routed to ready proxy Pods") - if err := er.Update(ctx, eps); err != nil { + lg.Info("Updating EndpointSlice to ensure traffic is routed to ready proxy Pods") + if err = er.Update(ctx, eps); err != nil { return res, fmt.Errorf("error updating EndpointSlice: %w", err) } } + return res, nil } -func podIPv4(pod *corev1.Pod) (string, error) { +func podIPForFamily(pod *corev1.Pod, addrType discoveryv1.AddressType) (string, error) { for _, ip := range pod.Status.PodIPs { parsed, err := netip.ParseAddr(ip.IP) if err != nil { return "", fmt.Errorf("error parsing IP address %s: %w", ip, err) } - if parsed.Is4() { + switch { + case addrType == discoveryv1.AddressTypeIPv4 && parsed.Is4(): + return parsed.String(), nil + case addrType == discoveryv1.AddressTypeIPv6 && parsed.Is6(): return parsed.String(), nil } } @@ -153,61 +162,76 @@ func podIPv4(pod *corev1.Pod) (string, error) { // podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to // route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service // status written there to the desired service configuration. -func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, lg *zap.SugaredLogger) (bool, error) { +func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, addrType discoveryv1.AddressType, lg *zap.SugaredLogger) (bool, error) { lg = lg.With("proxy_pod", pod.Name) - lg.Debugf("checking whether proxy is ready to route to egress service") + lg.Debug("checking whether proxy is ready to route to egress service") if !pod.DeletionTimestamp.IsZero() { - lg.Debugf("proxy Pod is being deleted, ignore") + lg.Debug("proxy Pod is being deleted, ignore") return false, nil } - podIP, err := podIPv4(&pod) - if err != nil { + podIP, err := podIPForFamily(&pod, addrType) + switch { + case err != nil: return false, fmt.Errorf("error determining Pod IP address: %v", err) - } - if podIP == "" { - lg.Infof("[unexpected] Pod does not have an IPv4 address, and IPv6 is not currently supported") + case podIP == "": + lg.Debugf("Pod does not have an address for family %s", addrType) return false, nil } + stateS := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: pod.Name, Namespace: pod.Namespace, }, } + err = er.Get(ctx, client.ObjectKeyFromObject(stateS), stateS) - if apierrors.IsNotFound(err) { - lg.Debugf("proxy does not have a state Secret, waiting...") + switch { + case apierrors.IsNotFound(err): + lg.Debug("proxy does not yet have a state Secret, waiting...") return false, nil + case err != nil: + return false, fmt.Errorf("error retrieving state Secret: %w", err) } - if err != nil { - return false, fmt.Errorf("error getting state Secret: %w", err) - } + svcStatusBS := stateS.Data[egressservices.KeyEgressServices] if len(svcStatusBS) == 0 { - lg.Debugf("proxy's state Secret does not contain egress services status, waiting...") + lg.Debug("proxy's state Secret does not contain egress services status, waiting...") return false, nil } + svcStatus := &egressservices.Status{} - if err := json.Unmarshal(svcStatusBS, svcStatus); err != nil { + if err = json.Unmarshal(svcStatusBS, svcStatus); err != nil { return false, fmt.Errorf("error unmarshalling egress service status: %w", err) } - if !strings.EqualFold(podIP, svcStatus.PodIPv4) { - lg.Infof("proxy's egress service status is for Pod IP %s, current proxy's Pod IP %s, waiting for the proxy to reconfigure...", svcStatus.PodIPv4, podIP) + var statusIP string + switch addrType { + case discoveryv1.AddressTypeIPv4: + statusIP = svcStatus.PodIPv4 + case discoveryv1.AddressTypeIPv6: + statusIP = svcStatus.PodIPv6 + } + if !strings.EqualFold(podIP, statusIP) { + lg.Infof("proxy's egress service status is for Pod IP %q, current proxy's Pod IP %q, waiting for the proxy to reconfigure...", statusIP, podIP) return false, nil } - st, ok := (*svcStatus).Services[tailnetSvcName] + + st, ok := svcStatus.Services[tailnetSvcName] if !ok { lg.Infof("proxy's state Secret does not have egress service status, waiting...") return false, nil } + if !reflect.DeepEqual(cfg.TailnetTarget, st.TailnetTarget) { - lg.Infof("proxy has configured egress service for tailnet target %v, current target is %v, waiting for proxy to reconfigure...", st.TailnetTarget, cfg.TailnetTarget) + lg.Infof("proxy has configured egress service for tailnet target %q, current target is %q, waiting for proxy to reconfigure...", st.TailnetTarget, cfg.TailnetTarget) return false, nil } + if !reflect.DeepEqual(cfg.Ports, st.Ports) { lg.Debugf("proxy has configured egress service for ports %#+v, wants ports %#+v, waiting for proxy to reconfigure", st.Ports, cfg.Ports) return false, nil } - lg.Debugf("proxy is ready to route traffic to egress service") + + lg.Debug("proxy is ready to route traffic to egress service") return true, nil } diff --git a/cmd/k8s-operator/egress-eps_test.go b/cmd/k8s-operator/egress-eps_test.go index 6335b4eb8..14ed38d34 100644 --- a/cmd/k8s-operator/egress-eps_test.go +++ b/cmd/k8s-operator/egress-eps_test.go @@ -98,7 +98,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { t.Run("pods_are_ready_to_route_traffic", func(t *testing.T) { pod, stateS := podAndSecretForProxyGroup("foo") - stBs := serviceStatusForPodIP(t, svc, pod.Status.PodIPs[0].IP, port) + stBs := serviceStatusForPodIPs(t, svc, pod.Status.PodIPs[0].IP, "", port) mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) { mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) }) @@ -115,8 +115,8 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { expectEqual(t, fc, eps) }) t.Run("status_does_not_match_pod_ip", func(t *testing.T) { - _, stateS := podAndSecretForProxyGroup("foo") // replica Pod has IP 10.0.0.1 - stBs := serviceStatusForPodIP(t, svc, "10.0.0.2", port) // status is for a Pod with IP 10.0.0.2 + _, stateS := podAndSecretForProxyGroup("foo") // replica Pod has IP 10.0.0.1 + stBs := serviceStatusForPodIPs(t, svc, "10.0.0.2", "", port) // status is for a Pod with IP 10.0.0.2 mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) { mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) }) @@ -124,6 +124,117 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { eps.Endpoints = []discoveryv1.Endpoint{} expectEqual(t, fc, eps) }) + + // Dual-stack. + epsV6 := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-ipv6", + Namespace: "operator-ns", + Labels: map[string]string{ + LabelParentName: "test", + LabelParentNamespace: "default", + labelSvcType: typeEgress, + labelProxyGroup: "foo", + }, + }, + AddressType: discoveryv1.AddressTypeIPv6, + } + mustCreate(t, fc, epsV6) + t.Run("dual_stack_pod_ready_to_route", func(t *testing.T) { + mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) + dualPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-0", + Namespace: "operator-ns", + Labels: pgLabels("foo", nil), + UID: "foo", + }, + Status: corev1.PodStatus{ + PodIPs: []corev1.PodIP{{IP: "10.0.0.1"}, {IP: "fd00::1"}}, + }, + } + mustCreate(t, fc, dualPod) + stBs := serviceStatusForPodIPs(t, svc, "10.0.0.1", "fd00::1", port) + mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { + mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) + }) + expectReconciled(t, er, "operator-ns", "foo") + eps.Endpoints = []discoveryv1.Endpoint{{ + Addresses: []string{"10.0.0.1"}, + Hostname: new("foo"), + Conditions: discoveryv1.EndpointConditions{ + Serving: new(true), + Ready: new(true), + Terminating: new(false), + }, + }} + expectEqual(t, fc, eps) + expectReconciled(t, er, "operator-ns", "foo-ipv6") + epsV6.Endpoints = []discoveryv1.Endpoint{{ + Addresses: []string{"fd00::1"}, + Hostname: new("foo"), + Conditions: discoveryv1.EndpointConditions{ + Serving: new(true), + Ready: new(true), + Terminating: new(false), + }, + }} + expectEqual(t, fc, epsV6) + }) + + // IPv6-only. + t.Run("ipv4_only_pod_skipped_for_ipv6_slice", func(t *testing.T) { + mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) + ipv4Pod, _ := podAndSecretForProxyGroup("foo") + mustCreate(t, fc, ipv4Pod) + stBs := serviceStatusForPodIPs(t, svc, "10.0.0.1", "", port) + mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { + mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) + }) + expectReconciled(t, er, "operator-ns", "foo-ipv6") + // IPv4-only pod should not appear in the IPv6 EndpointSlice. + epsV6.Endpoints = []discoveryv1.Endpoint{} + expectEqual(t, fc, epsV6) + }) + ipv6Pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-0", + Namespace: "operator-ns", + Labels: pgLabels("foo", nil), + UID: "foo", + }, + Status: corev1.PodStatus{ + PodIPs: []corev1.PodIP{{IP: "fd00::1"}}, + }, + } + t.Run("ipv6_status_does_not_match_pod_ip", func(t *testing.T) { + mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) + mustCreate(t, fc, ipv6Pod) + stBs := serviceStatusForPodIPs(t, svc, "", "fd00::99", port) + mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { + mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) + }) + expectReconciled(t, er, "operator-ns", "foo-ipv6") + epsV6.Endpoints = []discoveryv1.Endpoint{} + expectEqual(t, fc, epsV6) + }) + t.Run("ipv6_pod_ready_to_route", func(t *testing.T) { + stBs := serviceStatusForPodIPs(t, svc, "", ipv6Pod.Status.PodIPs[0].IP, port) + mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { + mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) + }) + expectReconciled(t, er, "operator-ns", "foo-ipv6") + epsV6.Endpoints = append(epsV6.Endpoints, discoveryv1.Endpoint{ + Addresses: []string{"fd00::1"}, + Hostname: new("foo"), + Conditions: discoveryv1.EndpointConditions{ + Serving: new(true), + Ready: new(true), + Terminating: new(false), + }, + }) + expectEqual(t, fc, epsV6) + }) } func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.ConfigMap { @@ -157,7 +268,7 @@ func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.Config return cm } -func serviceStatusForPodIP(t *testing.T, svc *corev1.Service, ip string, p uint16) []byte { +func serviceStatusForPodIPs(t *testing.T, svc *corev1.Service, ipv4, ipv6 string, p uint16) []byte { t.Helper() ports := make(map[egressservices.PortMap]struct{}) for _, port := range svc.Spec.Ports { @@ -172,7 +283,8 @@ func serviceStatusForPodIP(t *testing.T, svc *corev1.Service, ip string, p uint1 } svcName := tailnetSvcName(svc) st := egressservices.Status{ - PodIPv4: ip, + PodIPv4: ipv4, + PodIPv6: ipv6, Services: map[string]*egressservices.ServiceStatus{svcName: &svcSt}, } bs, err := json.Marshal(st) diff --git a/cmd/k8s-operator/egress-pod-readiness.go b/cmd/k8s-operator/egress-pod-readiness.go index a8f306353..acd68cb13 100644 --- a/cmd/k8s-operator/egress-pod-readiness.go +++ b/cmd/k8s-operator/egress-pod-readiness.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "net/http" + "net/netip" "slices" "strings" "sync/atomic" @@ -23,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/tstime" @@ -87,8 +89,9 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req lg.Debugf("Pod is being deleted, do nothing") return res, nil } + if pod.Labels[LabelParentType] != proxyTypeProxyGroup { - lg.Infof("[unexpected] reconciler called for a Pod that is not a ProxyGroup Pod") + lg.Warn("reconciler called for a Pod that is not a ProxyGroup Pod") return res, nil } @@ -106,10 +109,12 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req if err := er.Get(ctx, types.NamespacedName{Name: proxyGroupName}, pg); err != nil { return res, fmt.Errorf("error getting ProxyGroup %q: %w", proxyGroupName, err) } + if pg.Spec.Type != typeEgress { - lg.Infof("[unexpected] reconciler called for %q ProxyGroup Pod", pg.Spec.Type) + lg.Warnf("reconciler called for %q ProxyGroup Pod", pg.Spec.Type) return res, nil } + // Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup. lbls := map[string]string{ kubetypes.LabelManaged: "true", @@ -223,13 +228,24 @@ func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *c lg.Debugf("Pod does not have health check enabled, unable to verify if it is currently routable via Service") return cannotVerify, nil } - wantsIP, err := podIPv4(pod) - if err != nil { - return -1, fmt.Errorf("error determining Pod's IP address: %w", err) - } - if wantsIP == "" { + // Use the Pod's primary IP (PodIPs[0]) to identify this Pod in the health check + // response. The primary IP family is determined by the cluster's IP family configuration. + + // Note: we do not control which IP family the request uses, so on a dual-stack + // cluster either IPv4 or IPv6 could be used. In either case, a matching IP header + // comfirms the request reached this Pod. + if len(pod.Status.PodIPs) == 0 || pod.Status.PodIPs[0].IP == "" { return podNotReady, nil } + wantsIP := pod.Status.PodIPs[0].IP + parsed, err := netip.ParseAddr(wantsIP) + if err != nil { + return -1, fmt.Errorf("error parsing Pod IP %q: %w", wantsIP, err) + } + header := kubetypes.PodIPv4Header + if parsed.Is6() { + header = kubetypes.PodIPv6Header + } ctx, cancel := context.WithTimeout(ctx, time.Second*3) defer cancel() @@ -246,7 +262,7 @@ func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *c return unreachable, nil } defer resp.Body.Close() - gotIP := resp.Header.Get(kubetypes.PodIPv4Header) + gotIP := resp.Header.Get(header) if gotIP == "" { lg.Debugf("Health check does not return Pod's IP header, unable to verify if Pod is currently routable via Service") return cannotVerify, nil diff --git a/cmd/k8s-operator/egress-pod-readiness_test.go b/cmd/k8s-operator/egress-pod-readiness_test.go index 0cf9108f5..6a087031b 100644 --- a/cmd/k8s-operator/egress-pod-readiness_test.go +++ b/cmd/k8s-operator/egress-pod-readiness_test.go @@ -420,6 +420,44 @@ func TestEgressPodReadiness(t *testing.T) { expectEqual(t, fc, pod) mustDeleteAll(t, fc, pod, svc, svc2, svc3) }) + t.Run("ipv6_only_pod_already_routed_to", func(t *testing.T) { + pod := podTemplate.DeepCopy() + pod.Status.PodIPs = []corev1.PodIP{{IP: "fd00::2"}} + + svc, hep := newSvc("svc", 9002) + mustCreateAll(t, fc, svc, pod) + resp := readyRespsV6("fd00::2", 1) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{hep: resp}, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc) + }) + t.Run("dual_stack_pod", func(t *testing.T) { + pod := podTemplate.DeepCopy() + pod.Status.PodIPs = []corev1.PodIP{{IP: "10.0.0.2"}, {IP: "fd00::2"}} + + svc, hep := newSvc("svc", 9002) + mustCreateAll(t, fc, svc, pod) + // Dual-stack pod: the reconciler uses PodIPs[0] (the primary IP), + // which in this case is IPv4. + resp := readyResps("10.0.0.2", 1) + httpCl := fakeHTTPClient{ + t: t, + state: map[string][]fakeResponse{hep: resp}, + } + rec.httpClient = &httpCl + expectReconciled(t, rec, "operator-ns", pod.Name) + + podSetReady(pod, cl) + expectEqual(t, fc, pod) + mustDeleteAll(t, fc, pod, svc) + }) } func readyResps(ip string, num int) (resps []fakeResponse) { @@ -429,6 +467,13 @@ func readyResps(ip string, num int) (resps []fakeResponse) { return resps } +func readyRespsV6(ip string, num int) (resps []fakeResponse) { + for range num { + resps = append(resps, fakeResponse{statusCode: 200, podIP: ip, header: kubetypes.PodIPv6Header}) + } + return resps +} + func unreadyResps(ip string, num int) (resps []fakeResponse) { for range num { resps = append(resps, fakeResponse{statusCode: 503, podIP: ip}) @@ -513,7 +558,11 @@ func (f *fakeHTTPClient) Do(req *http.Request) (*http.Response, error) { Header: make(http.Header), Body: io.NopCloser(bytes.NewReader([]byte{})), } - r.Header.Add(kubetypes.PodIPv4Header, resp.podIP) + h := kubetypes.PodIPv4Header + if resp.header != "" { + h = resp.header + } + r.Header.Add(h, resp.podIP) return &r, nil } @@ -521,4 +570,5 @@ type fakeResponse struct { err error statusCode int podIP string // for the Pod IP header + header string // header key to use; defaults to PodIPv4Header } diff --git a/cmd/k8s-operator/egress-services-readiness.go b/cmd/k8s-operator/egress-services-readiness.go index 965dc08f8..76a8fdb21 100644 --- a/cmd/k8s-operator/egress-services-readiness.go +++ b/cmd/k8s-operator/egress-services-readiness.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "go.uber.org/zap" @@ -20,9 +21,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tstime" + "tailscale.com/util/set" ) const ( @@ -71,19 +74,57 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re }() crl := egressSvcChildResourceLabels(svc) - eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, esrr.Client, esrr.tsNamespace, crl) - if err != nil { - err = fmt.Errorf("error getting EndpointSlice: %w", err) + epsList := &discoveryv1.EndpointSliceList{} + if err = esrr.List(ctx, epsList, client.InNamespace(esrr.tsNamespace), client.MatchingLabels(crl)); err != nil { + err = fmt.Errorf("error listing EndpointSlices: %w", err) reason = reasonReadinessCheckFailed msg = err.Error() return res, err } - if eps == nil { - lg.Infof("EndpointSlice for Service does not yet exist, waiting...") + if len(epsList.Items) == 0 { + lg.Infof("EndpointSlices for Service do not yet exist, waiting...") reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady st = metav1.ConditionFalse return res, nil } + // If an EndpointSlice for an expected family is missing, we mark the Service as NotReady. + // + // Setting the NotReady condition here is also used for best-effort recovery. The + // egress-svcs-reconciler does not watch EndpointSlices, so a deleted EndpointSlice is only + // recreated when this status change re-triggers a Service reconcile. + // + // TODO(beckypauley): refactor so EndpointSlice recovery is not dependent on Service status. + clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esrr.Client, esrr.tsNamespace, crl) + if err != nil { + err = fmt.Errorf("error retrieving ClusterIP Service: %w", err) + reason = reasonReadinessCheckFailed + msg = err.Error() + return res, err + } + if clusterIPSvc == nil { + lg.Infof("ClusterIP Service for egress Service does not yet exist, waiting...") + reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady + st = metav1.ConditionFalse + return res, nil + } + gotAddrTypes := make(set.Set[discoveryv1.AddressType], len(epsList.Items)) + for _, eps := range epsList.Items { + gotAddrTypes.Add(eps.AddressType) + } + wantAddrTypes, err := addrTypesForClusterIPSvc(clusterIPSvc) + if err != nil { + reason = reasonReadinessCheckFailed + msg = err.Error() + return res, err + } + for _, wantAddrType := range wantAddrTypes { + if !gotAddrTypes.Contains(wantAddrType) { + lg.Infof("EndpointSlice for %s is missing, waiting...", wantAddrType) + reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady + st = metav1.ConditionFalse + return res, nil + } + } pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: svc.Annotations[AnnotationProxyGroup], @@ -118,6 +159,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re } podLabels := pgLabels(pg.Name, nil) var readyReplicas int32 +nextReplica: for i := range replicas { podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i) pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels) @@ -127,24 +169,24 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re msg = err.Error() return res, err } + if pod == nil { - lg.Warnf("[unexpected] ProxyGroup is ready, but replica %d was not found", i) + lg.Warnf("ProxyGroup is ready, but replica %d was not found", i) reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady return res, nil } + lg.Debugf("looking at Pod with IPs %v", pod.Status.PodIPs) - ready := false - for _, ep := range eps.Endpoints { - lg.Debugf("looking at endpoint with addresses %v", ep.Addresses) - if endpointReadyForPod(&ep, pod, lg) { - lg.Debugf("endpoint is ready for Pod") - ready = true - break + for _, eps := range epsList.Items { + lg.Debugf("looking at %s EndpointSlice %s", eps.AddressType, eps.Name) + if !slices.ContainsFunc(eps.Endpoints, func(ep discoveryv1.Endpoint) bool { + return endpointReadyForPod(&ep, pod, eps.AddressType, lg) + }) { + continue nextReplica } } - if ready { - readyReplicas++ - } + lg.Debugf("endpoint is ready for Pod") + readyReplicas++ } msg = fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas) if readyReplicas == 0 { @@ -161,14 +203,18 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re return res, nil } -// endpointReadyForPod returns true if the endpoint is for the Pod's IPv4 address and is ready to serve traffic. -// Endpoint must not be nil. -func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, lg *zap.SugaredLogger) bool { - podIP, err := podIPv4(pod) +// endpointReadyForPod returns true if the endpoint is for the Pod's address (for the given address family) +// and is ready to serve traffic. Endpoint must not be nil. +func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, addrType discoveryv1.AddressType, lg *zap.SugaredLogger) bool { + podIP, err := podIPForFamily(pod, addrType) if err != nil { - lg.Warnf("[unexpected] error retrieving Pod's IPv4 address: %v", err) + lg.Warnf("error retrieving Pod's %s address: %v", addrType, err) return false } + if podIP == "" { + return false + } + // Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this. if len(ep.Addresses) != 1 { return false diff --git a/cmd/k8s-operator/egress-services-readiness_test.go b/cmd/k8s-operator/egress-services-readiness_test.go index 96d76cc4e..c21a111d5 100644 --- a/cmd/k8s-operator/egress-services-readiness_test.go +++ b/cmd/k8s-operator/egress-services-readiness_test.go @@ -47,7 +47,14 @@ func TestEgressServiceReadiness(t *testing.T) { }, }, } - fakeClusterIPSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-app", Namespace: "operator-ns"}} + fakeClusterIPSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app", + Namespace: "operator-ns", + Labels: egressSvcChildResourceLabels(egressSvc), + }, + Spec: corev1.ServiceSpec{ClusterIPs: []string{"10.0.0.1"}}, + } labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) eps := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -63,6 +70,7 @@ func TestEgressServiceReadiness(t *testing.T) { }, } mustCreate(t, fc, egressSvc) + mustCreate(t, fc, fakeClusterIPSvc) setClusterNotReady(egressSvc, cl, zl.Sugar()) t.Run("endpointslice_does_not_exist", func(t *testing.T) { expectReconciled(t, rec, "dev", "my-app") @@ -117,6 +125,212 @@ func TestEgressServiceReadiness(t *testing.T) { }) } +func TestEgressServiceReadinessDualStack(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.ProxyGroup{}). + Build() + zl, _ := zap.NewDevelopment() + cl := tstest.NewClock(tstest.ClockOpts{}) + rec := &egressSvcsReadinessReconciler{ + tsNamespace: "operator-ns", + Client: fc, + logger: zl.Sugar(), + clock: cl, + } + tailnetFQDN := "my-app.tailnetxyz.ts.net" + egressSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app", + Namespace: "dev", + Annotations: map[string]string{ + AnnotationProxyGroup: "dev", + AnnotationTailnetTargetFQDN: tailnetFQDN, + }, + }, + } + fakeClusterIPSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app", + Namespace: "operator-ns", + Labels: egressSvcChildResourceLabels(egressSvc), + }, + Spec: corev1.ServiceSpec{ClusterIPs: []string{"10.0.0.1", "fd00::1"}}, + } + labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) + epsV4 := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app-ipv4", + Namespace: "operator-ns", + Labels: labels, + }, + AddressType: discoveryv1.AddressTypeIPv4, + } + labelsV6 := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) + epsV6 := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app-ipv6", + Namespace: "operator-ns", + Labels: labelsV6, + }, + AddressType: discoveryv1.AddressTypeIPv6, + } + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dev", + }, + Spec: tsapi.ProxyGroupSpec{ + Replicas: new(int32(1)), + Type: tsapi.ProxyGroupTypeEgress, + }, + } + mustCreate(t, fc, egressSvc) + mustCreate(t, fc, fakeClusterIPSvc) + mustCreate(t, fc, epsV4) + mustCreate(t, fc, epsV6) + mustCreate(t, fc, pg) + setPGReady(pg, cl, zl.Sugar()) + mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { + p.Status = pg.Status + }) + + // Create a dual-stack pod. + p := pod(pg, 0) + p.Status.PodIPs = append(p.Status.PodIPs, corev1.PodIP{IP: "fd00::0"}) + mustCreate(t, fc, p) + mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) { + existing.Status.PodIPs = p.Status.PodIPs + }) + + t.Run("not_ready_missing_from_ipv6_slice", func(t *testing.T) { + setEndpointForReplicaWithIP("10.0.0.0", epsV4) + mustUpdate(t, fc, epsV4.Namespace, epsV4.Name, func(e *discoveryv1.EndpointSlice) { + e.Endpoints = epsV4.Endpoints + }) + expectReconciled(t, rec, "dev", "my-app") + setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg)) + expectEqual(t, fc, egressSvc) + }) + t.Run("ready_in_both_slices", func(t *testing.T) { + setEndpointForReplicaWithIP("fd00::", epsV6) + mustUpdate(t, fc, epsV6.Namespace, epsV6.Name, func(e *discoveryv1.EndpointSlice) { + e.Endpoints = epsV6.Endpoints + }) + expectReconciled(t, rec, "dev", "my-app") + setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg)) + expectEqual(t, fc, egressSvc) + }) + t.Run("not_ready_when_ipv6_slice_missing", func(t *testing.T) { + // Delete the IPv6 EndpointSlice while the ClusterIP Service still + // wants an IPv6 family; the Service should report NotReady even though + // the IPv4 EndpointSlice is healthy. + if err := fc.Delete(t.Context(), epsV6); err != nil { + t.Fatalf("error deleting IPv6 EndpointSlice: %v", err) + } + expectReconciled(t, rec, "dev", "my-app") + setClusterNotReady(egressSvc, cl, zl.Sugar()) + expectEqual(t, fc, egressSvc) + }) +} + +func TestEgressServiceReadinessIPv6Only(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.ProxyGroup{}). + Build() + zl, _ := zap.NewDevelopment() + cl := tstest.NewClock(tstest.ClockOpts{}) + rec := &egressSvcsReadinessReconciler{ + tsNamespace: "operator-ns", + Client: fc, + logger: zl.Sugar(), + clock: cl, + } + egressSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app", + Namespace: "dev", + Annotations: map[string]string{ + AnnotationProxyGroup: "dev", + AnnotationTailnetTargetFQDN: "my-app.tailnetxyz.ts.net", + }, + }, + } + fakeClusterIPSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app", + Namespace: "operator-ns", + Labels: egressSvcChildResourceLabels(egressSvc), + }, + Spec: corev1.ServiceSpec{ClusterIPs: []string{"fd00::1"}}, + } + labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) + eps := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app-ipv6", + Namespace: "operator-ns", + Labels: labels, + }, + AddressType: discoveryv1.AddressTypeIPv6, + } + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dev", + }, + } + mustCreate(t, fc, egressSvc) + mustCreate(t, fc, fakeClusterIPSvc) + mustCreate(t, fc, eps) + mustCreate(t, fc, pg) + setPGReady(pg, cl, zl.Sugar()) + mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { + p.Status = pg.Status + }) + + // Create IPv6-only pods. + for i := range pgReplicas(pg) { + p := ipv6OnlyPod(pg, i) + mustCreate(t, fc, p) + mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) { + existing.Status.PodIPs = p.Status.PodIPs + }) + } + + t.Run("no_ready_replicas", func(t *testing.T) { + expectReconciled(t, rec, "dev", "my-app") + setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg)) + expectEqual(t, fc, egressSvc) + }) + t.Run("all_replicas_ready", func(t *testing.T) { + for i := range pgReplicas(pg) { + p := ipv6OnlyPod(pg, i) + setEndpointForReplicaWithIP(p.Status.PodIPs[0].IP, eps) + } + mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) { + e.Endpoints = eps.Endpoints + }) + setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg)) + expectReconciled(t, rec, "dev", "my-app") + expectEqual(t, fc, egressSvc) + }) +} + +func ipv6OnlyPod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod { + labels := pgLabels(pg.Name, nil) + labels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal) + ip := fmt.Sprintf("fd00::%d", ordinal+1) // +1 to avoid fd00::0 normalization issues + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", pg.Name, ordinal), + Namespace: "operator-ns", + Labels: labels, + }, + Status: corev1.PodStatus{ + PodIPs: []corev1.PodIP{{IP: ip}}, + }, + } +} + func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger) { tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, lg) } @@ -166,3 +380,14 @@ func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod { }, } } + +func setEndpointForReplicaWithIP(ip string, eps *discoveryv1.EndpointSlice) { + eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{ + Addresses: []string{ip}, + Conditions: discoveryv1.EndpointConditions{ + Ready: new(true), + Serving: new(true), + Terminating: new(false), + }, + }) +} diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 4949db80a..f935fad64 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "math/rand/v2" + "net/netip" "reflect" "slices" "strings" @@ -30,6 +31,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" @@ -201,6 +203,10 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1 return nil } + if err := esr.ensureEndpointSlices(ctx, svc, clusterIPSvc, lg); err != nil { + return err + } + // Update ExternalName Service to point at the ClusterIP Service. clusterDomain := retrieveClusterDomain(esr.tsNamespace, lg) clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain) @@ -217,6 +223,60 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1 return nil } +// addrTypesForClusterIPSvc returns the EndpointSlice address types (IP families) +// that the given ClusterIP Service supports, derived from its ClusterIPs. +// TODO(beckypauley): this could read Spec.IPFamilies directly instead of parsing +// ClusterIPs to determine the family. +func addrTypesForClusterIPSvc(clusterIPSvc *corev1.Service) ([]discoveryv1.AddressType, error) { + addrTypes := make([]discoveryv1.AddressType, 0, len(clusterIPSvc.Spec.ClusterIPs)) + for _, clusterIP := range clusterIPSvc.Spec.ClusterIPs { + ip, err := netip.ParseAddr(clusterIP) + if err != nil { + return nil, fmt.Errorf("error parsing ClusterIP %q: %w", clusterIP, err) + } + addrType := discoveryv1.AddressTypeIPv4 + if ip.Is6() { + addrType = discoveryv1.AddressTypeIPv6 + } + addrTypes = append(addrTypes, addrType) + } + return addrTypes, nil +} + +// ensureEndpointSlices ensures that EndpointSlices exist for the egress service +// for each IP family supported by the cluster, and that their ports are up to +// date. +func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) error { + crl := egressSvcEpsLabels(svc, clusterIPSvc) + // Only create EndpointSlices for IP families supported by the cluster. + addrTypes, err := addrTypesForClusterIPSvc(clusterIPSvc) + if err != nil { + return err + } + for _, addrType := range addrTypes { + eps := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", clusterIPSvc.Name, strings.ToLower(string(addrType))), + Namespace: esr.tsNamespace, + Labels: crl, + }, + AddressType: addrType, + Ports: epsPortsFromSvc(clusterIPSvc), + } + if _, err := createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) { + e.Labels = eps.Labels + e.AddressType = eps.AddressType + e.Ports = eps.Ports + for _, p := range e.Endpoints { + p.Conditions.Ready = nil + } + }); err != nil { + return fmt.Errorf("error ensuring %s EndpointSlice: %w", addrType, err) + } + } + return nil +} + func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) (*corev1.Service, bool, error) { lg.Infof("updating configuration...") usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName) @@ -315,29 +375,6 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s } } - crl := egressSvcEpsLabels(svc, clusterIPSvc) - // TODO(irbekrm): support IPv6, but need to investigate how kube proxy - // sets up Service -> Pod routing when IPv6 is involved. - eps := &discoveryv1.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-ipv4", clusterIPSvc.Name), - Namespace: esr.tsNamespace, - Labels: crl, - }, - AddressType: discoveryv1.AddressTypeIPv4, - Ports: epsPortsFromSvc(clusterIPSvc), - } - if eps, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) { - e.Labels = eps.Labels - e.AddressType = eps.AddressType - e.Ports = eps.Ports - for _, p := range e.Endpoints { - p.Conditions.Ready = nil - } - }); err != nil { - return nil, false, fmt.Errorf("error ensuring EndpointSlice: %w", err) - } - cm, cfgs, err := egressSvcsConfigs(ctx, esr.Client, proxyGroupName, esr.tsNamespace) if err != nil { return nil, false, fmt.Errorf("error retrieving egress services configuration: %w", err) @@ -347,11 +384,11 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s return nil, false, nil } tailnetSvc := tailnetSvcName(svc) - gotCfg := (*cfgs)[tailnetSvc] + gotCfg := cfgs[tailnetSvc] wantsCfg := egressSvcCfg(svc, clusterIPSvc, esr.tsNamespace, lg) if !reflect.DeepEqual(gotCfg, wantsCfg) { lg.Debugf("updating egress services ConfigMap %s", cm.Name) - mak.Set(cfgs, tailnetSvc, wantsCfg) + mak.Set(&cfgs, tailnetSvc, wantsCfg) bs, err := json.Marshal(cfgs) if err != nil { return nil, false, fmt.Errorf("error marshalling egress services configs: %w", err) @@ -485,19 +522,19 @@ func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, lggr.Debugf("ConfigMap does not contain egress service configs") return nil } - cfgs := &egressservices.Configs{} - if err := json.Unmarshal(bs, cfgs); err != nil { + cfgs := egressservices.Configs{} + if err := json.Unmarshal(bs, &cfgs); err != nil { return fmt.Errorf("error unmarshalling egress services configs") } tailnetSvc := tailnetSvcName(svc) - _, ok := (*cfgs)[tailnetSvc] + _, ok := cfgs[tailnetSvc] if !ok { lggr.Debugf("ConfigMap does not contain egress service config, likely because it was already deleted") return nil } - lggr.Infof("before deleting config %+#v", *cfgs) - delete(*cfgs, tailnetSvc) - lggr.Infof("after deleting config %+#v", *cfgs) + lggr.Infof("before deleting config %+#v", cfgs) + delete(cfgs, tailnetSvc) + lggr.Infof("after deleting config %+#v", cfgs) bs, err := json.Marshal(cfgs) if err != nil { return fmt.Errorf("error marshalling egress services configs: %w", err) @@ -649,7 +686,7 @@ func isEgressSvcForProxyGroup(obj client.Object) bool { // egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well // as unmarshalled configuration from the ConfigMap. -func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs *egressservices.Configs, err error) { +func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs egressservices.Configs, err error) { name := pgEgressCMName(proxyGroupName) cm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -664,9 +701,9 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts if err != nil { return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err) } - cfgs = &egressservices.Configs{} + cfgs = egressservices.Configs{} if len(cm.BinaryData[egressservices.KeyEgressServices]) != 0 { - if err := json.Unmarshal(cm.BinaryData[egressservices.KeyEgressServices], cfgs); err != nil { + if err := json.Unmarshal(cm.BinaryData[egressservices.KeyEgressServices], &cfgs); err != nil { return nil, nil, fmt.Errorf("error unmarshaling egress services config %v: %w", cm.BinaryData[egressservices.KeyEgressServices], err) } } diff --git a/cmd/k8s-operator/egress-services_test.go b/cmd/k8s-operator/egress-services_test.go index 8443a1573..20efe195d 100644 --- a/cmd/k8s-operator/egress-services_test.go +++ b/cmd/k8s-operator/egress-services_test.go @@ -21,6 +21,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" "tailscale.com/tstest" @@ -49,6 +51,9 @@ func TestTailscaleEgressServices(t *testing.T) { WithScheme(tsapi.GlobalScheme). WithObjects(pg, cm). WithStatusSubresource(pg). + WithInterceptorFuncs(interceptor.Funcs{ + Create: clusterIPInterceptor("10.96.0.1"), + }). Build() zl, err := zap.NewDevelopment() if err != nil { @@ -116,6 +121,23 @@ func TestTailscaleEgressServices(t *testing.T) { validateReadyService(t, fc, esr, svc, clock, zl, cm) }) + t.Run("endpointslice_deletion_recovery", func(t *testing.T) { + name := findGenNameForEgressSvcResources(t, fc, svc) + epsName := fmt.Sprintf("%s-ipv4", name) + // Delete the EndpointSlice and verify it is recreated. + eps := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: epsName, + Namespace: "operator-ns", + }, + } + if err := fc.Delete(t.Context(), eps); err != nil { + t.Fatalf("error deleting EndpointSlice: %v", err) + } + expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) + validateReadyService(t, fc, esr, svc, clock, zl, cm) + }) + t.Run("delete_external_name_service", func(t *testing.T) { name := findGenNameForEgressSvcResources(t, fc, svc) if err := fc.Delete(context.Background(), svc); err != nil { @@ -134,10 +156,10 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco expectReconciled(t, esr, "default", "test") // Verify that a ClusterIP Service has been created. name := findGenNameForEgressSvcResources(t, fc, svc) - expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc) + expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc, removeClusterIPsFromSvc) clusterSvc := mustGetClusterIPSvc(t, fc, name) // Verify that an EndpointSlice has been created. - expectEqual(t, fc, endpointSlice(name, svc, clusterSvc)) + expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4)) // Verify that ConfigMap contains configuration for the new egress service. mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm, zl) r := svcConfiguredReason(svc, true, zl.Sugar()) @@ -223,18 +245,22 @@ func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Se return svc } -func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service) *discoveryv1.EndpointSlice { +func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service, addrType discoveryv1.AddressType) *discoveryv1.EndpointSlice { labels := egressSvcChildResourceLabels(extNSvc) labels[discoveryv1.LabelManagedBy] = "tailscale.com" labels[discoveryv1.LabelServiceName] = name + suffix := "ipv4" + if addrType == discoveryv1.AddressTypeIPv6 { + suffix = "ipv6" + } return &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-ipv4", name), + Name: fmt.Sprintf("%s-%s", name, suffix), Namespace: "operator-ns", Labels: labels, }, Ports: portsForEndpointSlice(clusterIPSvc), - AddressType: discoveryv1.AddressTypeIPv4, + AddressType: addrType, } } @@ -284,13 +310,155 @@ func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressser if !ok { return nil } - cfgs := &egressservices.Configs{} - if err := json.Unmarshal(cfgBs, cfgs); err != nil { + cfgs := egressservices.Configs{} + if err := json.Unmarshal(cfgBs, &cfgs); err != nil { t.Fatalf("error unmarshalling config: %v", err) } - cfg, ok := (*cfgs)[svcName] + cfg, ok := cfgs[svcName] if ok { return &cfg } return nil } + +func TestTailscaleEgressServicesDualStack(t *testing.T) { + pg := &tsapi.ProxyGroup{ + TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + UID: types.UID("1234-UID"), + }, + Spec: tsapi.ProxyGroupSpec{ + Replicas: pointer.To[int32](3), + Type: tsapi.ProxyGroupTypeEgress, + }, + } + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgEgressCMName("foo"), + Namespace: "operator-ns", + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg, cm). + WithStatusSubresource(pg). + WithInterceptorFuncs(interceptor.Funcs{ + Create: clusterIPInterceptor("10.96.0.1", "fd00::1"), + }). + Build() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + clock := tstest.NewClock(tstest.ClockOpts{}) + + esr := &egressSvcsReconciler{ + Client: fc, + logger: zl.Sugar(), + clock: clock, + tsNamespace: "operator-ns", + } + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + AnnotationTailnetTargetFQDN: "foo.bar.ts.net.", + AnnotationProxyGroup: "foo", + }, + }, + Spec: corev1.ServiceSpec{ + ExternalName: "placeholder", + Type: corev1.ServiceTypeExternalName, + Selector: nil, + Ports: []corev1.ServicePort{ + { + Protocol: "TCP", + Port: 80, + }, + }, + }, + } + + t.Run("dual_stack_creates_both_endpoint_slices", func(t *testing.T) { + mustCreate(t, fc, svc) + expectReconciled(t, esr, "default", "test") + validateReadyService(t, fc, esr, svc, clock, zl, cm) + // Also verify the IPv6 EndpointSlice was created. + name := findGenNameForEgressSvcResources(t, fc, svc) + clusterSvc := mustGetClusterIPSvc(t, fc, name) + expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) + }) + + t.Run("dual_stack_endpointslice_deletion_recovery", func(t *testing.T) { + name := findGenNameForEgressSvcResources(t, fc, svc) + // Delete both IPv4 and IPv6 EndpointSlices. + for _, suffix := range []string{"ipv4", "ipv6"} { + epsName := fmt.Sprintf("%s-%s", name, suffix) + eps := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: epsName, + Namespace: "operator-ns", + }, + } + if err := fc.Delete(t.Context(), eps); err != nil { + t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err) + } + expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) + } + // Reconcile should recreate both. + validateReadyService(t, fc, esr, svc, clock, zl, cm) + clusterSvc := mustGetClusterIPSvc(t, fc, name) + expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) + }) + + t.Run("dual_stack_single_endpointslice_deletion_recovery", func(t *testing.T) { + name := findGenNameForEgressSvcResources(t, fc, svc) + // Delete only the IPv6 EndpointSlice. + epsName := fmt.Sprintf("%s-ipv6", name) + eps := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: epsName, + Namespace: "operator-ns", + }, + } + if err := fc.Delete(t.Context(), eps); err != nil { + t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err) + } + expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) + // Reconcile should recreate the missing IPv6 EndpointSlice while leaving + // the IPv4 one untouched. + validateReadyService(t, fc, esr, svc, clock, zl, cm) + clusterSvc := mustGetClusterIPSvc(t, fc, name) + expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) + expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4)) + }) + + t.Run("delete_dual_stack_service", func(t *testing.T) { + name := findGenNameForEgressSvcResources(t, fc, svc) + if err := fc.Delete(context.Background(), svc); err != nil { + t.Fatalf("error deleting ExternalName Service: %v", err) + } + expectReconciled(t, esr, "default", "test") + expectMissing[corev1.Service](t, fc, "operator-ns", name) + expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name)) + expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv6", name)) + mustNotHaveConfigForSvc(t, fc, svc, cm) + }) +} + +// clusterIPInterceptor returns an interceptor.Funcs Create function that +// simulates the API server assigning ClusterIPs to ClusterIP Services. +// This is required because the reconciler iterates ClusterIPs to create +// per-family EndpointSlices but the fake client does not assign ClusterIPs. +func clusterIPInterceptor(clusterIPs ...string) func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + return func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if svc, ok := obj.(*corev1.Service); ok && svc.Spec.Type == corev1.ServiceTypeClusterIP { + svc.Spec.ClusterIPs = clusterIPs + svc.Spec.ClusterIP = clusterIPs[0] + } + return c.Create(ctx, obj, opts...) + } +} diff --git a/cmd/k8s-operator/generate/main.go b/cmd/k8s-operator/generate/main.go index 840812ea3..b3e44c117 100644 --- a/cmd/k8s-operator/generate/main.go +++ b/cmd/k8s-operator/generate/main.go @@ -28,6 +28,7 @@ const ( proxyGroupCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml" tailnetCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_tailnets.yaml" proxyGroupPolicyCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygrouppolicies.yaml" + peerRelayCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_peerrelays.yaml" helmTemplatesPath = operatorDeploymentFilesPath + "/chart/templates" connectorCRDHelmTemplatePath = helmTemplatesPath + "/connector.yaml" proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml" @@ -36,6 +37,7 @@ const ( proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml" tailnetCRDHelmTemplatePath = helmTemplatesPath + "/tailnet.yaml" proxyGroupPolicyCRDHelmTemplatePath = helmTemplatesPath + "/proxygrouppolicy.yaml" + peerRelayCRDHelmTemplatePath = helmTemplatesPath + "/peerrelay.yaml" helmConditionalStart = "{{ if .Values.installCRDs -}}\n" helmConditionalEnd = "{{- end -}}" @@ -160,6 +162,7 @@ func generate(baseDir string) error { {proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath}, {tailnetCRDPath, tailnetCRDHelmTemplatePath}, {proxyGroupPolicyCRDPath, proxyGroupPolicyCRDHelmTemplatePath}, + {peerRelayCRDPath, peerRelayCRDHelmTemplatePath}, } { if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil { return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err) @@ -178,6 +181,7 @@ func cleanup(baseDir string) error { proxyGroupCRDHelmTemplatePath, tailnetCRDHelmTemplatePath, proxyGroupPolicyCRDHelmTemplatePath, + peerRelayCRDHelmTemplatePath, } { if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) { return fmt.Errorf("error cleaning up %s: %w", path, err) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 37d0ed014..f76a7da2a 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -173,14 +173,14 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin logger.Infof("error validating tailscale IngressClass: %v.", err) return false, nil } - // Get and validate ProxyGroup readiness + + // We only act on services that are annotated as using a proxy group. pgName := ing.Annotations[AnnotationProxyGroup] if pgName == "" { - logger.Infof("[unexpected] no ProxyGroup annotation, skipping Tailscale Service provisioning") return false, nil } - logger = logger.With("ProxyGroup", pgName) + logger = logger.With("ProxyGroup", pgName) if !tsoperator.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") return false, nil @@ -455,8 +455,10 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger if err := r.List(ctx, ingList); err != nil { return false, fmt.Errorf("listing Ingresses: %w", err) } - serveConfigChanged := false - // For each Tailscale Service in serve config... + + // Collect orphans first so we are not mutating cfg.Services during + // iteration. + var orphans []tailcfg.ServiceName for tsSvcName := range cfg.Services { // ...check if there is currently an Ingress with this hostname found := false @@ -469,40 +471,23 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger } if !found { - logger.Infof("Tailscale Service %q is not owned by any Ingress, cleaning up", tsSvcName) - tsService, err := tsClient.VIPServices().Get(ctx, tsSvcName.String()) - switch { - case tailscale.IsNotFound(err): - return false, nil - case err != nil: - return false, fmt.Errorf("getting Tailscale Service %q: %w", tsSvcName, err) - } - - // Delete the Tailscale Service from control if necessary. - svcsChanged, err = r.cleanupTailscaleService(ctx, tsService, logger, tsClient) - if err != nil { - return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err) - } - - // Make sure the Tailscale Service is not advertised in tailscaled or serve config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil { - return false, fmt.Errorf("failed to update tailscaled config services: %w", err) - } - - _, ok := cfg.Services[tsSvcName] - if ok { - logger.Infof("Removing Tailscale Service %q from serve config", tsSvcName) - delete(cfg.Services, tsSvcName) - serveConfigChanged = true - } - - if err = cleanupCertResources(ctx, r.Client, r.tsNamespace, tsSvcName, pg); err != nil { - return false, fmt.Errorf("failed to clean up cert resources: %w", err) - } + orphans = append(orphans, tsSvcName) } } - if serveConfigChanged { + // 1. Remove all orphans from serve config in a single ConfigMap Update + // so the proxy cancels every cert loop before we start deleting + // VIPServices, and we only pay one fsnotify propagation window. + updated := false + for _, tsSvcName := range orphans { + logger.Infof("Tailscale Service %q is not owned by any Ingress, cleaning up", tsSvcName) + _, ok := cfg.Services[tsSvcName] + if ok { + delete(cfg.Services, tsSvcName) + updated = true + } + } + if updated { cfgBytes, err := json.Marshal(cfg) if err != nil { return false, fmt.Errorf("marshaling serve config: %w", err) @@ -511,7 +496,37 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger if err := r.Update(ctx, cm); err != nil { return false, fmt.Errorf("updating serve config: %w", err) } + logger.Infof("Removed Tailscale Services from serve config: %v", orphans) } + + for _, tsSvcName := range orphans { + // 2. Unadvertise the Tailscale Service in tailscaled config. + if err := r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil { + return svcsChanged, fmt.Errorf("failed to update tailscaled config services: %w", err) + } + + // 3. Delete the Tailscale Service from the control plane. + tsService, err := tsClient.VIPServices().Get(ctx, tsSvcName.String()) + switch { + case tailscale.IsNotFound(err): + // Already gone at the control plane; continue with cluster + // cleanup rather than aborting the sweep. + case err != nil: + return svcsChanged, fmt.Errorf("getting Tailscale Service %q: %w", tsSvcName, err) + default: + updated, err := r.cleanupTailscaleService(ctx, tsService, logger, tsClient) + if err != nil { + return svcsChanged, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err) + } + svcsChanged = svcsChanged || updated + } + + // 4. Clean up cluster cert resources. + if err := cleanupCertResources(ctx, r.Client, r.tsNamespace, tsSvcName, pg); err != nil { + return svcsChanged, fmt.Errorf("failed to clean up cert resources: %w", err) + } + } + return svcsChanged, nil } @@ -519,6 +534,10 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger // Ingress is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the Tailscale Service is only // deleted if it does not contain any other owner references. If it does the cleanup only removes the owner reference // corresponding to this Ingress. +// +// Steps are ordered so the proxy cancels its cert loop (via serve config +// removal) before the VIPService is deleted; otherwise the loop retries +// against a domain the control plane no longer recognises. func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsclient.Client, pg *tsapi.ProxyGroup) (svcChanged bool, err error) { logger.Debugf("Ensuring any resources for Ingress are cleaned up") ix := slices.Index(ing.Finalizers, FinalizerNamePG) @@ -543,49 +562,53 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, err = r.deleteFinalizer(ctx, ing, logger) }() - // 1. Check if there is a Tailscale Service associated with this Ingress. cm, cfg, err := r.proxyGroupServeConfig(ctx, pg.Name) if err != nil { return false, fmt.Errorf("error getting ProxyGroup serve config: %w", err) } - // Tailscale Service is always first added to serve config and only then created in the Tailscale API, so if it is not - // found in the serve config, we can assume that there is no Tailscale Service. (If the serve config does not exist at - // all, it is possible that the ProxyGroup has been deleted before cleaning up the Ingress, so carry on with - // cleanup). - if cfg != nil && cfg.Services != nil && cfg.Services[serviceName] == nil { - return false, nil + // 1. Remove the Tailscale Service from the proxy's serve config. The proxy + // picks up the change via fsnotify on the mounted ConfigMap and cancels + // its cert loop for this domain before we proceed to delete the + // VIPService. + if cfg != nil && cfg.Services != nil { + if _, ok := cfg.Services[serviceName]; ok { + logger.Infof("Removing TailscaleService %q from serve config for ProxyGroup %q", hostname, pg.Name) + delete(cfg.Services, serviceName) + cfgBytes, err := json.Marshal(cfg) + if err != nil { + return false, fmt.Errorf("error marshaling serve config: %w", err) + } + mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes) + if err := r.Update(ctx, cm); err != nil { + return false, fmt.Errorf("error updating serve config: %w", err) + } + } } - // 2. Clean up the Tailscale Service resources. + // 2. Unadvertise the Tailscale Service in each proxy's tailscaled config. + // Skipped if the ProxyGroup itself has been deleted (no config Secrets to + // update). + if cfg != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil { + return false, fmt.Errorf("failed to update tailscaled config services: %w", err) + } + } + + // 3. Delete the Tailscale Service from the control plane. By now the + // proxy has stopped serving HTTPS for the domain and stopped trying to + // renew its cert. svcChanged, err = r.cleanupTailscaleService(ctx, svc, logger, tsClient) if err != nil { return false, fmt.Errorf("error deleting Tailscale Service: %w", err) } - // 3. Clean up any cluster resources + // 4. Clean up cluster cert resources (TLS Secret + RBAC). if err = cleanupCertResources(ctx, r.Client, r.tsNamespace, serviceName, pg); err != nil { return false, fmt.Errorf("failed to clean up cert resources: %w", err) } - if cfg == nil || cfg.Services == nil { // user probably deleted the ProxyGroup - return svcChanged, nil - } - - // 4. Unadvertise the Tailscale Service in tailscaled config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil { - return false, fmt.Errorf("failed to update tailscaled config services: %w", err) - } - - // 5. Remove the Tailscale Service from the serve config for the ProxyGroup. - logger.Infof("Removing TailscaleService %q from serve config for ProxyGroup %q", hostname, pg.Name) - delete(cfg.Services, serviceName) - cfgBytes, err := json.Marshal(cfg) - if err != nil { - return false, fmt.Errorf("error marshaling serve config: %w", err) - } - mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes) - return svcChanged, r.Update(ctx, cm) + return svcChanged, nil } func (r *HAIngressReconciler) deleteFinalizer(ctx context.Context, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error { @@ -685,9 +708,10 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki // It is invalid to have multiple Ingress resources for the same Tailscale Service in one cluster. ingList := &networkingv1.IngressList{} if err := r.List(ctx, ingList); err != nil { - errs = append(errs, fmt.Errorf("[unexpected] error listing Ingresses: %w", err)) + errs = append(errs, fmt.Errorf("failed to list ingresses: %w", err)) return errors.Join(errs...) } + for _, i := range ingList.Items { if r.shouldExpose(&i) && hostnameForIngress(&i) == hostname && i.UID != ing.UID { errs = append(errs, fmt.Errorf("found duplicate Ingress %q for hostname %q - multiple Ingresses for the same hostname in the same cluster are not allowed", client.ObjectKeyFromObject(&i), hostname)) @@ -876,14 +900,16 @@ func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string] } if svc == nil { c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} - json, err := json.Marshal(c) + data, err := json.Marshal(c) if err != nil { - return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service's owner annotation contents: %w, please report this", err) + return nil, fmt.Errorf("failed to marshal Tailscale Service's owner annotation contents: %w", err) } + return map[string]string{ - ownerAnnotation: string(json), + ownerAnnotation: string(data), }, nil } + o, err := parseOwnerAnnotation(svc) if err != nil { return nil, err @@ -1081,7 +1107,7 @@ func certResourceLabels(pgName, domain string) map[string]string { return map[string]string{ kubetypes.LabelManaged: "true", labelProxyGroup: pgName, - labelDomain: domain, + labelDomain: tsoperator.TruncateLabelValue(domain), } } diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 4952e789f..8ea6d2f47 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "net" "slices" "strings" "sync" @@ -364,7 +365,7 @@ func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl clien proto = "https+insecure://" } mak.Set(&handlers, path, &ipn.HTTPHandler{ - Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path, + Proxy: proto + net.JoinHostPort(svc.Spec.ClusterIP, fmt.Sprint(port)) + path, }) } addIngressBackend(ing.Spec.DefaultBackend, "/") diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index c2a1198cc..92065b43d 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -942,3 +942,91 @@ func TestTailscaleIngressWithHTTPRedirect(t *testing.T) { t.Errorf("incorrect status ports after removing redirect: got %v, want %v", ing.Status.LoadBalancer.Ingress[0].Ports, wantPorts) } } + +func TestTailscaleIngressIPv6(t *testing.T) { + fc := fake.NewFakeClient(ingressClass()) + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + // Create a Service with an IPv6 ClusterIP + ipv6Svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ipv6", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "fda9:e575:6e22:2::25", + Ports: []corev1.ServicePort{ + { + Port: 2283, + Name: "http", + }, + }, + }, + } + mustCreate(t, fc, ipv6Svc) + + // Create an Ingress that routes to the IPv6 service + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ipv6", + Namespace: "default", + UID: "1234-UID-IPV6", + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test-ipv6", + Port: networkingv1.ServiceBackendPort{ + Number: 2283, + }, + }, + }, + }, + } + mustCreate(t, fc, ing) + + ingR := &IngressReconciler{ + Client: fc, + ingressClassName: "tailscale", + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(&fakeTSClient{}), + tsnetServer: &fakeTSNetServer{certDomains: []string{"test-host"}}, + defaultTags: []string{"tag:test"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + expectReconciled(t, ingR, "default", "test-ipv6") + + // Verify the generated serveConfig has properly bracketed IPv6 address + fullName, _ := findGenName(t, fc, "default", "test-ipv6", "ingress") + opts := configOpts{ + replicas: new(int32(1)), + stsName: "tailscale-ipv6-ingress-test-ipv6", + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test-ipv6-ingress", + app: kubetypes.AppIngressResource, + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://[fda9:e575:6e22:2::25]:2283/"}, + }}, + }, + }, + } + // expectedSecret hardcodes the parent-resource label to "test", so fix it for our IPv6 test + secret := expectedSecret(t, fc, opts) + secret.Labels[LabelParentName] = "test-ipv6" + expectEqual(t, fc, secret) +} diff --git a/cmd/k8s-operator/metrics_resources.go b/cmd/k8s-operator/metrics_resources.go index c7c329a7e..0f76693f4 100644 --- a/cmd/k8s-operator/metrics_resources.go +++ b/cmd/k8s-operator/metrics_resources.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" ) @@ -54,7 +55,7 @@ type ServiceMonitorSpec struct { JobLabel string `json:"jobLabel"` // NamespaceSelector selects the namespace of Service(s) that this ServiceMonitor allows to scrape. // https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L88 - NamespaceSelector ServiceMonitorNamespaceSelector `json:"namespaceSelector,omitempty"` + NamespaceSelector ServiceMonitorNamespaceSelector `json:"namespaceSelector"` // Selector is the label selector for Service(s) that this ServiceMonitor allows to scrape. // https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L85 Selector metav1.LabelSelector `json:"selector"` @@ -227,13 +228,13 @@ func metricsResourceLabels(opts *metricsOpts) map[string]string { kubetypes.LabelManaged: "true", labelMetricsTarget: opts.proxyStsName, labelPromProxyType: opts.proxyType, - labelPromProxyParentName: opts.proxyLabels[LabelParentName], + labelPromProxyParentName: kube.TruncateLabelValue(opts.proxyLabels[LabelParentName]), } // Include namespace label for proxies created for a namespaced type. if isNamespacedProxyType(opts.proxyType) { - lbls[labelPromProxyParentNamespace] = opts.proxyLabels[LabelParentNamespace] + lbls[labelPromProxyParentNamespace] = kube.TruncateLabelValue(opts.proxyLabels[LabelParentNamespace]) } - lbls[labelPromJob] = promJobName(opts) + lbls[labelPromJob] = kube.TruncateLabelValue(promJobName(opts)) return lbls } @@ -250,11 +251,11 @@ func promJobName(opts *metricsOpts) string { func metricsSvcSelector(proxyLabels map[string]string, proxyType string) map[string]string { sel := map[string]string{ labelPromProxyType: proxyType, - labelPromProxyParentName: proxyLabels[LabelParentName], + labelPromProxyParentName: kube.TruncateLabelValue(proxyLabels[LabelParentName]), } // Include namespace label for proxies created for a namespaced type. if isNamespacedProxyType(proxyType) { - sel[labelPromProxyParentNamespace] = proxyLabels[LabelParentNamespace] + sel[labelPromProxyParentNamespace] = kube.TruncateLabelValue(proxyLabels[LabelParentNamespace]) } return sel } diff --git a/cmd/k8s-operator/nameserver.go b/cmd/k8s-operator/nameserver.go index 869e5bb26..f5565e5d3 100644 --- a/cmd/k8s-operator/nameserver.go +++ b/cmd/k8s-operator/nameserver.go @@ -190,6 +190,8 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa } if tsDNSCfg.Spec.Nameserver.Pod != nil { dCfg.tolerations = tsDNSCfg.Spec.Nameserver.Pod.Tolerations + dCfg.affinity = tsDNSCfg.Spec.Nameserver.Pod.Affinity + dCfg.nodeSelector = tsDNSCfg.Spec.Nameserver.Pod.NodeSelector } for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} { @@ -217,14 +219,16 @@ type deployable struct { } type deployConfig struct { - replicas int32 - imageRepo string - imageTag string - labels map[string]string - ownerRefs []metav1.OwnerReference - namespace string - clusterIP string - tolerations []corev1.Toleration + replicas int32 + imageRepo string + imageTag string + labels map[string]string + ownerRefs []metav1.OwnerReference + namespace string + clusterIP string + tolerations []corev1.Toleration + affinity *corev1.Affinity + nodeSelector map[string]string } var ( @@ -250,6 +254,8 @@ var ( d.ObjectMeta.Labels = cfg.labels d.ObjectMeta.OwnerReferences = cfg.ownerRefs d.Spec.Template.Spec.Tolerations = cfg.tolerations + d.Spec.Template.Spec.Affinity = cfg.affinity + d.Spec.Template.Spec.NodeSelector = cfg.nodeSelector updateF := func(oldD *appsv1.Deployment) { oldD.Spec = d.Spec } diff --git a/cmd/k8s-operator/nameserver_test.go b/cmd/k8s-operator/nameserver_test.go index e35c72fc0..3ec00d5ed 100644 --- a/cmd/k8s-operator/nameserver_test.go +++ b/cmd/k8s-operator/nameserver_test.go @@ -43,6 +43,9 @@ func TestNameserverReconciler(t *testing.T) { ClusterIP: "5.4.3.2", }, Pod: &tsapi.NameserverPod{ + NodeSelector: map[string]string{ + "foo": "bar", + }, Tolerations: []corev1.Toleration{ { Key: "some-key", @@ -51,6 +54,23 @@ func TestNameserverReconciler(t *testing.T) { Effect: corev1.TaintEffectNoSchedule, }, }, + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "some-key", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"some-value"}, + }, + }, + }, + }, + }, + }, + }, }, }, }, @@ -97,6 +117,26 @@ func TestNameserverReconciler(t *testing.T) { Effect: corev1.TaintEffectNoSchedule, }, } + wantsDeploy.Spec.Template.Spec.Affinity = &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "some-key", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"some-value"}, + }, + }, + }, + }, + }, + }, + } + wantsDeploy.Spec.Template.Spec.NodeSelector = map[string]string{ + "foo": "bar", + } expectEqual(t, fc, wantsDeploy) }) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index c0ef96a68..1663046f4 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -55,6 +55,7 @@ import ( "tailscale.com/ipn/store/kubestore" apiproxy "tailscale.com/k8s-operator/api-proxy" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/peerrelay" "tailscale.com/k8s-operator/reconciler/proxygrouppolicy" "tailscale.com/k8s-operator/reconciler/tailnet" "tailscale.com/k8s-operator/tsclient" @@ -95,8 +96,10 @@ func main() { tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") defaultProxyClass = defaultEnv("PROXY_DEFAULT_CLASS", "") isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) + sharedACMEAccountKey = defaultBool("OPERATOR_SHARED_ACME_ACCOUNT_KEY", false) loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/") ingressClassName = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale") + operatorSAName = defaultEnv("OPERATOR_SERVICE_ACCOUNT_NAME", "operator") ) var opts []kzap.Opts @@ -157,6 +160,7 @@ func main() { tsServer: s, tsClient: tsc, tailscaleNamespace: tsNamespace, + operatorSAName: operatorSAName, restConfig: restConfig, proxyImage: image, k8sProxyImage: k8sProxyImage, @@ -167,6 +171,7 @@ func main() { defaultProxyClass: defaultProxyClass, loginServer: loginServer, ingressClassName: ingressClassName, + sharedACMEAccountKey: sharedACMEAccountKey, }) } @@ -349,6 +354,7 @@ func runReconcilers(opts reconcilerOpts) { tailnetOptions := tailnet.ReconcilerOptions{ Client: mgr.GetClient(), TailscaleNamespace: opts.tailscaleNamespace, + OperatorSAName: opts.operatorSAName, Clock: tstime.DefaultClock{}, Logger: opts.log, Registry: clients, @@ -366,6 +372,19 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not register proxygrouppolicy reconciler: %v", err) } + peerRelayOptions := peerrelay.ReconcilerOptions{ + Client: mgr.GetClient(), + TailscaleNamespace: opts.tailscaleNamespace, + ProxyImage: opts.proxyImage, + DefaultTags: strings.Split(opts.proxyTags, ","), + Clients: clients, + Logger: opts.log, + } + + if err = peerrelay.NewReconciler(peerRelayOptions).Register(mgr); err != nil { + startlog.Fatalf("could not register peerrelay reconciler: %v", err) + } + svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler) svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc")) // If a ProxyClass changes, enqueue all Services labeled with that @@ -692,12 +711,14 @@ func runReconcilers(opts reconcilerOpts) { Watches(&rbacv1.Role{}, recorderFilter). Watches(&rbacv1.RoleBinding{}, recorderFilter). Complete(&RecorderReconciler{ - recorder: eventRecorder, - tsNamespace: opts.tailscaleNamespace, - Client: mgr.GetClient(), - log: opts.log.Named("recorder-reconciler"), - clock: tstime.DefaultClock{}, - clients: clients, + recorder: eventRecorder, + tsNamespace: opts.tailscaleNamespace, + Client: mgr.GetClient(), + log: opts.log.Named("recorder-reconciler"), + clock: tstime.DefaultClock{}, + clients: clients, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), }) if err != nil { startlog.Fatalf("could not create Recorder reconciler: %v", err) @@ -733,6 +754,7 @@ func runReconcilers(opts reconcilerOpts) { proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog)) saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog)) + acmeSecretFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(acmeAccountsSecretHandlerForProxyGroup(mgr.GetClient(), opts.tailscaleNamespace, opts.sharedACMEAccountKey, startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyGroup{}). Named("proxygroup-reconciler"). @@ -741,6 +763,9 @@ func runReconcilers(opts reconcilerOpts) { Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter). Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup). Watches(&corev1.Secret{}, ownedByProxyGroupFilter). + // The shared ACME accounts Secret has no ProxyGroup owner ref, so + // watch it by name to react to its deletion/recreation. + Watches(&corev1.Secret{}, acmeSecretFilterForProxyGroup). Watches(&rbacv1.Role{}, ownedByProxyGroupFilter). Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter). Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup). @@ -761,6 +786,8 @@ func runReconcilers(opts reconcilerOpts) { loginServer: opts.tsServer.ControlURL, authKeyRateLimits: make(map[string]*rate.Limiter), authKeyReissuing: make(map[string]bool), + + sharedACMEAccountKey: opts.sharedACMEAccountKey, }) if err != nil { startlog.Fatalf("could not create ProxyGroup reconciler: %v", err) @@ -816,6 +843,17 @@ type reconcilerOpts struct { // ingressClassName is the name of the ingress class used by reconcilers of Ingress resources. This defaults // to "tailscale" but can be customised. ingressClassName string + // sharedACMEAccountKey is the operator-wide default for the + // shared-ACME-account feature. When true, every ProxyGroup uses the + // shared per-tailnet account key unless the ProxyGroup explicitly + // opts out via tailscale.com/share-acme-account=false. When false, + // ProxyGroups opt in individually via + // tailscale.com/share-acme-account=true. + sharedACMEAccountKey bool + // operatorSAName is the name of the ServiceAccount that the operator pod runs as. It is used as the target + // ServiceAccount when minting tokens via the Kubernetes TokenRequest API for Tailnets that authenticate using + // workload identity federation. + operatorSAName string } // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each @@ -1207,6 +1245,30 @@ func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLog } } +// acmeAccountsSecretHandlerForProxyGroup enqueues ProxyGroups that use the +// shared ACME account when the shared ACME accounts Secret changes. The +// Secret carries no owner reference, so the owner-based Secret watch never +// matches it. +func acmeAccountsSecretHandlerForProxyGroup(cl client.Client, tsNamespace string, sharedACMEAccountDefault bool, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + if o.GetName() != kubetypes.ACMEAccountsSecretName || o.GetNamespace() != tsNamespace { + return nil + } + pgList := new(tsapi.ProxyGroupList) + if err := cl.List(ctx, pgList); err != nil { + logger.Debugf("error listing ProxyGroups for shared ACME accounts Secret: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0, len(pgList.Items)) + for _, pg := range pgList.Items { + if sharedACMEAccountEnabled(&pg, sharedACMEAccountDefault) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + } + } + return reqs + } +} + // serviceHandlerForIngress returns a handler for Service events for ingress // reconciler that ensures that if the Service associated with an event is of // interest to the reconciler, the associated Ingress(es) gets be reconciled. @@ -1468,9 +1530,10 @@ func HAIngressesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler. return func(ctx context.Context, o client.Object) []reconcile.Request { secret, ok := o.(*corev1.Secret) if !ok { - logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret") + logger.Warn("Secret handler triggered for an object that is not a Secret") return nil } + if isTLSSecret(secret) { return []reconcile.Request{ { @@ -1507,15 +1570,16 @@ func HAIngressesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler. } } -// HAServiceFromSecret returns a handler that returns reconcile requests for +// HAServicesFromSecret returns a handler that returns reconcile requests for // all HA Services that should be reconciled in response to a Secret event. func HAServicesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { secret, ok := o.(*corev1.Secret) if !ok { - logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret") + logger.Warn("Secret handler triggered for an object that is not a Secret") return nil } + if !isPGStateSecret(secret) { return nil } @@ -1547,9 +1611,10 @@ func kubeAPIServerPGsFromSecret(cl client.Client, logger *zap.SugaredLogger) han return func(ctx context.Context, o client.Object) []reconcile.Request { secret, ok := o.(*corev1.Secret) if !ok { - logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret") + logger.Warn("Secret handler triggered for an object that is not a Secret") return nil } + if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" || secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" { return nil @@ -1585,9 +1650,10 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) return func(ctx context.Context, o client.Object) []reconcile.Request { pg, ok := o.(*tsapi.ProxyGroup) if !ok { - logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup") + logger.Warn("ProxyGroup handler triggered for an object that is not a ProxyGroup") return nil } + if pg.Spec.Type != tsapi.ProxyGroupTypeEgress { return nil } @@ -1615,9 +1681,10 @@ func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger) return func(ctx context.Context, o client.Object) []reconcile.Request { pg, ok := o.(*tsapi.ProxyGroup) if !ok { - logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup") + logger.Warn("ProxyGroup handler triggered for an object that is not a ProxyGroup") return nil } + if pg.Spec.Type != tsapi.ProxyGroupTypeIngress { return nil } @@ -1645,9 +1712,10 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns return func(ctx context.Context, o client.Object) []reconcile.Request { svc, ok := o.(*corev1.Service) if !ok { - logger.Infof("[unexpected] Service handler triggered for an object that is not a Service") + logger.Warn("Service handler triggered for an object that is not a Service") return nil } + if !isEgressSvcForProxyGroup(svc) { return nil } @@ -1674,9 +1742,10 @@ func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) h return func(ctx context.Context, o client.Object) []reconcile.Request { eps, ok := o.(*discoveryv1.EndpointSlice) if !ok { - logger.Infof("[unexpected] EndpointSlice handler triggered for an object that is not a EndpointSlice") + logger.Warn("EndpointSlice handler triggered for an object that is not a EndpointSlice") return nil } + if eps.Labels[labelProxyGroup] == "" { return nil } @@ -1713,18 +1782,21 @@ func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) return func(ctx context.Context, o client.Object) []reconcile.Request { crd, ok := o.(*apiextensionsv1.CustomResourceDefinition) if !ok { - logger.Debugf("[unexpected] ServiceMonitor CRD handler received an object that is not a CustomResourceDefinition") + logger.Warn("ServiceMonitor CRD handler received an object that is not a CustomResourceDefinition") return nil } + if crd.Name != serviceMonitorCRD { - logger.Debugf("[unexpected] ServiceMonitor CRD handler received an unexpected CRD %q", crd.Name) + logger.Warnf("ServiceMonitor CRD handler received an unexpected CRD %q", crd.Name) return nil } + pcl := &tsapi.ProxyClassList{} if err := cl.List(ctx, pcl); err != nil { - logger.Debugf("[unexpected] error listing ProxyClasses: %v", err) + logger.Errorf("failed to list ProxyClass resources: %v", err) return nil } + reqs := make([]reconcile.Request, 0) for _, pc := range pcl.Items { if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && pc.Spec.Metrics.ServiceMonitor.Enable { @@ -1733,6 +1805,7 @@ func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) }) } } + return reqs } } @@ -1742,9 +1815,10 @@ func crdTransformer(log *zap.SugaredLogger) toolscache.TransformFunc { return func(o any) (any, error) { crd, ok := o.(*apiextensionsv1.CustomResourceDefinition) if !ok { - log.Infof("[unexpected] CRD transformer called for a non-CRD type") + log.Warn("CRD transformer called for a non-CRD type") return crd, nil } + crd.Spec = apiextensionsv1.CustomResourceDefinitionSpec{} return crd, nil } diff --git a/cmd/k8s-operator/proxyclass.go b/cmd/k8s-operator/proxyclass.go index c0ea46116..1484cf448 100644 --- a/cmd/k8s-operator/proxyclass.go +++ b/cmd/k8s-operator/proxyclass.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tstime" @@ -170,10 +171,11 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl } } } + if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && pc.Spec.Metrics.ServiceMonitor.Enable { found, err := hasServiceMonitorCRD(ctx, pcr.Client) if err != nil { - pcr.logger.Infof("[unexpected]: error retrieving %q CRD: %v", serviceMonitorCRD, err) + pcr.logger.Errorf("error retrieving %q CRD: %v", serviceMonitorCRD, err) // best effort validation - don't error out here } else if !found { msg := fmt.Sprintf("ProxyClass defines that a ServiceMonitor custom resource should be created, but %q CRD was not found", serviceMonitorCRD) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 4bd015701..b4955d7af 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -56,6 +56,7 @@ const ( reasonProxyGroupCreating = "ProxyGroupCreating" reasonProxyGroupInvalid = "ProxyGroupInvalid" reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable" + reasonACMEAccountsPendingDeletion = "ACMEAccountsPendingDeletion" // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" @@ -102,6 +103,14 @@ type ProxyGroupReconciler struct { apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge authKeyRateLimits map[string]*rate.Limiter // per-ProxyGroup rate limiters for auth key re-issuance. authKeyReissuing map[string]bool + + // sharedACMEAccountKey is the operator-wide default for the + // shared-ACME-account feature. When true, every ProxyGroup uses the + // shared per-tailnet account key unless the ProxyGroup explicitly + // opts out via tailscale.com/share-acme-account=false. When false, + // only ProxyGroups annotated with tailscale.com/share-acme-account=true + // use it. + sharedACMEAccountKey bool } func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger { @@ -354,7 +363,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl } } - role := pgRole(pg, r.tsNamespace) + role := pgRole(pg, r.tsNamespace, r.sharedACMEAccountEnabledFor(pg)) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { r.ObjectMeta.Labels = role.ObjectMeta.Labels r.ObjectMeta.Annotations = role.ObjectMeta.Annotations @@ -394,13 +403,36 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl }); err != nil { return r.notReadyErrf(pg, logger, "error provisioning ingress ConfigMap %q: %w", cm.Name, err) } + + // Ensure the shared ACME accounts Secret exists (with finalizer) + // when this ProxyGroup opts into the feature. Proxy pods + // populate its fields on first cert issuance. See #18251. + if r.sharedACMEAccountEnabledFor(pg) { + acmeSecret := pgACMEAccountSecret(r.tsNamespace) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, acmeSecret, func(existing *corev1.Secret) { + if !existing.DeletionTimestamp.IsZero() { + // Deletion can't be undone; warn so the account keys + // get backed up before the finalizer is removed. + msg := fmt.Sprintf("shared ACME accounts Secret %q is marked for deletion but retained by the %q finalizer. Its data remains readable until the finalizer is removed - back it up first to preserve the ACME account keys.", existing.Name, kubetypes.ACMEAccountsFinalizer) + r.recorder.Event(existing, corev1.EventTypeWarning, reasonACMEAccountsPendingDeletion, msg) + logger.Warn(msg) + return + } + existing.Labels = acmeSecret.Labels + if !slices.Contains(existing.Finalizers, kubetypes.ACMEAccountsFinalizer) { + existing.Finalizers = append(existing.Finalizers, kubetypes.ACMEAccountsFinalizer) + } + }); err != nil { + return r.notReadyErrf(pg, logger, "error provisioning shared ACME accounts Secret %q: %w", acmeSecret.Name, err) + } + } } defaultImage := r.tsProxyImage if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { defaultImage = r.k8sProxyImage } - ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass) + ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass, r.sharedACMEAccountEnabledFor(pg)) if err != nil { return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err) } @@ -1104,9 +1136,35 @@ func (r *ProxyGroupReconciler) findStaticEndpoints(ctx context.Context, existing return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to find any `status.addresses` of type %q on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass %q", corev1.NodeExternalIP, proxyClass.Name)} } + // If we ended up selecting the same set of addresses already in use, keep + // the existing order. nodes.Items from r.List is not guaranteed to be in + // a stable order across calls, so without this the slice can permute on + // each reconcile, making the marshalled config Secret differ byte-for-byte + // even though nothing has effectively changed. That trips the DeepEqual + // check on the config Secret, which writes the Secret, which fires a + // watch event, which re-enqueues the ProxyGroup, and so on. + if len(currAddrs) > 0 && sameAddrPortSet(endpoints, currAddrs) { + return currAddrs, nil + } + return endpoints, nil } +// sameAddrPortSet reports whether a and b contain the same AddrPorts, +// ignoring order. Both slices are assumed to be free of duplicates, which +// holds for callers in this package. +func sameAddrPortSet(a, b []netip.AddrPort) bool { + if len(a) != len(b) { + return false + } + for _, x := range a { + if !slices.Contains(b, x) { + return false + } + } + return true +} + func getStaticEndpointAddress(a *corev1.NodeAddress, port uint16) *netip.AddrPort { addr, err := netip.ParseAddr(a.Address) if err != nil { @@ -1160,6 +1218,9 @@ func (r *ProxyGroupReconciler) ensureStateRemovedForProxyGroup(pg *tsapi.ProxyGr gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) delete(r.authKeyRateLimits, pg.Name) + for i := range pgReplicas(pg) { + delete(r.authKeyReissuing, pgStateSecretName(pg.Name, i)) + } } func pgTailscaledConfig(pg *tsapi.ProxyGroup, loginServer string, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string) (tailscaledConfigs, error) { @@ -1318,6 +1379,25 @@ func notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, }, nil } +// sharedACMEAccountEnabledFor reports whether the shared-ACME-account +// feature should be applied to pg. The per-PG +// tailscale.com/share-acme-account annotation wins when set; otherwise +// the operator's OPERATOR_SHARED_ACME_ACCOUNT_KEY setting is the default +// for every ProxyGroup. +func (r *ProxyGroupReconciler) sharedACMEAccountEnabledFor(pg *tsapi.ProxyGroup) bool { + return sharedACMEAccountEnabled(pg, r.sharedACMEAccountKey) +} + +// sharedACMEAccountEnabled reports whether pg should use the shared ACME +// account, with the tailscale.com/share-acme-account annotation overriding +// the operator-wide default. +func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool { + if v, ok := pg.Annotations[AnnotationShareACMEAccount]; ok { + return v == "true" + } + return operatorDefault +} + func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { err := fmt.Errorf(format, a...) if strings.Contains(err.Error(), optimisticLockErrorMsg) { diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 60b4bddd5..fc9b532b5 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/yaml" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" "tailscale.com/kube/ingressservices" @@ -63,8 +64,12 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor } // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be -// applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { +// applied over the top after. shareACMEAccount, when true, injects the env +// vars that route the pod's ACME account key to the shared per-tailnet +// Secret and drops TS_DEBUG_ACME_FORCE_RENEWAL so ARI-based renewals are +// attempted; the caller is responsible for checking the operator setting +// and the PG opt-in annotation. +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass, shareACMEAccount bool) (*appsv1.StatefulSet, error) { if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { return kubeAPIServerStatefulSet(pg, namespace, image, port) } @@ -74,10 +79,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string } // Validate some base assumptions. if len(ss.Spec.Template.Spec.InitContainers) != 1 { - return nil, fmt.Errorf("[unexpected] base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers)) + return nil, fmt.Errorf("base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers)) } if len(ss.Spec.Template.Spec.Containers) != 1 { - return nil, fmt.Errorf("[unexpected] base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers)) + return nil, fmt.Errorf("base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers)) } // StatefulSet config. @@ -186,14 +191,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)", }, - { - // This ensures that cert renewals can succeed if ACME account - // keys have changed since issuance. We cannot guarantee or - // validate that the account key has not changed, see - // https://github.com/tailscale/tailscale/issues/18251 - Name: "TS_DEBUG_ACME_FORCE_RENEWAL", - Value: "true", - }, } if port != nil { @@ -251,6 +248,29 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Value: "true", }, ) + if shareACMEAccount { + envs = append(envs, + corev1.EnvVar{ + Name: "TS_ACME_ACCOUNT_SECRET_NAME", + Value: kubetypes.ACMEAccountsSecretName, + }, + corev1.EnvVar{ + Name: "TS_ACME_ACCOUNT_FIELD", + Value: pgACMEAccountField(pg), + }, + ) + } else { + // Without a shared account key we cannot guarantee that + // the account key that issued the previous cert is the + // same one attempting renewal. Force plain new-order flow + // so renewals do not silently fail on rejected ARI + // "replaces" claims. See + // https://github.com/tailscale/tailscale/issues/18251. + envs = append(envs, corev1.EnvVar{ + Name: "TS_DEBUG_ACME_FORCE_RENEWAL", + Value: "true", + }) + } } return append(c.Env, envs...) }() @@ -406,7 +426,7 @@ func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAcc } } -func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { +func pgRole(pg *tsapi.ProxyGroup, namespace string, shareACMEAccount bool) *rbacv1.Role { return &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: pg.Name, @@ -438,6 +458,12 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { pgPodName(pg.Name, i), // State. ) } + // Ingress ProxyGroup write replicas need access to the + // shared ACME account Secret so they can read the + // per-tailnet account key and write it on first use. + if pg.Spec.Type == tsapi.ProxyGroupTypeIngress && shareACMEAccount { + secrets = append(secrets, kubetypes.ACMEAccountsSecretName) + } return secrets }(), }, @@ -476,6 +502,35 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { } } +// pgACMEAccountField returns the field name used inside the shared +// tailscale-acme-accounts Secret for this ProxyGroup's tailnet. The blank +// tailnet (operator-default credentials) is represented by a reserved +// identifier so it gets a stable, unique field. +func pgACMEAccountField(pg *tsapi.ProxyGroup) string { + tn := pg.Spec.Tailnet + if tn == "" { + tn = kubetypes.ACMEAccountDefaultKey + } + return tn + kubetypes.ACMEAccountKeySuffix +} + +// pgACMEAccountSecret returns the shared per-tailnet ACME account key +// Secret, keyed by tailnet inside its data. Not owned by any ProxyGroup +// so it outlives ProxyGroup deletion. +func pgACMEAccountSecret(namespace string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubetypes.ACMEAccountsSecretName, + Namespace: namespace, + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + }, + // Block accidental deletion. + Finalizers: []string{kubetypes.ACMEAccountsFinalizer}, + }, + } +} + // kube-apiserver proxies in auth mode use a static ServiceAccount. Everything // else uses a per-ProxyGroup ServiceAccount. func pgServiceAccountName(pg *tsapi.ProxyGroup) string { diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 95eb7bd5f..ebd452559 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -811,6 +811,90 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { } } +// TestFindStaticEndpointsStableOrder verifies that findStaticEndpoints returns +// the existing endpoint order from the config Secret when the resulting set of +// addresses is unchanged. nodes.Items from r.List is not order-stable across +// calls, so without this guarantee the slice can permute on each reconcile, +// triggering a spurious config Secret rewrite which fires a watch event that +// re-enqueues the ProxyGroup, looping forever (issue #19700). +func TestFindStaticEndpointsStableOrder(t *testing.T) { + const ( + addrA = "10.0.0.1" + addrB = "10.0.0.2" + port = uint16(30001) + ) + + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pc"}, + Spec: tsapi.ProxyClassSpec{ + StaticEndpoints: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{{Port: port}}, + Selector: map[string]string{"foo/bar": "baz"}, + }, + }, + }, + } + + // Existing config Secret already pins the order [B, A]. The fake client + // lists nodes in name order ([node-a, node-b]) so without the stable-order + // guard findStaticEndpoints would return [A, B], differing from currAddrs + // and causing a spurious Secret rewrite. + currAddrs := []netip.AddrPort{ + netip.MustParseAddrPort(addrB + ":30001"), + netip.MustParseAddrPort(addrA + ":30001"), + } + cfg := ipn.ConfigVAlpha{StaticEndpoints: currAddrs} + cfgJSON, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + existingSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-0-config", Namespace: tsNamespace}, + Data: map[string][]byte{tsoperator.TailscaledConfigFileName(106): cfgJSON}, + } + + nodes := []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{Name: "node-a", Labels: map[string]string{"foo/bar": "baz"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeExternalIP, Address: addrA}, + }}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "node-b", Labels: map[string]string{"foo/bar": "baz"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeExternalIP, Address: addrB}, + }}, + }, + } + + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc, nodes[0], nodes[1], existingSecret). + Build() + + zl, _ := zap.NewDevelopment() + r := &ProxyGroupReconciler{Client: fc} + + got, err := r.findStaticEndpoints(t.Context(), existingSecret, pc, port, zl.Sugar()) + if err != nil { + t.Fatalf("findStaticEndpoints: %v", err) + } + if !slices.Equal(got, currAddrs) { + t.Errorf("findStaticEndpoints returned %v, want %v (order must match currAddrs to avoid reconcile churn)", got, currAddrs) + } + + // Repeat to confirm the result is stable across calls. + got2, err := r.findStaticEndpoints(t.Context(), existingSecret, pc, port, zl.Sugar()) + if err != nil { + t.Fatalf("findStaticEndpoints (2nd call): %v", err) + } + if !slices.Equal(got, got2) { + t.Errorf("findStaticEndpoints not stable across calls: first=%v second=%v", got, got2) + } +} + func TestProxyGroup(t *testing.T) { pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ @@ -1052,14 +1136,15 @@ func TestProxyGroupTypes(t *testing.T) { zl, _ := zap.NewDevelopment() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - tsProxyImage: testProxyImage, - Client: fc, - log: zl.Sugar(), - clients: tsclient.NewProvider(&fakeTSClient{}), - clock: tstest.NewClock(tstest.ClockOpts{}), - authKeyRateLimits: make(map[string]*rate.Limiter), - authKeyReissuing: make(map[string]bool), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zl.Sugar(), + clients: tsclient.NewProvider(&fakeTSClient{}), + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), + sharedACMEAccountKey: true, } t.Run("egress_type", func(t *testing.T) { @@ -1179,6 +1264,9 @@ func TestProxyGroupTypes(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-ingress", UID: "test-ingress-uid", + Annotations: map[string]string{ + AnnotationShareACMEAccount: "true", + }, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, @@ -1199,6 +1287,44 @@ func TestProxyGroupTypes(t *testing.T) { verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress) verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json") verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true") + verifyEnvVar(t, sts, "TS_ACME_ACCOUNT_SECRET_NAME", kubetypes.ACMEAccountsSecretName) + // pg.Spec.Tailnet is empty here so the default tailnet field is used. + verifyEnvVar(t, sts, "TS_ACME_ACCOUNT_FIELD", kubetypes.ACMEAccountDefaultKey+kubetypes.ACMEAccountKeySuffix) + // TS_DEBUG_ACME_FORCE_RENEWAL must NOT be set when the PG is + // opted in to the shared ACME account. + for _, e := range sts.Spec.Template.Spec.Containers[0].Env { + if e.Name == "TS_DEBUG_ACME_FORCE_RENEWAL" { + t.Errorf("TS_DEBUG_ACME_FORCE_RENEWAL must not be set on ingress ProxyGroup pods that share an ACME account") + } + } + + // Verify the shared ACME accounts Secret exists and has the + // deletion finalizer (see tailscale/tailscale#18251). + acmeSecret := &corev1.Secret{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: kubetypes.ACMEAccountsSecretName}, acmeSecret); err != nil { + t.Errorf("failed to get shared ACME accounts Secret: %v", err) + } + if !slices.Contains(acmeSecret.Finalizers, kubetypes.ACMEAccountsFinalizer) { + t.Errorf("shared ACME accounts Secret missing finalizer %q (got %v)", kubetypes.ACMEAccountsFinalizer, acmeSecret.Finalizers) + } + + // Verify the per-ProxyGroup Role grants access to the shared + // ACME accounts Secret (write replicas need it to read/write the + // per-tailnet account key). + role := &rbacv1.Role{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, role); err != nil { + t.Fatalf("failed to get ProxyGroup Role: %v", err) + } + var sawACMEAccess bool + for _, rule := range role.Rules { + if slices.Contains(rule.Verbs, "patch") && slices.Contains(rule.ResourceNames, kubetypes.ACMEAccountsSecretName) { + sawACMEAccess = true + break + } + } + if !sawACMEAccess { + t.Errorf("ProxyGroup Role does not grant patch access to %q", kubetypes.ACMEAccountsSecretName) + } // Verify ConfigMap volume mount cmName := fmt.Sprintf("%s-ingress-config", pg.Name) @@ -1228,6 +1354,60 @@ func TestProxyGroupTypes(t *testing.T) { } }) + t.Run("ingress_type_shared_acme_opt_out", func(t *testing.T) { + // The reconciler has sharedACMEAccountKey=true, so ingress PGs + // default to shared. Explicit tailscale.com/share-acme-account=false + // must opt this PG out: no shared-Secret env vars, no Role + // access to the shared Secret, and TS_DEBUG_ACME_FORCE_RENEWAL + // must still be set so ARI "replaces" doesn't silently fail. + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress-optout", + UID: "test-ingress-optout-uid", + Annotations: map[string]string{ + AnnotationShareACMEAccount: "false", + }, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + Replicas: new(int32(0)), + }, + } + if err := fc.Create(t.Context(), pg); err != nil { + t.Fatal(err) + } + expectReconciled(t, reconciler, "", pg.Name) + + sts := &appsv1.StatefulSet{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + for _, e := range sts.Spec.Template.Spec.Containers[0].Env { + switch e.Name { + case "TS_ACME_ACCOUNT_SECRET_NAME", "TS_ACME_ACCOUNT_FIELD": + t.Errorf("env %q unexpectedly present on opt-out PG", e.Name) + } + } + var sawForceRenewal bool + for _, e := range sts.Spec.Template.Spec.Containers[0].Env { + if e.Name == "TS_DEBUG_ACME_FORCE_RENEWAL" { + sawForceRenewal = true + } + } + if !sawForceRenewal { + t.Errorf("TS_DEBUG_ACME_FORCE_RENEWAL must be set on opt-out PG (avoids silent ARI \"replaces\" rejection)") + } + role := &rbacv1.Role{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, role); err != nil { + t.Fatalf("failed to get ProxyGroup Role: %v", err) + } + for _, rule := range role.Rules { + if slices.Contains(rule.ResourceNames, kubetypes.ACMEAccountsSecretName) { + t.Errorf("opt-out PG Role must not grant access to %q", kubetypes.ACMEAccountsSecretName) + } + } + }) + t.Run("kubernetes_api_server_type", func(t *testing.T) { pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -1247,7 +1427,7 @@ func TestProxyGroupTypes(t *testing.T) { } expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 1, 2, 1) + verifyProxyGroupCounts(t, reconciler, 2, 2, 1) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1952,10 +2132,11 @@ func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string) func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, proxyClass *tsapi.ProxyClass) { t.Helper() - role := pgRole(pg, tsNamespace) + shareACMEAccount := pg.Annotations[AnnotationShareACMEAccount] == "true" + role := pgRole(pg, tsNamespace, shareACMEAccount) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass) + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass, shareACMEAccount) if err != nil { t.Fatal(err) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 66c2e917f..f25f83185 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -68,6 +68,12 @@ const ( AnnotationProxyGroup = "tailscale.com/proxy-group" + // AnnotationShareACMEAccount opts a single ProxyGroup into ("true") + // or out of ("false") using the shared per-tailnet ACME account key. + // When absent, OPERATOR_SHARED_ACME_ACCOUNT_KEY on the operator is + // the default. See tailscale/tailscale#18251. + AnnotationShareACMEAccount = "tailscale.com/share-acme-account" + // Annotations settable by users on ingresses. AnnotationFunnel = "tailscale.com/funnel" AnnotationHTTPRedirect = "tailscale.com/http-redirect" @@ -783,7 +789,7 @@ func (r *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S // No need to error out if now or in future we end up in a // situation where app info cannot be determined for one of the // many proxy configurations that the operator can produce. - logger.Error("[unexpected] unable to determine proxy type") + logger.Error("unable to determine proxy type") } else { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_INTERNAL_APP", @@ -993,7 +999,7 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { if isMainContainer(&c) { if debug { ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, - // Serve tailscaled's debug metrics on on + // Serve tailscaled's debug metrics on // :9001/debug/metrics. If we didn't specify Pod IP // here, the proxy would, in some cases, also listen to its // Tailscale IP- we don't want folks to start relying on this @@ -1321,7 +1327,7 @@ func proxyCapVer(sec *corev1.Secret, podUID string, log *zap.SugaredLogger) tail } capVer, err := strconv.Atoi(string(sec.Data[kubetypes.KeyCapVer])) if err != nil { - log.Infof("[unexpected]: unexpected capability version in proxy's state Secret, expected an integer, got %q", string(sec.Data[kubetypes.KeyCapVer])) + log.Warnf("unexpected capability version in proxy's state Secret, expected an integer, got %q", string(sec.Data[kubetypes.KeyCapVer])) return tailcfg.CapabilityVersion(-1) } if !strings.EqualFold(podUID, string(sec.Data[kubetypes.KeyPodUID])) { diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index f55f582a6..c5ea7862d 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -41,10 +41,7 @@ func Test_statefulSetNameBase(t *testing.T) { if _, err := b.WriteString("a"); err != nil { t.Fatalf("error writing to string builder: %v", err) } - baseLength := b.Len() - if baseLength > 43 { - baseLength = 43 // currently 43 is the max base length - } + baseLength := min(b.Len(), 43) // currently 43 is the max base length wantsNameR := regexp.MustCompile(`^ts-a{` + fmt.Sprint(baseLength) + `}-$`) // to match a string like ts-aaaa- gotName := statefulSetNameBase(b.String()) if !wantsNameR.MatchString(gotName) { diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 29d1a1ebd..395e89eb9 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -99,7 +99,6 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque pgName := svc.Annotations[AnnotationProxyGroup] if pgName == "" { - logger.Infof("[unexpected] no ProxyGroup annotation, skipping Tailscale Service provisioning") return res, nil } @@ -674,7 +673,7 @@ func (r *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con case shouldBeAdvertised: replicaName, ok := strings.CutSuffix(secret.Name, "-config") if !ok { - logger.Infof("[unexpected] unable to determine replica name from config Secret name %q, unable to determine if backend routing has been configured", secret.Name) + logger.Warnf("unable to determine replica name from config Secret name %q, unable to determine if backend routing has been configured", secret.Name) return nil } ready, err := r.backendRoutesSetup(ctx, serviceName.String(), replicaName, cfg, logger) @@ -825,14 +824,40 @@ func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.S } svcList := &corev1.ServiceList{} if err := r.List(ctx, svcList); err != nil { - errs = append(errs, fmt.Errorf("[unexpected] error listing Services: %w", err)) + errs = append(errs, fmt.Errorf("error listing Services: %w", err)) return errors.Join(errs...) } svcName := nameForService(svc) for _, s := range svcList.Items { - if r.shouldExpose(&s) && nameForService(&s) == svcName && s.UID != svc.UID { - errs = append(errs, fmt.Errorf("found duplicate Service %q for hostname %q - multiple HA Services for the same hostname in the same cluster are not allowed", client.ObjectKeyFromObject(&s), svcName)) + if s.UID == svc.UID { + continue } + // Only check services managed by the ProxyGroup reconciler. Services + // exposed via the single-proxy path in svc.go have their own + // hostname tracking and live in a separate per-proxy tailnet + // namespace; flagging them as duplicates here breaks multi-tailnet + // setups where a single-proxy Service on the primary tailnet shares + // a hostname with a ProxyGroup ingress on a secondary tailnet. + if !r.isTailscaleService(&s) { + continue + } + if nameForService(&s) != svcName { + continue + } + // Two ProxyGroups joined to different tailnets each have their own + // DNS namespace, so a hostname collision across them is not a real + // conflict. Look up the other Service's ProxyGroup and skip the + // duplicate report when the tailnets differ; if the lookup fails + // fall through and flag the collision so a genuine duplicate isn't + // silently allowed. + otherPGName := s.Annotations[AnnotationProxyGroup] + if otherPGName != "" && otherPGName != pg.Name { + otherPG := &tsapi.ProxyGroup{} + if err := r.Get(ctx, client.ObjectKey{Name: otherPGName}, otherPG); err == nil && otherPG.Spec.Tailnet != pg.Spec.Tailnet { + continue + } + } + errs = append(errs, fmt.Errorf("found duplicate Service %q for hostname %q - multiple HA Services for the same hostname on the same tailnet are not allowed", client.ObjectKeyFromObject(&s), svcName)) } return errors.Join(errs...) } diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index 455d3363c..9d2296112 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -258,7 +258,7 @@ func TestValidateService(t *testing.T) { Status: metav1.ConditionFalse, Reason: reasonIngressSvcInvalid, LastTransitionTime: metav1.NewTime(cl.Now().Truncate(time.Second)), - Message: `found duplicate Service "ns-2/my-app2" for hostname "my-app" - multiple HA Services for the same hostname in the same cluster are not allowed`, + Message: `found duplicate Service "ns-2/my-app2" for hostname "my-app" - multiple HA Services for the same hostname on the same tailnet are not allowed`, }, }, }, @@ -270,6 +270,142 @@ func TestValidateService(t *testing.T) { expectEqual(t, lc, wantSvc) } +// Regression test for #20069. The pre-fix duplicate-hostname check scanned +// every Service with shouldExpose=true, which meant a Service exposed on +// one tailnet via the single-proxy path (svc.go) would block a ProxyGroup +// ingress Service for the same hostname on a different tailnet. The +// ProxyGroup path's validateService must skip Services that aren't +// themselves managed by a ProxyGroup. +func TestValidateService_SingleProxyServiceDoesNotCollideWithProxyGroup(t *testing.T) { + pgr, _, lc, _, _ := setupServiceTest(t) + // Service exposed via the single-proxy path: tailscale.com/expose=true + // and no tailscale.com/proxy-group annotation. Its hostname matches + // the ProxyGroup-managed Service below, but it lives in a different + // reconciler entirely and must not be flagged as a duplicate. + singleProxySvc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "single-proxy", + Namespace: "ns-1", + UID: types.UID("single-proxy-uid"), + Annotations: map[string]string{ + "tailscale.com/expose": "true", + "tailscale.com/hostname": "my-app", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: corev1.ServiceTypeClusterIP, + }, + } + // ProxyGroup-managed Service for the same hostname. + pgSvc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pg-svc", + Namespace: "ns-2", + UID: types.UID("pg-svc-uid"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/hostname": "my-app", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.5", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), + }, + } + + mustCreate(t, lc, singleProxySvc) + mustCreate(t, lc, pgSvc) + expectReconciled(t, pgr, pgSvc.Namespace, pgSvc.Name) + + got := &corev1.Service{} + if err := lc.Get(context.Background(), client.ObjectKeyFromObject(pgSvc), got); err != nil { + t.Fatalf("get Service: %v", err) + } + for _, c := range got.Status.Conditions { + if c.Type == string(tsapi.IngressSvcValid) && c.Status == metav1.ConditionFalse { + t.Fatalf("ProxyGroup Service flagged invalid by a single-proxy Service on a different tailnet: %s", c.Message) + } + } +} + +// Regression test for #20069. Two ProxyGroup-managed Services with the same +// hostname but joined to different tailnets each have their own DNS +// namespace and must not be flagged as duplicates. The Service being +// reconciled here is on the default tailnet (so it uses the configured +// fake tsclient); the conflicting Service already exists in-cluster on +// a different tailnet and must be skipped by the duplicate check. +func TestValidateService_DifferentTailnetDoesNotCollide(t *testing.T) { + pgr, _, lc, _, _ := setupServiceTest(t) + // Pre-create a ProxyGroup joined to a different tailnet. + secondaryPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secondary-pg", + Generation: 1, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + Tailnet: "secondary", + }, + } + if err := lc.Create(context.Background(), secondaryPG); err != nil { + t.Fatalf("create secondary ProxyGroup: %v", err) + } + // Pre-existing Service on the secondary tailnet with the conflicting hostname. + otherTailnetSvc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "other-tailnet-svc", + Namespace: "ns-2", + UID: types.UID("other-tailnet-svc-uid"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "secondary-pg", + "tailscale.com/hostname": "my-app", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.5", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), + }, + } + mustCreate(t, lc, otherTailnetSvc) + + // Service being reconciled: same hostname, default tailnet ProxyGroup. + primarySvc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "primary-svc", + Namespace: "ns-1", + UID: types.UID("primary-svc-uid"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/hostname": "my-app", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), + }, + } + mustCreate(t, lc, primarySvc) + expectReconciled(t, pgr, primarySvc.Namespace, primarySvc.Name) + + got := &corev1.Service{} + if err := lc.Get(context.Background(), client.ObjectKeyFromObject(primarySvc), got); err != nil { + t.Fatalf("get Service: %v", err) + } + for _, c := range got.Status.Conditions { + if c.Type == string(tsapi.IngressSvcValid) && c.Status == metav1.ConditionFalse { + t.Fatalf("ProxyGroup Service flagged invalid by a Service on a different tailnet with the same hostname: %s", c.Message) + } + } +} + func TestServicePGReconciler_MultiCluster(t *testing.T) { var ft *fakeTSClient for i := 0; i <= 10; i++ { diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index 6f12148c8..eb39d0029 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -468,29 +468,29 @@ func retrieveClusterDomain(namespace string, logger *zap.SugaredLogger) string { if err != nil { // Vast majority of clusters use the cluster.local domain, so it // is probably better to fall back to that than error out. - logger.Infof("[unexpected] error parsing /etc/resolv.conf to determine cluster domain, defaulting to 'cluster.local'.") + logger.Warn("error parsing /etc/resolv.conf to determine cluster domain, defaulting to 'cluster.local'.") return defaultClusterDomain } return clusterDomainFromResolverConf(conf, namespace, logger) } // clusterDomainFromResolverConf attempts to retrieve cluster domain from the provided resolver config. -// It expects the first three search domains in the resolver config to be be ['.svc., svc., , ...] +// It expects the first three search domains in the resolver config to be ['.svc., svc., , ...] // If the first three domains match the expected structure, it returns the third. // If the domains don't match the expected structure or an error is encountered, it defaults to 'cluster.local' domain. func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string, logger *zap.SugaredLogger) string { if len(conf.SearchDomains) < 3 { - logger.Infof("[unexpected] resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.") + logger.Warnf(" resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.", len(conf.SearchDomains)) return defaultClusterDomain } first := conf.SearchDomains[0] if !strings.HasPrefix(string(first), namespace+".svc") { - logger.Infof("[unexpected] first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.") + logger.Warnf("first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.") return defaultClusterDomain } second := conf.SearchDomains[1] if !strings.HasPrefix(string(second), "svc") { - logger.Infof("[unexpected] second search domain in resolver config is %s; expected 'svc.'.\nDefaulting cluster domain to 'cluster.local'.", second) + logger.Warnf("second search domain in resolver config is %s; expected 'svc.'.\nDefaulting cluster domain to 'cluster.local'.", second) return defaultClusterDomain } // Trim the trailing dot for backwards compatibility purposes as the @@ -499,7 +499,7 @@ func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string probablyClusterDomain := strings.TrimPrefix(second.WithoutTrailingDot(), "svc.") third := conf.SearchDomains[2] if !strings.EqualFold(third.WithoutTrailingDot(), probablyClusterDomain) { - logger.Infof("[unexpected] expected resolver config to contain serch domains .svc., svc., ; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third) + logger.Warnf("expected resolver config to contain serch domains .svc., svc., ; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third) return defaultClusterDomain } logger.Infof("Cluster domain %q extracted from resolver config", probablyClusterDomain) diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 074d92094..6c0d77a8c 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -984,6 +984,11 @@ func removeTargetPortsFromSvc(svc *corev1.Service) { svc.Spec.Ports = newPorts } +func removeClusterIPsFromSvc(svc *corev1.Service) { + svc.Spec.ClusterIP = "" + svc.Spec.ClusterIPs = nil +} + func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) { return func(secret *corev1.Secret) { t.Helper() diff --git a/cmd/k8s-operator/tsclient.go b/cmd/k8s-operator/tsclient.go index 702f4cc53..0670d2bcf 100644 --- a/cmd/k8s-operator/tsclient.go +++ b/cmd/k8s-operator/tsclient.go @@ -6,16 +6,11 @@ package main import ( - "context" "fmt" "net/url" "os" - "sync" - "time" "go.uber.org/zap" - "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" "tailscale.com/client/tailscale/v2" "tailscale.com/ipn" @@ -58,67 +53,18 @@ func newTSClient(logger *zap.SugaredLogger, clientID, clientIDPath, clientSecret } } else { // Use workload identity federation. - tokenSrc := &jwtTokenSource{ - logger: logger, - jwtPath: oidcJWTPath, - baseCfg: clientcredentials.Config{ - ClientID: clientID, - TokenURL: fmt.Sprintf("%s%s", baseURL, "/api/v2/oauth/token-exchange"), - }, - } - client.Auth = &tailscale.IdentityFederation{ ClientID: clientID, IDTokenFunc: func() (string, error) { - token, err := tokenSrc.Token() + token, err := os.ReadFile(oidcJWTPath) if err != nil { return "", err } - return token.AccessToken, nil + return string(token), nil }, } } return client, nil } - -// jwtTokenSource implements the [oauth2.TokenSource] interface, but with the -// ability to regenerate a fresh underlying token source each time a new value -// of the JWT parameter is needed due to expiration. -type jwtTokenSource struct { - logger *zap.SugaredLogger - jwtPath string // Path to the file containing an automatically refreshed JWT. - baseCfg clientcredentials.Config // Holds config that doesn't change for the lifetime of the process. - - mu sync.Mutex // Guards underlying. - underlying oauth2.TokenSource // The oauth2 client implementation. Does its own separate caching of the access token. -} - -func (s *jwtTokenSource) Token() (*oauth2.Token, error) { - s.mu.Lock() - defer s.mu.Unlock() - - if s.underlying != nil { - t, err := s.underlying.Token() - if err == nil && t != nil && t.Valid() { - return t, nil - } - } - - s.logger.Debugf("Refreshing JWT from %s", s.jwtPath) - tk, err := os.ReadFile(s.jwtPath) - if err != nil { - return nil, fmt.Errorf("error reading JWT from %q: %w", s.jwtPath, err) - } - - // Shallow copy of the base config. - credentials := s.baseCfg - credentials.EndpointParams = map[string][]string{ - "jwt": {string(tk)}, - } - - src := credentials.TokenSource(context.Background()) - s.underlying = oauth2.ReuseTokenSourceWithExpiry(nil, src, time.Minute) - return s.underlying.Token() -} diff --git a/cmd/k8s-operator/tsrecorder.go b/cmd/k8s-operator/tsrecorder.go index 881d82354..86669d212 100644 --- a/cmd/k8s-operator/tsrecorder.go +++ b/cmd/k8s-operator/tsrecorder.go @@ -14,9 +14,11 @@ import ( "strconv" "strings" "sync" + "time" "go.uber.org/zap" xslices "golang.org/x/exp/slices" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -57,14 +59,15 @@ var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount // Recorder CRs. type RecorderReconciler struct { client.Client - log *zap.SugaredLogger - recorder record.EventRecorder - clock tstime.Clock - clients ClientProvider - tsNamespace string - - mu sync.Mutex // protects following - recorders set.Slice[types.UID] // for recorders gauge + log *zap.SugaredLogger + recorder record.EventRecorder + clock tstime.Clock + clients ClientProvider + tsNamespace string + authKeyRateLimits map[string]*rate.Limiter // per-Recorder rate limiters for auth key re-issuance. + authKeyReissuing map[string]bool + mu sync.Mutex // protects following + recorders set.Slice[types.UID] // for recorders gauge } func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger { @@ -164,9 +167,23 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { logger := r.logger(tsr.Name) + var replicas int32 = 1 + if tsr.Spec.Replicas != nil { + replicas = *tsr.Spec.Replicas + } + r.mu.Lock() r.recorders.Add(tsr.UID) gaugeRecorderResources.Set(int64(r.recorders.Len())) + if _, ok := r.authKeyRateLimits[tsr.Name]; !ok { + r.authKeyRateLimits[tsr.Name] = rate.NewLimiter(rate.Every(30*time.Second), int(replicas)) + } + for replica := range replicas { + name := fmt.Sprintf("%s-%d", tsr.Name, replica) + if _, ok := r.authKeyReissuing[name]; !ok { + r.authKeyReissuing[name] = false + } + } r.mu.Unlock() if err := r.ensureAuthSecretsCreated(ctx, tsClient, tsr); err != nil { @@ -174,11 +191,6 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } // State Secrets are pre-created so we can use the Recorder CR as its owner ref. - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - for replica := range replicas { sec := tsrStateSecret(tsr, r.tsNamespace, replica) _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) { @@ -423,6 +435,10 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record r.mu.Lock() r.recorders.Remove(tsr.UID) gaugeRecorderResources.Set(int64(r.recorders.Len())) + delete(r.authKeyRateLimits, tsr.Name) + for replica := range replicas { + delete(r.authKeyReissuing, fmt.Sprintf("%s-%d", tsr.Name, replica)) + } r.mu.Unlock() return true, nil @@ -447,28 +463,122 @@ func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsCli Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), } - err := r.Get(ctx, key, &corev1.Secret{}) + existingSecret := &corev1.Secret{} + err := r.Get(ctx, key, existingSecret) switch { case err == nil: - logger.Debugf("auth Secret %q already exists", key.Name) + reissue, err := r.shouldReissueAuthKey(ctx, tsClient, tsr, replica, existingSecret) + if err != nil { + return fmt.Errorf("error checking auth key reissue for replica %d: %w", replica, err) + } + if !reissue { + logger.Debugf("auth Secret %q already exists, no reissue needed", key.Name) + continue + } + authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) + if err != nil { + return err + } + existingSecret.Data["authkey"] = []byte(authKey) + if err = r.Update(ctx, existingSecret); err != nil { + return err + } continue - case !apierrors.IsNotFound(err): + case apierrors.IsNotFound(err): + authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) + if err != nil { + return err + } + if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil { + return err + } + default: return fmt.Errorf("failed to get Secret %q: %w", key.Name, err) } - - authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) - if err != nil { - return err - } - - if err = r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil { - return err - } } return nil } +// shouldReissueAuthKey returns true if the proxy needs a new auth key. It +// tracks in-flight reissues via authKeyReissuing to avoid duplicate API calls +// across reconciles. +func (r *RecorderReconciler) shouldReissueAuthKey(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder, replica int32, authSecret *corev1.Secret) (shouldReissue bool, err error) { + stateSecret, err := r.getStateSecret(ctx, tsr.Name, replica) + if err != nil || stateSecret == nil { + return false, err + } + + stateSecretName := fmt.Sprintf("%s-%d", tsr.Name, replica) + + r.mu.Lock() + reissuing := r.authKeyReissuing[stateSecretName] + r.mu.Unlock() + + if reissuing { + _, requestStillPresent := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !requestStillPresent { + r.mu.Lock() + r.authKeyReissuing[stateSecretName] = false + r.mu.Unlock() + r.log.Debugf("auth key reissue completed for %q", stateSecretName) + return false, nil + } + r.log.Debugf("auth key already in process of re-issuance for %q, waiting", stateSecretName) + return false, nil + } + + defer func() { + r.mu.Lock() + r.authKeyReissuing[stateSecretName] = shouldReissue + r.mu.Unlock() + }() + + brokenAuthkey, ok := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !ok { + return false, nil + } + + cfgAuthKey := string(authSecret.Data["authkey"]) + empty := cfgAuthKey == "" + broken := cfgAuthKey == string(brokenAuthkey) + + if !empty && !broken { + return false, nil + } + + lim := r.authKeyRateLimits[tsr.Name] + if !lim.Allow() { + r.log.Debugf("auth key re-issuance rate limit exceeded, limit: %.2f, burst: %d, tokens: %.2f", + lim.Limit(), lim.Burst(), lim.Tokens()) + return false, fmt.Errorf("auth key re-issuance rate limit exceeded for Recorder %q, will retry with backoff", tsr.Name) + } + + r.log.Infof("Recorder replica %s failing to auth; attempting cleanup and new key", stateSecretName) + if tsID := stateSecret.Data[kubetypes.KeyDeviceID]; len(tsID) > 0 { + id := tailcfg.StableNodeID(tsID) + if err := r.ensureDeviceDeleted(ctx, tsClient, id, r.log); err != nil { + return false, err + } + } + + return true, nil +} + +func (r *RecorderReconciler) ensureDeviceDeleted(ctx context.Context, tsClient tsclient.Client, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error { + logger.Debugf("deleting device %s from control", string(id)) + err := tsClient.Devices().Delete(ctx, string(id)) + switch { + case tailscale.IsNotFound(err): + logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id)) + case err != nil: + return fmt.Errorf("error deleting device: %w", err) + default: + logger.Debugf("device %s deleted from control", string(id)) + } + return nil +} + func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) error { if !tsr.Spec.EnableUI && tsr.Spec.Storage.S3 == nil { return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible") diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go index 6bd47e07b..8f189728c 100644 --- a/cmd/k8s-operator/tsrecorder_test.go +++ b/cmd/k8s-operator/tsrecorder_test.go @@ -14,6 +14,7 @@ import ( "github.com/google/go-cmp/cmp" "go.uber.org/zap" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -55,12 +56,14 @@ func TestRecorder(t *testing.T) { fr := record.NewFakeRecorder(2) cl := tstest.NewClock(tstest.ClockOpts{}) reconciler := &RecorderReconciler{ - tsNamespace: tsNamespace, - Client: fc, - clients: tsclient.NewProvider(tsClient), - recorder: fr, - log: zl.Sugar(), - clock: cl, + tsNamespace: tsNamespace, + Client: fc, + clients: tsclient.NewProvider(tsClient), + recorder: fr, + log: zl.Sugar(), + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) { diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go index 38a86a5e0..8d9b9faf0 100644 --- a/cmd/k8s-proxy/k8s-proxy.go +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -31,6 +31,7 @@ import ( "k8s.io/utils/strings/slices" "tailscale.com/client/local" "tailscale.com/cmd/k8s-proxy/internal/config" + "tailscale.com/health" "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/store" @@ -41,6 +42,7 @@ import ( "tailscale.com/kube/certs" healthz "tailscale.com/kube/health" "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" klc "tailscale.com/kube/localclient" "tailscale.com/kube/metrics" @@ -171,10 +173,31 @@ func run(logger *zap.SugaredLogger) error { // If Pod UID unset, assume we're running outside of a cluster/not managed // by the operator, so no need to set additional state keys. + var kc kubeclient.Client + var stateSecretName string if podUID != "" { if err := state.SetInitialKeys(st, podUID); err != nil { return fmt.Errorf("error setting initial state: %w", err) } + + if cfg.Parsed.State != nil { + if name, ok := strings.CutPrefix(*cfg.Parsed.State, "kube:"); ok { + stateSecretName = name + + kc, err = kubeclient.New(k8sProxyFieldManager) + if err != nil { + return err + } + + var configAuthKey string + if cfg.Parsed.AuthKey != nil { + configAuthKey = *cfg.Parsed.AuthKey + } + if err := resetState(ctx, kc, stateSecretName, podUID, configAuthKey); err != nil { + return fmt.Errorf("error resetting state: %w", err) + } + } + } } var authKey string @@ -197,23 +220,69 @@ func run(logger *zap.SugaredLogger) error { ts.Hostname = *cfg.Parsed.Hostname } - // Make sure we crash loop if Up doesn't complete in reasonable time. - upCtx, upCancel := context.WithTimeout(ctx, time.Minute) - defer upCancel() - if _, err := ts.Up(upCtx); err != nil { - return fmt.Errorf("error starting tailscale server: %w", err) - } - defer ts.Close() lc, err := ts.LocalClient() if err != nil { return fmt.Errorf("error getting local client: %w", err) } - // Setup for updating state keys. + // Make sure we crash loop if Up doesn't complete in reasonable time. + upCtx, upCancel := context.WithTimeout(ctx, 30*time.Second) + defer upCancel() + + // ts.Up() deliberately ignores NeedsLogin because it fires transiently + // during normal auth-key login. We can watch for the login-state health + // warning here though, which only fires on terminal auth failure, and + // cancel early. + go func() { + w, err := lc.WatchIPNBus(upCtx, ipn.NotifyInitialHealthState) + if err != nil { + return + } + defer w.Close() + for { + n, err := w.Next() + if err != nil { + logger.Debugf("failed to process message from ipn bus: %s", err.Error()) + return + } + if n.Health != nil { + if _, ok := n.Health.Warnings[health.LoginStateWarnable.Code]; ok { + upCancel() + return + } + } + } + }() + + if _, err := ts.Up(upCtx); err != nil { + if kc != nil && stateSecretName != "" { + return handleAuthKeyReissue(ctx, lc, kc, stateSecretName, authKey, cfgChan, logger) + } + return err + } + + defer ts.Close() + + reissueCh := make(chan struct{}, 1) if podUID != "" { group.Go(func() error { return state.KeepKeysUpdated(ctx, st, klc.New(lc)) }) + + if kc != nil && stateSecretName != "" { + needsReissue, err := checkInitialAuthState(ctx, lc) + if err != nil { + return fmt.Errorf("error checking initial auth state: %w", err) + } + if needsReissue { + logger.Info("Auth key missing or invalid after startup, requesting new key from operator") + return handleAuthKeyReissue(ctx, lc, kc, stateSecretName, authKey, cfgChan, logger) + } + + group.Go(func() error { + return monitorAuthHealth(ctx, lc, reissueCh, logger) + }) + } } if cfg.Parsed.HealthCheckEnabled.EqualBool(true) || cfg.Parsed.MetricsEnabled.EqualBool(true) { @@ -232,8 +301,15 @@ func run(logger *zap.SugaredLogger) error { } if cfg.Parsed.HealthCheckEnabled.EqualBool(true) { - ipV4, _ := ts.TailscaleIPs() - hz := healthz.RegisterHealthHandlers(mux, ipV4.String(), logger.Infof) + ipV4, ipV6 := ts.TailscaleIPs() + var v4, v6 string + if ipV4.IsValid() { + v4 = ipV4.String() + } + if ipV6.IsValid() { + v6 = ipV6.String() + } + hz := healthz.RegisterHealthHandlers(mux, v4, v6, logger.Infof) group.Go(func() error { err := hz.MonitorHealth(ctx, lc) if err == nil || errors.Is(err, context.Canceled) { @@ -362,6 +438,8 @@ func run(logger *zap.SugaredLogger) error { } cfgLogger.Infof("Config reloaded") + case <-reissueCh: + return handleAuthKeyReissue(ctx, lc, kc, stateSecretName, authKey, cfgChan, logger) } } } diff --git a/cmd/k8s-proxy/kube.go b/cmd/k8s-proxy/kube.go new file mode 100644 index 000000000..1d9348f1a --- /dev/null +++ b/cmd/k8s-proxy/kube.go @@ -0,0 +1,161 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "go.uber.org/zap" + "tailscale.com/client/local" + "tailscale.com/health" + "tailscale.com/ipn" + "tailscale.com/kube/authkey" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubeapi" + "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" +) + +const k8sProxyFieldManager = "tailscale-k8s-proxy" + +// resetState clears k8s-proxy state from previous runs and sets +// initial values. This ensures the operator doesn't use stale state when a Pod +// is first recreated. +// +// It also clears the reissue_authkey marker if the operator has actioned it +// (i.e., the config now has a different auth key than what was marked for +// reissue). +func resetState(ctx context.Context, kc kubeclient.Client, stateSecretName string, podUID string, configAuthKey string) error { + existingSecret, err := kc.GetSecret(ctx, stateSecretName) + switch { + case kubeclient.IsNotFoundErr(err): + return nil + case err != nil: + return fmt.Errorf("failed to read state Secret %q to reset state: %w", stateSecretName, err) + } + + s := &kubeapi.Secret{ + Data: map[string][]byte{ + kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion), + }, + } + if podUID != "" { + s.Data[kubetypes.KeyPodUID] = []byte(podUID) + } + + // Only clear reissue_authkey if the operator has actioned it. + brokenAuthkey, ok := existingSecret.Data[kubetypes.KeyReissueAuthkey] + if ok && configAuthKey != "" && string(brokenAuthkey) != configAuthKey { + s.Data[kubetypes.KeyReissueAuthkey] = nil + } + + return kc.StrategicMergePatchSecret(ctx, stateSecretName, s, k8sProxyFieldManager) +} + +// needsAuthKeyReissue reports whether the given backend state and health +// warnings indicate a terminal auth failure requiring a new key from the +// operator. +func needsAuthKeyReissue(backendState string, healthWarnings []string) bool { + if backendState == ipn.NeedsLogin.String() { + return true + } + loginWarnableCode := string(health.LoginStateWarnable.Code) + for _, h := range healthWarnings { + if strings.Contains(h, loginWarnableCode) { + return true + } + } + return false +} + +// checkInitialAuthState checks if the tsnet server is in an auth failure state +// immediately after coming up. Returns true if auth key reissue is needed. +func checkInitialAuthState(ctx context.Context, lc *local.Client) (bool, error) { + status, err := lc.Status(ctx) + if err != nil { + return false, fmt.Errorf("error getting status: %w", err) + } + return needsAuthKeyReissue(status.BackendState, status.Health), nil +} + +// monitorAuthHealth watches the IPN bus for auth failures and triggers reissue +// when needed. Runs until context is cancelled or auth failure is detected. +func monitorAuthHealth(ctx context.Context, lc *local.Client, reissueCh chan<- struct{}, logger *zap.SugaredLogger) error { + w, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialHealthState) + if err != nil { + return fmt.Errorf("failed to watch IPN bus for auth health: %w", err) + } + defer w.Close() + + for { + if ctx.Err() != nil { + return ctx.Err() + } + n, err := w.Next() + if err != nil { + return err + } + if n.Health != nil { + if _, ok := n.Health.Warnings[health.LoginStateWarnable.Code]; ok { + logger.Info("Auth key failed to authenticate (may be expired or single-use), requesting new key from operator") + select { + case reissueCh <- struct{}{}: + case <-ctx.Done(): + } + return nil + } + } + } +} + +// handleAuthKeyReissue orchestrates the auth key reissue flow: +// 1. Disconnect from control +// 2. Set reissue marker in state Secret +// 3. Wait for operator to provide new key +// 4. Exit cleanly (Kubernetes will restart the pod with the new key) +func handleAuthKeyReissue(ctx context.Context, lc *local.Client, kc kubeclient.Client, stateSecretName string, currentAuthKey string, cfgChan <-chan *conf.Config, logger *zap.SugaredLogger) error { + if err := lc.DisconnectControl(ctx); err != nil { + return fmt.Errorf("error disconnecting from control: %w", err) + } + if err := authkey.SetReissueAuthKey(ctx, kc, stateSecretName, currentAuthKey, k8sProxyFieldManager); err != nil { + return fmt.Errorf("failed to set reissue_authkey in Kubernetes Secret: %w", err) + } + + var mu sync.Mutex + var latestAuthKey string + notify := make(chan struct{}, 1) + + // we use this go func to abstract away conf.Config from the shared function + go func() { + for cfg := range cfgChan { + if cfg.Parsed.AuthKey != nil { + mu.Lock() + latestAuthKey = *cfg.Parsed.AuthKey + mu.Unlock() + select { + case notify <- struct{}{}: + default: + } + } + } + }() + + getAuthKey := func() string { + mu.Lock() + defer mu.Unlock() + return latestAuthKey + } + clearFn := func(ctx context.Context) error { + return authkey.ClearReissueAuthKey(ctx, kc, stateSecretName, k8sProxyFieldManager) + } + + return authkey.WaitForAuthKeyReissue(ctx, currentAuthKey, 10*time.Minute, getAuthKey, clearFn, notify) +} diff --git a/cmd/k8s-proxy/kube_test.go b/cmd/k8s-proxy/kube_test.go new file mode 100644 index 000000000..c7e0f33d0 --- /dev/null +++ b/cmd/k8s-proxy/kube_test.go @@ -0,0 +1,141 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/health" + "tailscale.com/kube/kubeapi" + "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" +) + +func TestResetState(t *testing.T) { + tests := []struct { + name string + existingData map[string][]byte + podUID string + configAuthKey string + wantPatched map[string][]byte + }{ + { + name: "sets_capver_and_pod_uid", + existingData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("device-123"), + kubetypes.KeyDeviceFQDN: []byte("node.tailnet"), + kubetypes.KeyDeviceIPs: []byte(`["100.64.0.1"]`), + }, + podUID: "pod-123", + configAuthKey: "new-key", + wantPatched: map[string][]byte{ + kubetypes.KeyPodUID: []byte("pod-123"), + }, + }, + { + name: "clears_reissue_marker_when_actioned", + existingData: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-key"), + }, + podUID: "pod-123", + configAuthKey: "new-key", + wantPatched: map[string][]byte{ + kubetypes.KeyPodUID: []byte("pod-123"), + kubetypes.KeyReissueAuthkey: nil, + }, + }, + { + name: "keeps_reissue_marker_when_not_actioned", + existingData: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-key"), + }, + podUID: "pod-123", + configAuthKey: "old-key", + wantPatched: map[string][]byte{ + kubetypes.KeyPodUID: []byte("pod-123"), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.wantPatched[kubetypes.KeyCapVer] = fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) + + var patched map[string][]byte + kc := &kubeclient.FakeClient{ + GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { + return &kubeapi.Secret{Data: tt.existingData}, nil + }, + StrategicMergePatchSecretImpl: func(ctx context.Context, name string, s *kubeapi.Secret, fm string) error { + patched = s.Data + return nil + }, + } + + err := resetState(context.Background(), kc, "test-secret", tt.podUID, tt.configAuthKey) + if err != nil { + t.Fatalf("resetState() error = %v", err) + } + + if diff := cmp.Diff(tt.wantPatched, patched); diff != "" { + t.Errorf("resetState() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestNeedsAuthKeyReissue(t *testing.T) { + loginWarnableCode := string(health.LoginStateWarnable.Code) + + tests := []struct { + name string + backendState string + health []string + want bool + }{ + { + name: "running_healthy", + backendState: "Running", + want: false, + }, + { + name: "needs_login", + backendState: "NeedsLogin", + want: true, + }, + { + name: "running_with_login_warning", + backendState: "Running", + health: []string{"warning: " + loginWarnableCode + ": you are logged out"}, + want: true, + }, + { + name: "running_with_unrelated_warning", + backendState: "Running", + health: []string{"dns-not-working"}, + want: false, + }, + { + name: "running_no_warnings", + backendState: "Running", + health: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := needsAuthKeyReissue(tt.backendState, tt.health) + if got != tt.want { + t.Errorf("needsAuthKeyReissue() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/nardump/nardump.go b/cmd/nardump/nardump.go index c8db24cb6..38a2a6731 100644 --- a/cmd/nardump/nardump.go +++ b/cmd/nardump/nardump.go @@ -9,22 +9,13 @@ // git-pull-oss.sh having Nix available. package main -// For the format, see: -// See https://gist.github.com/jbeda/5c79d2b1434f0018d693 - import ( - "bufio" - "crypto/sha256" - "encoding/base64" - "encoding/binary" "flag" "fmt" - "io" - "io/fs" "log" "os" - "path" - "sort" + + "tailscale.com/cmd/nardump/nardump" ) var sri = flag.Bool("sri", false, "print SRI") @@ -34,167 +25,16 @@ func main() { if flag.NArg() != 1 { log.Fatal("usage: nardump ") } - arg := flag.Arg(0) - if err := os.Chdir(arg); err != nil { - log.Fatal(err) - } + fsys := os.DirFS(flag.Arg(0)) if *sri { - hash := sha256.New() - if err := writeNAR(hash, os.DirFS(".")); err != nil { + s, err := nardump.SRI(fsys) + if err != nil { log.Fatal(err) } - fmt.Printf("sha256-%s\n", base64.StdEncoding.EncodeToString(hash.Sum(nil))) + fmt.Println(s) return } - bw := bufio.NewWriter(os.Stdout) - if err := writeNAR(bw, os.DirFS(".")); err != nil { + if err := nardump.WriteNAR(os.Stdout, fsys); err != nil { log.Fatal(err) } - bw.Flush() -} - -// writeNARError is a sentinel panic type that's recovered by writeNAR -// and converted into the wrapped error. -type writeNARError struct{ err error } - -// narWriter writes NAR files. -type narWriter struct { - w io.Writer - fs fs.FS -} - -// writeNAR writes a NAR file to w from the root of fs. -func writeNAR(w io.Writer, fs fs.FS) (err error) { - defer func() { - if e := recover(); e != nil { - if we, ok := e.(writeNARError); ok { - err = we.err - return - } - panic(e) - } - }() - nw := &narWriter{w: w, fs: fs} - nw.str("nix-archive-1") - return nw.writeDir(".") -} - -func (nw *narWriter) writeDir(dirPath string) error { - ents, err := fs.ReadDir(nw.fs, dirPath) - if err != nil { - return err - } - sort.Slice(ents, func(i, j int) bool { - return ents[i].Name() < ents[j].Name() - }) - nw.str("(") - nw.str("type") - nw.str("directory") - for _, ent := range ents { - nw.str("entry") - nw.str("(") - nw.str("name") - nw.str(ent.Name()) - nw.str("node") - mode := ent.Type() - sub := path.Join(dirPath, ent.Name()) - var err error - switch { - case mode.IsDir(): - err = nw.writeDir(sub) - case mode.IsRegular(): - err = nw.writeRegular(sub) - case mode&os.ModeSymlink != 0: - err = nw.writeSymlink(sub) - default: - return fmt.Errorf("unsupported file type %v at %q", sub, mode) - } - if err != nil { - return err - } - nw.str(")") - } - nw.str(")") - return nil -} - -func (nw *narWriter) writeRegular(path string) error { - nw.str("(") - nw.str("type") - nw.str("regular") - fi, err := fs.Stat(nw.fs, path) - if err != nil { - return err - } - if fi.Mode()&0111 != 0 { - nw.str("executable") - nw.str("") - } - contents, err := fs.ReadFile(nw.fs, path) - if err != nil { - return err - } - nw.str("contents") - if err := writeBytes(nw.w, contents); err != nil { - return err - } - nw.str(")") - return nil -} - -func (nw *narWriter) writeSymlink(path string) error { - nw.str("(") - nw.str("type") - nw.str("symlink") - nw.str("target") - // broken symlinks are valid in a nar - // given we do os.chdir(dir) and os.dirfs(".") above - // readlink now resolves relative links even if they are broken - link, err := os.Readlink(path) - if err != nil { - return err - } - nw.str(link) - nw.str(")") - return nil -} - -func (nw *narWriter) str(s string) { - if err := writeString(nw.w, s); err != nil { - panic(writeNARError{err}) - } -} - -func writeString(w io.Writer, s string) error { - var buf [8]byte - binary.LittleEndian.PutUint64(buf[:], uint64(len(s))) - if _, err := w.Write(buf[:]); err != nil { - return err - } - if _, err := io.WriteString(w, s); err != nil { - return err - } - return writePad(w, len(s)) -} - -func writeBytes(w io.Writer, b []byte) error { - var buf [8]byte - binary.LittleEndian.PutUint64(buf[:], uint64(len(b))) - if _, err := w.Write(buf[:]); err != nil { - return err - } - if _, err := w.Write(b); err != nil { - return err - } - return writePad(w, len(b)) -} - -func writePad(w io.Writer, n int) error { - pad := n % 8 - if pad == 0 { - return nil - } - var zeroes [8]byte - _, err := w.Write(zeroes[:8-pad]) - return err } diff --git a/cmd/nardump/nardump/nardump.go b/cmd/nardump/nardump/nardump.go new file mode 100644 index 000000000..ab9ff1f3c --- /dev/null +++ b/cmd/nardump/nardump/nardump.go @@ -0,0 +1,193 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package nardump writes a NAR (Nix Archive) representation of an +// fs.FS to an io.Writer, or summarizes it as a Subresource Integrity +// hash, as used by Nix flake.nix vendor and toolchain hashes. +// +// For the format, see: +// https://gist.github.com/jbeda/5c79d2b1434f0018d693 +package nardump + +import ( + "bufio" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "io/fs" + "path" + "sort" +) + +// WriteNAR writes a NAR-encoded representation of fsys, rooted at +// the FS root, to w. +// +// The encoder issues many small writes; if w is not already a +// *bufio.Writer, WriteNAR wraps it in one and flushes on return so +// the caller doesn't have to. +// +// fsys must implement fs.ReadLinkFS to encode any symlinks it +// contains; os.DirFS satisfies this on Go 1.25+. +func WriteNAR(w io.Writer, fsys fs.FS) (err error) { + defer func() { + if e := recover(); e != nil { + if we, ok := e.(writeNARError); ok { + err = we.err + return + } + panic(e) + } + }() + bw, ok := w.(*bufio.Writer) + if !ok { + bw = bufio.NewWriter(w) + defer func() { + if flushErr := bw.Flush(); err == nil { + err = flushErr + } + }() + } + nw := &narWriter{w: bw, fs: fsys} + nw.str("nix-archive-1") + return nw.writeDir(".") +} + +// SRI returns the Subresource Integrity hash of the NAR encoding of +// fsys, in the form "sha256-". This is the format Nix +// expects for vendorHash and similar fields. +func SRI(fsys fs.FS) (string, error) { + h := sha256.New() + if err := WriteNAR(h, fsys); err != nil { + return "", err + } + return "sha256-" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil +} + +// writeNARError is a sentinel panic type that's recovered by +// WriteNAR and converted into the wrapped error. +type writeNARError struct{ err error } + +// narWriter writes NAR files. +type narWriter struct { + w io.Writer + fs fs.FS +} + +func (nw *narWriter) writeDir(dirPath string) error { + ents, err := fs.ReadDir(nw.fs, dirPath) + if err != nil { + return err + } + sort.Slice(ents, func(i, j int) bool { + return ents[i].Name() < ents[j].Name() + }) + nw.str("(") + nw.str("type") + nw.str("directory") + for _, ent := range ents { + nw.str("entry") + nw.str("(") + nw.str("name") + nw.str(ent.Name()) + nw.str("node") + mode := ent.Type() + sub := path.Join(dirPath, ent.Name()) + var err error + switch { + case mode.IsDir(): + err = nw.writeDir(sub) + case mode.IsRegular(): + err = nw.writeRegular(sub) + case mode&fs.ModeSymlink != 0: + err = nw.writeSymlink(sub) + default: + return fmt.Errorf("unsupported file type %v at %q", sub, mode) + } + if err != nil { + return err + } + nw.str(")") + } + nw.str(")") + return nil +} + +func (nw *narWriter) writeRegular(p string) error { + nw.str("(") + nw.str("type") + nw.str("regular") + fi, err := fs.Stat(nw.fs, p) + if err != nil { + return err + } + if fi.Mode()&0111 != 0 { + nw.str("executable") + nw.str("") + } + contents, err := fs.ReadFile(nw.fs, p) + if err != nil { + return err + } + nw.str("contents") + if err := writeBytes(nw.w, contents); err != nil { + return err + } + nw.str(")") + return nil +} + +func (nw *narWriter) writeSymlink(p string) error { + nw.str("(") + nw.str("type") + nw.str("symlink") + nw.str("target") + link, err := fs.ReadLink(nw.fs, p) + if err != nil { + return err + } + nw.str(link) + nw.str(")") + return nil +} + +func (nw *narWriter) str(s string) { + if err := writeString(nw.w, s); err != nil { + panic(writeNARError{err}) + } +} + +func writeString(w io.Writer, s string) error { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], uint64(len(s))) + if _, err := w.Write(buf[:]); err != nil { + return err + } + if _, err := io.WriteString(w, s); err != nil { + return err + } + return writePad(w, len(s)) +} + +func writeBytes(w io.Writer, b []byte) error { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], uint64(len(b))) + if _, err := w.Write(buf[:]); err != nil { + return err + } + if _, err := w.Write(b); err != nil { + return err + } + return writePad(w, len(b)) +} + +func writePad(w io.Writer, n int) error { + pad := n % 8 + if pad == 0 { + return nil + } + var zeroes [8]byte + _, err := w.Write(zeroes[:8-pad]) + return err +} diff --git a/cmd/nardump/nardump/nardump_test.go b/cmd/nardump/nardump/nardump_test.go new file mode 100644 index 000000000..16b690ee2 --- /dev/null +++ b/cmd/nardump/nardump/nardump_test.go @@ -0,0 +1,55 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package nardump + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" +) + +// setupTmpdir sets up a known golden layout, covering all allowed file/folder types in a nar. +func setupTmpdir(t *testing.T) string { + t.Helper() + tmpdir := t.TempDir() + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(os.MkdirAll(filepath.Join(tmpdir, "sub/dir"), 0755)) + must(os.Symlink("brokenfile", filepath.Join(tmpdir, "brokenlink"))) + must(os.Symlink("sub/dir", filepath.Join(tmpdir, "dirl"))) + must(os.Symlink("/abs/nonexistentdir", filepath.Join(tmpdir, "dirb"))) + f, err := os.Create(filepath.Join(tmpdir, "sub/dir/file1")) + must(err) + f.Close() + f, err = os.Create(filepath.Join(tmpdir, "file2m")) + must(err) + must(f.Truncate(2 * 1024 * 1024)) + f.Close() + must(os.Symlink("../file2m", filepath.Join(tmpdir, "sub/goodlink"))) + return tmpdir +} + +func TestWriteNAR(t *testing.T) { + if runtime.GOOS == "windows" { + // Skip test on Windows as the Nix package manager is not supported on this platform + t.Skip("nix package manager is not available on Windows") + } + dir := setupTmpdir(t) + // obtained via `nix-store --dump /tmp/... | sha256sum` of the above test dir + const expected = "727613a36f41030e93a4abf2649c3ec64a2757ccff364e3f6f7d544eb976e442" + h := sha256.New() + if err := WriteNAR(h, os.DirFS(dir)); err != nil { + t.Fatal(err) + } + if got := fmt.Sprintf("%x", h.Sum(nil)); got != expected { + t.Fatalf("sha256sum of nar: got %s, want %s", got, expected) + } +} diff --git a/cmd/nardump/nardump_test.go b/cmd/nardump/nardump_test.go deleted file mode 100644 index c1ca825e1..000000000 --- a/cmd/nardump/nardump_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -package main - -import ( - "crypto/sha256" - "fmt" - "os" - "runtime" - "testing" -) - -// setupTmpdir sets up a known golden layout, covering all allowed file/folder types in a nar -func setupTmpdir(t *testing.T) string { - tmpdir := t.TempDir() - pwd, _ := os.Getwd() - os.Chdir(tmpdir) - defer os.Chdir(pwd) - os.MkdirAll("sub/dir", 0755) - os.Symlink("brokenfile", "brokenlink") - os.Symlink("sub/dir", "dirl") - os.Symlink("/abs/nonexistentdir", "dirb") - os.Create("sub/dir/file1") - f, _ := os.Create("file2m") - _ = f.Truncate(2 * 1024 * 1024) - f.Close() - os.Symlink("../file2m", "sub/goodlink") - return tmpdir -} - -func TestWriteNar(t *testing.T) { - if runtime.GOOS == "windows" { - // Skip test on Windows as the Nix package manager is not supported on this platform - t.Skip("nix package manager is not available on Windows") - } - dir := setupTmpdir(t) - t.Run("nar", func(t *testing.T) { - // obtained via `nix-store --dump /tmp/... | sha256sum` of the above test dir - expected := "727613a36f41030e93a4abf2649c3ec64a2757ccff364e3f6f7d544eb976e442" - h := sha256.New() - os.Chdir(dir) - err := writeNAR(h, os.DirFS(".")) - if err != nil { - t.Fatal(err) - } - hash := fmt.Sprintf("%x", h.Sum(nil)) - if expected != hash { - t.Fatal("sha256sum of nar not matched", hash, expected) - } - }) -} diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index 877f16cc0..3ee09317e 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -20,6 +20,7 @@ import ( "net/netip" "os" "path/filepath" + "slices" "strings" "time" @@ -537,12 +538,7 @@ func (c *connector) ignoreDestination(dstAddrs []netip.Addr) bool { if c.ignoreDsts == nil { return false } - for _, a := range dstAddrs { - if c.ignoreDsts.Contains(a) { - return true - } - } - return false + return slices.ContainsFunc(dstAddrs, c.ignoreDsts.Contains) } func proxyTCPConn(c net.Conn, dest string, ctor *connector) { diff --git a/cmd/netlogfmt/main.go b/cmd/netlogfmt/main.go index 212b36fb6..af7baae46 100644 --- a/cmd/netlogfmt/main.go +++ b/cmd/netlogfmt/main.go @@ -45,6 +45,7 @@ import ( jsonv2 "github.com/go-json-experiment/json" "github.com/go-json-experiment/json/jsontext" "tailscale.com/tailcfg" + "tailscale.com/tstime" "tailscale.com/types/bools" "tailscale.com/types/logid" "tailscale.com/types/netlogtype" @@ -294,7 +295,7 @@ func printMessage(msg message) { fmt.Printf("NodeID: %s\n", msg.NodeID) } formatTime := func(t time.Time) string { - return t.In(time.Local).Format("2006-01-02 15:04:05.000") + return t.Local().Format(tstime.DateSpTimeMilliZ) } switch { case !msg.Logged.IsZero(): diff --git a/cmd/pgproxy/pgproxy.go b/cmd/pgproxy/pgproxy.go index ded6fa695..a138eacdc 100644 --- a/cmd/pgproxy/pgproxy.go +++ b/cmd/pgproxy/pgproxy.go @@ -291,7 +291,7 @@ func (p *proxy) serve(sessionID int64, c net.Conn) error { Certificates: p.downstreamCert, MinVersion: tls.VersionTLS12, }) - if err = uptc.HandshakeContext(ctx); err != nil { + if err = s.HandshakeContext(ctx); err != nil { p.errors.Add("client-tls", 1) return fmt.Errorf("client TLS handshake: %v", err) } diff --git a/cmd/sniproxy/sniproxy.go b/cmd/sniproxy/sniproxy.go index bd95cc113..f7ebc6aba 100644 --- a/cmd/sniproxy/sniproxy.go +++ b/cmd/sniproxy/sniproxy.go @@ -138,9 +138,9 @@ func run(ctx context.Context, ts *tsnet.Server, wgPort int, hostname string, pro } // Finally, start mainloop to configure app connector based on information - // in the netmap. - // We set the NotifyInitialNetMap flag so we will always get woken with the - // current netmap, before only being woken on changes. + // in the self node's CapMap. We set NotifyInitialNetMap so the first + // Notify carries the current self node (now via Notify.SelfChange); + // subsequent self changes wake us up too. bus, err := lc.WatchIPNBus(ctx, ipn.NotifyWatchEngineUpdates|ipn.NotifyInitialNetMap) if err != nil { log.Fatalf("watching IPN bus: %v", err) @@ -155,28 +155,30 @@ func run(ctx context.Context, ts *tsnet.Server, wgPort int, hostname string, pro log.Fatalf("reading IPN bus: %v", err) } - // NetMap contains app-connector configuration - if nm := msg.NetMap; nm != nil && nm.SelfNode.Valid() { - var c appctype.AppConnectorConfig - nmConf, err := tailcfg.UnmarshalNodeCapViewJSON[appctype.AppConnectorConfig](nm.SelfNode.CapMap(), configCapKey) - if err != nil { - log.Printf("failed to read app connector configuration from coordination server: %v", err) - } else if len(nmConf) > 0 { - c = nmConf[0] - } - - if c.AdvertiseRoutes { - if err := s.advertiseRoutesFromConfig(ctx, &c); err != nil { - log.Printf("failed to advertise routes: %v", err) - } - } - - // Backwards compatibility: combine any configuration from control with flags specified - // on the command line. This is intentionally done after we advertise any routes - // because its never correct to advertise the nodes native IP addresses. - s.mergeConfigFromFlags(&c, ports, forwards) - s.srv.Configure(&c) + self := msg.SelfChange + if self == nil { + continue } + var c appctype.AppConnectorConfig + // View() lets us reuse the existing CapView decoder. + nmConf, err := tailcfg.UnmarshalNodeCapViewJSON[appctype.AppConnectorConfig](self.View().CapMap(), configCapKey) + if err != nil { + log.Printf("failed to read app connector configuration from coordination server: %v", err) + } else if len(nmConf) > 0 { + c = nmConf[0] + } + + if c.AdvertiseRoutes { + if err := s.advertiseRoutesFromConfig(ctx, &c); err != nil { + log.Printf("failed to advertise routes: %v", err) + } + } + + // Backwards compatibility: combine any configuration from control with flags specified + // on the command line. This is intentionally done after we advertise any routes + // because its never correct to advertise the nodes native IP addresses. + s.mergeConfigFromFlags(&c, ports, forwards) + s.srv.Configure(&c) } } diff --git a/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go b/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go index a2cd3acd2..4f4262d9f 100644 --- a/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go +++ b/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go @@ -86,19 +86,20 @@ func main() { if cm.User() == "banners" { totalBanners = 5 } + for banner := 2; banner <= totalBanners; banner++ { time.Sleep(time.Second) if banner == totalBanners { - spac.SendAuthBanner(fmt.Sprintf("# Banner%d: access granted at %v\r\n", banner, time.Since(start))) + spac.SendAuthBanner(fmt.Sprintf("# Final banner saying access granted (+%v)\r\n", time.Since(start).Round(time.Millisecond))) } else { - spac.SendAuthBanner(fmt.Sprintf("# Banner%d at %v\r\n", banner, time.Since(start))) + spac.SendAuthBanner(fmt.Sprintf("# Another banner saying we're still waiting for auth server-side (+%v)\r\n", time.Since(start).Round(time.Millisecond))) } } return nil, nil }, BannerCallback: func(cm ssh.ConnMetadata) string { log.Printf("Got connection from user %q, %q from %v", cm.User(), cm.ClientVersion(), cm.RemoteAddr()) - return fmt.Sprintf("# Banner for user %q, %q\n", cm.User(), cm.ClientVersion()) + return fmt.Sprintf("# Example URL in auth bannner for %q, %q: https://github.com/tailscale/tailscale\r\n", cm.User(), cm.ClientVersion()) }, } }, @@ -136,6 +137,8 @@ func handleSessionPostSSHAuth(s gliderssh.Session) { } }() + fmt.Fprintf(s, "We're past auth phase now. Goodbye in ...\n") + for i := 10; i > 0; i-- { fmt.Fprintf(s, "%v ...\n", i) time.Sleep(time.Second) diff --git a/cmd/stund/depaware.txt b/cmd/stund/depaware.txt index 7804915dc..50538b6c3 100644 --- a/cmd/stund/depaware.txt +++ b/cmd/stund/depaware.txt @@ -9,6 +9,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/v1 from tailscale.com/types/logger github.com/munnerz/goautoneg from github.com/prometheus/common/expfmt 💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb/promvarz github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus @@ -61,6 +62,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar tailscale.com/net/tsaddr from tailscale.com/tsweb tailscale.com/syncs from tailscale.com/metrics+ tailscale.com/tailcfg from tailscale.com/version+ + tailscale.com/tstime from tailscale.com/tsweb tailscale.com/tsweb from tailscale.com/cmd/stund+ tailscale.com/tsweb/promvarz from tailscale.com/cmd/stund tailscale.com/tsweb/varz from tailscale.com/tsweb+ @@ -140,22 +142,22 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ - crypto/internal/fips140 from crypto/internal/fips140/aes+ + crypto/internal/fips140 from crypto/fips140+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ crypto/internal/fips140/alias from crypto/cipher+ crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ - crypto/internal/fips140/check from crypto/internal/fips140/aes+ - crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/check from crypto/fips140+ + crypto/internal/fips140/drbg from crypto/hpke+ crypto/internal/fips140/ecdh from crypto/ecdh crypto/internal/fips140/ecdsa from crypto/ecdsa crypto/internal/fips140/ed25519 from crypto/ed25519 crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 crypto/internal/fips140/edwards25519/field from crypto/ecdh+ - crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ + crypto/internal/fips140/hkdf from crypto/hkdf+ crypto/internal/fips140/hmac from crypto/hmac+ crypto/internal/fips140/mlkem from crypto/mlkem - crypto/internal/fips140/nistec from crypto/elliptic+ + crypto/internal/fips140/nistec from crypto/ecdsa+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ @@ -220,13 +222,13 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar internal/filepathlite from os+ internal/fmtsort from fmt internal/goarch from crypto/internal/fips140deps/cpu+ - internal/godebug from crypto/internal/fips140deps/godebug+ + internal/godebug from crypto/ed25519+ internal/godebugs from internal/godebug+ internal/goexperiment from net/http/pprof+ internal/goos from crypto/x509+ internal/msan from internal/runtime/maps+ internal/nettrace from net+ - internal/oserror from io/fs+ + internal/oserror from internal/syscall/windows+ internal/poll from net+ internal/profile from net/http/pprof internal/profilerecord from runtime+ @@ -236,9 +238,9 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar internal/runtime/atomic from internal/runtime/exithook+ L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime - internal/runtime/gc from runtime+ + internal/runtime/gc from internal/runtime/gc/scan+ internal/runtime/gc/scan from runtime - internal/runtime/maps from reflect+ + internal/runtime/maps from hash/maphash+ internal/runtime/math from internal/runtime/maps+ internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ @@ -252,7 +254,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar internal/synctest from sync internal/syscall/execenv from os LD internal/syscall/unix from crypto/internal/sysrand+ - W internal/syscall/windows from crypto/internal/sysrand+ + W internal/syscall/windows from crypto/internal/fips140deps/time+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 9dc35f142..68a339782 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -15,9 +15,11 @@ import ( ) var socket = flag.String("socket", paths.DefaultTailscaledSocket(), "path to tailscaled socket") +var theme = flag.String("theme", "dark", "color theme for Tailscale icon: dark, dark:nobg, light, light:nobg") func main() { flag.Parse() lc := &local.Client{Socket: *socket} + systray.SetTheme(*theme) new(systray.Menu).Run(lc) } diff --git a/cmd/tailscale/cli/appcroutes.go b/cmd/tailscale/cli/appcroutes.go index 04cbcdd83..73b251df1 100644 --- a/cmd/tailscale/cli/appcroutes.go +++ b/cmd/tailscale/cli/appcroutes.go @@ -28,9 +28,9 @@ var appcRoutesCmd = &ffcli.Command{ ShortHelp: "Print the current app connector routes", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("appc-routes") - fs.BoolVar(&appcRoutesArgs.all, "all", false, "Print learned domains and routes and extra policy configured routes.") - fs.BoolVar(&appcRoutesArgs.domainMap, "map", false, "Print the map of learned domains: [routes].") - fs.BoolVar(&appcRoutesArgs.n, "n", false, "Print the total number of routes this node advertises.") + fs.BoolVar(&appcRoutesArgs.all, "all", false, "print learned domains and routes and extra policy configured routes.") + fs.BoolVar(&appcRoutesArgs.domainMap, "map", false, "print the map of learned domains: [routes].") + fs.BoolVar(&appcRoutesArgs.n, "n", false, "print the total number of routes this node advertises.") return fs })(), LongHelp: strings.TrimSpace(` @@ -40,7 +40,7 @@ By default this command prints the domains configured in the app connector confi learned for each domain. --all prints the routes learned from the domains configured in the app connector configuration; and any extra routes provided -in the the policy app connector 'routes' field. +in the policy app connector 'routes' field. --map prints the routes learned from the domains configured in the app connector configuration. diff --git a/cmd/tailscale/cli/cert.go b/cmd/tailscale/cli/cert.go index 6d78a8d8a..bab83901f 100644 --- a/cmd/tailscale/cli/cert.go +++ b/cmd/tailscale/cli/cert.go @@ -23,7 +23,10 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "software.sslmate.com/src/go-pkcs12" "tailscale.com/atomicfile" + "tailscale.com/feature/buildfeatures" + "tailscale.com/health" "tailscale.com/ipn" + "tailscale.com/tsconst" "tailscale.com/version" ) @@ -112,6 +115,11 @@ func runCert(ctx context.Context, args []string) error { certArgs.certFile = fileBase + ".crt" certArgs.keyFile = fileBase + ".key" } + if buildfeatures.HasHealth { + watchCtx, cancel := context.WithCancel(ctx) + defer cancel() + go watchCertPendingHealth(watchCtx, domain) + } certPEM, keyPEM, err := localClient.CertPairWithValidity(ctx, domain, certArgs.minValidity) if err != nil { return err @@ -167,6 +175,44 @@ func runCert(ctx context.Context, args []string) error { return nil } +// watchCertPendingHealth subscribes to the IPN bus and prints the +// [tsconst.HealthWarnableTLSCertPending] warning to stderr if it appears +// for domain while a cert fetch is in flight. It returns once it has +// printed the warning or ctx is done. +// +// Subscription is delayed 1 second so we don't print anything when the +// daemon returns a cached cert quickly. +func watchCertPendingHealth(ctx context.Context, domain string) { + select { + case <-time.After(1 * time.Second): + case <-ctx.Done(): + return + } + watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialHealthState|ipn.NotifyNoNetMap) + if err != nil { + return + } + defer watcher.Close() + for { + n, err := watcher.Next() + if err != nil { + return + } + if n.Health == nil { + continue + } + ws, ok := n.Health.Warnings[tsconst.HealthWarnableTLSCertPending] + if !ok { + continue + } + if !strings.Contains(ws.Args[health.ArgDomains], domain) { + continue + } + fmt.Fprintf(os.Stderr, "%s: %s\n", ws.Title, ws.Text) + return + } +} + func writeIfChanged(filename string, contents []byte, mode os.FileMode) (changed bool, err error) { if filename == "-" { Stdout.Write(contents) diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index 8a2c2b9ef..2f9243013 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -28,6 +28,7 @@ import ( "tailscale.com/feature" "tailscale.com/paths" "tailscale.com/util/slicesx" + "tailscale.com/util/testenv" "tailscale.com/version/distro" ) @@ -92,8 +93,8 @@ var localClient = local.Client{ Socket: paths.DefaultTailscaledSocket(), } -// Run runs the CLI. The args do not include the binary name. -func Run(args []string) (err error) { +// RunWithContext runs the CLI. The args do not include the binary name. +func RunWithContext(ctx context.Context, args []string) (err error) { if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 && len(args) == 0 { // We're running on gokrazy and the user did not specify 'up'. // Don't run the tailscale CLI and spam logs with usage; just exit. @@ -163,7 +164,7 @@ func Run(args []string) (err error) { return } - err = rootCmd.Run(context.Background()) + err = rootCmd.Run(ctx) if local.IsAccessDeniedError(err) && os.Getuid() != 0 && runtime.GOOS != "windows" { return fmt.Errorf("%v\n\nUse 'sudo tailscale %s'.\nTo not require root, use 'sudo tailscale set --operator=$USER' once.", err, strings.Join(args, " ")) } @@ -173,6 +174,11 @@ func Run(args []string) (err error) { return err } +// Run is equivalent to calling [RunWithContext] with the background context. +func Run(args []string) (err error) { + return RunWithContext(context.Background(), args) +} + type onceFlagValue struct { flag.Value set bool @@ -194,26 +200,49 @@ func (v *onceFlagValue) IsBoolFlag() bool { return ok && bf.IsBoolFlag() } -// noDupFlagify modifies c recursively to make all the -// flag values be wrappers that permit setting the value -// at most once. -func noDupFlagify(c *ffcli.Command) { - if c.FlagSet != nil { - c.FlagSet.VisitAll(func(f *flag.Flag) { - f.Value = &onceFlagValue{Value: f.Value} - }) +// noDupFlagify modifies c recursively to make all the flag values be +// wrappers that permit setting the value at most once. If tb is +// non-nil, the original values are restored when the test completes. +func noDupFlagify(c *ffcli.Command, tb testenv.TB) { + if tb == nil && testenv.InTest() { + return } - for _, sub := range c.Subcommands { - noDupFlagify(sub) + type restore struct { + f *flag.Flag + v flag.Value + } + var restores []restore + var walk func(*ffcli.Command) + walk = func(c *ffcli.Command) { + if c.FlagSet != nil { + c.FlagSet.VisitAll(func(f *flag.Flag) { + if tb != nil { + restores = append(restores, restore{f, f.Value}) + } + f.Value = &onceFlagValue{Value: f.Value} + }) + } + for _, sub := range c.Subcommands { + walk(sub) + } + } + walk(c) + if tb != nil { + tb.Cleanup(func() { + for _, r := range restores { + r.f.Value = r.v + } + }) } } var ( fileCmd, sysPolicyCmd, + maybeRoutecheckCmd, maybeWebCmd, maybeDriveCmd, - maybeNetlockCmd, + maybeTailnetLockCmd, maybeFunnelCmd, maybeServeCmd, maybeCertCmd, @@ -221,7 +250,7 @@ var ( _ func() *ffcli.Command ) -func newRootCmd() *ffcli.Command { +func newRootCmd(tb ...testenv.TB) *ffcli.Command { rootfs := newFlagSet("tailscale") rootfs.Func("socket", "path to tailscaled socket", func(s string) error { localClient.Socket = s @@ -246,12 +275,14 @@ change in the future. upCmd, downCmd, setCmd, + getCmd, loginCmd, logoutCmd, switchCmd, configureCmd(), nilOrCall(sysPolicyCmd), netcheckCmd, + nilOrCall(maybeRoutecheckCmd), ipCmd, dnsCmd, statusCmd, @@ -261,16 +292,18 @@ change in the future. sshCmd, nilOrCall(maybeFunnelCmd), nilOrCall(maybeServeCmd), + serviceCmd, versionCmd, nilOrCall(maybeWebCmd), nilOrCall(fileCmd), bugReportCmd, nilOrCall(maybeCertCmd), - nilOrCall(maybeNetlockCmd), + nilOrCall(maybeTailnetLockCmd), licensesCmd, exitNodeCmd(), nilOrCall(maybeUpdateCmd), whoisCmd, + whoamiCmd, debugCmd(), nilOrCall(maybeDriveCmd), idTokenCmd, @@ -303,7 +336,11 @@ change in the future. }) ffcomplete.Inject(rootCmd, func(c *ffcli.Command) { c.LongHelp = hidden + c.LongHelp }, usageFunc) - noDupFlagify(rootCmd) + var t testenv.TB + if len(tb) > 0 { + t = tb[0] + } + noDupFlagify(rootCmd, t) return rootCmd } @@ -452,7 +489,7 @@ func usageFuncOpt(c *ffcli.Command, withDefaults bool) string { showDefault := f.DefValue != "" && withDefaults // Issue 6766: don't show the default Windows socket path. It's long - // and distracting. And people on on Windows aren't likely to ever + // and distracting. And people on Windows aren't likely to ever // change it anyway. if runtime.GOOS == "windows" && f.Name == "socket" && strings.HasPrefix(f.DefValue, `\\.\pipe\ProtectedPrefix\`) { showDefault = false diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index f95d84695..6ad7e0888 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -30,6 +30,7 @@ import ( "tailscale.com/tka" "tailscale.com/tstest" "tailscale.com/tstest/deptest" + "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/opt" "tailscale.com/types/persist" @@ -769,7 +770,22 @@ func TestPrefsFromUpArgs(t *testing.T) { args: upArgsT{ exitNodeIP: "foo", }, - wantErr: `invalid value "foo" for --exit-node; must be IP or hostname`, + st: &ipnstate.Status{ + Peer: map[key.NodePublic]*ipnstate.PeerStatus{ + key.NewNode().Public(): { + DNSName: "example.com.", + TailscaleIPs: []netip.Addr{netip.MustParseAddr("1.0.0.2")}, + }, + }, + }, + wantErr: `invalid value "foo" for --exit-node; must be IP or peer hostname`, + }, + { + name: "error_exit_node_not_started", + args: upArgsT{ + exitNodeIP: "foo", + }, + wantErr: `cannot resolve exit node by hostname while Tailscale is starting up; please use its Tailscale IP address instead`, }, { name: "error_exit_node_allow_lan_without_exit_node", @@ -779,11 +795,43 @@ func TestPrefsFromUpArgs(t *testing.T) { wantErr: `--exit-node-allow-lan-access can only be used with --exit-node`, }, { - name: "error_tag_prefix", + name: "error_tag_bad_prefix", args: upArgsT{ - advertiseTags: "foo", + advertiseTags: "notatag:foo", + }, + wantErr: `tag: "notatag:foo": tags must start with 'tag:'`, + }, + { + name: "tag_auto_prefix", + args: upArgsFromOSArgs("linux", "--advertise-tags=foo,bar"), + want: &ipn.Prefs{ + ControlURL: ipn.DefaultControlURL, + WantRunning: true, + CorpDNS: true, + AdvertiseTags: []string{"tag:foo", "tag:bar"}, + NoSNAT: false, + NoStatefulFiltering: "true", + NetfilterMode: preftype.NetfilterOn, + AutoUpdate: ipn.AutoUpdatePrefs{ + Check: true, + }, + }, + }, + { + name: "tag_mixed_prefix", + args: upArgsFromOSArgs("linux", "--advertise-tags=tag:foo,bar"), + want: &ipn.Prefs{ + ControlURL: ipn.DefaultControlURL, + WantRunning: true, + CorpDNS: true, + AdvertiseTags: []string{"tag:foo", "tag:bar"}, + NoSNAT: false, + NoStatefulFiltering: "true", + NetfilterMode: preftype.NetfilterOn, + AutoUpdate: ipn.AutoUpdatePrefs{ + Check: true, + }, }, - wantErr: `tag: "foo": tags must start with 'tag:'`, }, { name: "error_long_hostname", @@ -968,9 +1016,6 @@ func TestPrefFlagMapping(t *testing.T) { continue } switch prefName { - case "AllowSingleHosts": - // Fake pref for downgrade compat. See #12058. - continue case "WantRunning", "Persist", "LoggedOut": // All explicitly handled (ignored) by checkForAccidentalSettingReverts. continue @@ -1566,7 +1611,7 @@ func TestParseNLArgs(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { - keys, disablements, err := parseNLArgs(tc.input, tc.parseKeys, tc.parseDisablements) + keys, disablements, err := parseTLArgs(tc.input, tc.parseKeys, tc.parseDisablements) if (tc.wantErr == nil && err != nil) || (tc.wantErr != nil && err == nil) || (tc.wantErr != nil && err != nil && tc.wantErr.Error() != err.Error()) { @@ -1618,7 +1663,7 @@ func TestNoDups(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cmd := newRootCmd() + cmd := newRootCmd(t) makeQuietContinueOnError(cmd) err := cmd.Parse(tt.args) if got := fmt.Sprint(err); got != tt.want { diff --git a/cmd/tailscale/cli/configure-flash-appliance.go b/cmd/tailscale/cli/configure-flash-appliance.go new file mode 100644 index 000000000..c9f9af724 --- /dev/null +++ b/cmd/tailscale/cli/configure-flash-appliance.go @@ -0,0 +1,515 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_flashappliance + +package cli + +import ( + "archive/zip" + "context" + "errors" + "flag" + "fmt" + "io" + "net/url" + "os" + "runtime" + "slices" + "sort" + "strings" + + "github.com/bradfitz/monogok/disklayout" + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/clientupdate" + "tailscale.com/clientupdate/distsign" + "tailscale.com/gokrazy/mkfs" + "tailscale.com/util/progresstracking" + "tailscale.com/util/prompt" +) + +var flashApplianceArgs struct { + variant string + disk string + track string + yes bool + gaf string + addSSHAuthorizedKeys string +} + +func flashApplianceCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "flash-appliance", + ShortUsage: "tailscale configure flash-appliance [flags]", + ShortHelp: "Download a signed Tailscale appliance image and write it to a local disk [experimental]", + LongHelp: hidden + strings.TrimSpace(` +This experimental command downloads a signed Tailscale appliance image (Gokrazy archive +format, "GAF") from pkgs.tailscale.com, verifies its signature, and writes +it to a local block device (SD card, USB drive, virtual disk). + +On macOS, the target disk is auto-discovered from 'diskutil list physical', +excluding whichever disks back the running root. On Linux, you must pass +--disk=/dev/sdX explicitly. + +This command requires mkfs.ext4 in $PATH to format the writable /perm +partition. On macOS, 'brew install e2fsprogs' provides it. +`), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("flash-appliance") + fs.StringVar(&flashApplianceArgs.variant, "variant", "", `appliance variant: "pi-arm64", "vm-amd64", or "vm-arm64". Empty prompts interactively.`) + fs.StringVar(&flashApplianceArgs.disk, "disk", "", "target block device (e.g. /dev/sdb or /dev/disk4)") + fs.StringVar(&flashApplianceArgs.track, "track", "", `which track to download from; defaults to "`+clientupdate.CurrentTrack+`"`) + fs.BoolVar(&flashApplianceArgs.yes, "yes", false, "skip the destructive-write confirmation prompt") + fs.StringVar(&flashApplianceArgs.gaf, "gaf", "", "use a local GAF file instead of downloading (skips signature verification)") + fs.StringVar(&flashApplianceArgs.addSSHAuthorizedKeys, "add-ssh-authorized-keys", "", "path to an authorized_keys file to include on the appliance for breakglass SSH access") + return fs + })(), + Exec: runFlashAppliance, + } +} + +func runFlashAppliance(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unknown arguments") + } + if runtime.GOOS == "windows" { + return errors.New("flash-appliance is not supported on Windows yet; consider running under WSL") + } + if os.Geteuid() != 0 { + return errors.New("writing to a raw block device requires root; re-run with sudo") + } + + disk, err := resolveTargetDisk(ctx, flashApplianceArgs.disk) + if err != nil { + return err + } + + gafPath, gafLabel, variant, cleanup, err := obtainGAF(ctx, gafDownloadArgs{ + localGAF: flashApplianceArgs.gaf, + track: flashApplianceArgs.track, + variant: flashApplianceArgs.variant, + }) + if err != nil { + return err + } + defer cleanup() + + zr, err := zip.OpenReader(gafPath) + if err != nil { + return fmt.Errorf("open GAF: %w", err) + } + defer zr.Close() + + bootCode, err := readGAFMember(zr.File, "mbr.img", 1<<20) + if err != nil { + return err + } + + if !flashApplianceArgs.yes { + msg := fmt.Sprintf("This will ERASE %s. Flash %s?", disk.Path, gafLabel) + if !prompt.YesNo(msg, false) { + return errors.New("aborted") + } + } + + printf("Unmounting %s...\n", disk.Path) + if err := unmountDisk(ctx, disk.Path); err != nil { + return fmt.Errorf("unmount %s: %w", disk.Path, err) + } + + if err := writeGAFToDisk(zr.File, disk.Path, bootCode, variant); err != nil { + return err + } + + var permFiles []mkfs.PermFile + if flashApplianceArgs.addSSHAuthorizedKeys != "" { + keys, err := os.ReadFile(flashApplianceArgs.addSSHAuthorizedKeys) + if err != nil { + return fmt.Errorf("reading --add-ssh-authorized-keys: %w", err) + } + permFiles = append(permFiles, mkfs.PermFile{ + Path: "breakglass.authorized_keys", + Content: keys, + }) + printf("Including SSH authorized_keys for breakglass access.\n") + } + if err := formatPermExt4(disk.Path, permFiles); err != nil { + return fmt.Errorf("formatting perm: %w", err) + } + + ejected, err := ejectDisk(ctx, disk.Path) + if err != nil { + // Non-fatal: the user can eject manually. + fmt.Fprintf(Stderr, "ejecting %s: %v\n", disk.Path, err) + } + + printf("Done. %s\n", flashSuccessHint(disk.Path, variant, ejected)) + return nil +} + +// formatPermExt4 creates an ext4 filesystem inside the gokrazy perm +// partition of the disk at diskPath, delegating to gokrazy/mkfs.Perm. +// +// On macOS we open the buffered /dev/diskN path (not /dev/rdiskN) +// because go-diskfs writes ext4 metadata in small unaligned chunks +// that the raw character device rejects. +func formatPermExt4(diskPath string, files []mkfs.PermFile) error { + f, err := os.OpenFile(diskPath, os.O_RDWR, 0) + if err != nil { + return err + } + defer f.Close() + + devsize, err := blockDeviceSize(f) + if err != nil { + return fmt.Errorf("sizing %s: %w", diskPath, err) + } + return mkfs.Perm(f, devsize, files...) +} + +// flashSuccessHint returns a per-variant next-step hint shown after a +// successful flash. variant is empty when the user passed --gaf +// directly. ejected reports whether we already released the disk (true +// on macOS after diskutil eject); when false, the message tells the +// user to eject it themselves. +func flashSuccessHint(diskPath, variant string, ejected bool) string { + verb := "Eject" + if ejected { + verb = "Pull" + } + switch variant { + case "pi-arm64": + return fmt.Sprintf("%s %s and boot your Raspberry Pi.", verb, diskPath) + case "vm-amd64": + return fmt.Sprintf("%s %s and boot an x86_64 VM from it.", verb, diskPath) + case "vm-arm64": + return fmt.Sprintf("%s %s and boot an arm64 VM from it.", verb, diskPath) + default: + return fmt.Sprintf("%s %s and boot the target device.", verb, diskPath) + } +} + +// diskCandidate describes a flashable disk on the host. +type diskCandidate struct { + Path string // e.g. /dev/disk4 or /dev/sdb + SizeBytes int64 + Description string // human-readable model + size, e.g. "Generic MassStorage (62.5 GB)" +} + +func (d diskCandidate) String() string { + if d.Description != "" { + return fmt.Sprintf("%s: %s", d.Path, d.Description) + } + return d.Path +} + +// resolveTargetDisk returns the disk the user wants to flash. On macOS, an +// empty userDisk triggers auto-discovery. On Linux, userDisk is required and +// validated. +func resolveTargetDisk(ctx context.Context, userDisk string) (diskCandidate, error) { + if userDisk != "" { + if err := validateDiskPath(userDisk); err != nil { + return diskCandidate{}, err + } + return diskCandidate{Path: userDisk}, nil + } + + disks, err := discoverExternalDisks(ctx) + if err != nil { + return diskCandidate{}, err + } + switch len(disks) { + case 0: + return diskCandidate{}, errors.New("no candidate disks found; insert an SD card or USB drive, or pass --disk") + case 1: + printf("Found 1 candidate disk: %s\n", disks[0]) + return disks[0], nil + default: + printf("Multiple candidate disks found:\n") + for i, d := range disks { + printf(" %d) %s\n", i+1, d) + } + return diskCandidate{}, errors.New("pass --disk=/dev/... to pick one") + } +} + +// gafDownloadArgs are the inputs to [obtainGAF]. All fields are optional; +// zero values mean "download the latest and prompt/pick sensible defaults". +type gafDownloadArgs struct { + localGAF string // if set, skip the network and use this local GAF path + track string // release track (empty → clientupdate.CurrentTrack) + variant string // GAF variant key (e.g. "vm-amd64"); empty prompts interactively +} + +// obtainGAF returns a path to a local GAF file the caller can read, +// along with the appliance variant it corresponds to (empty for the +// --gaf path). If args.localGAF is set, the local file is returned +// directly. Otherwise the latest appliance GAF is fetched from +// pkgs.tailscale.com (with signature verification) into a temp file. +// cleanup removes any temp file it created. +func obtainGAF(ctx context.Context, args gafDownloadArgs) (path, label, variant string, cleanup func(), err error) { + cleanup = func() {} + if args.localGAF != "" { + // With a local GAF there's no manifest to learn the variant + // from, so we trust whatever variant the caller passed (may be + // empty). rootArchForVariant defaults to arm64 when empty. + return args.localGAF, args.localGAF, args.variant, cleanup, nil + } + + track := args.track + if track == "" { + track = clientupdate.CurrentTrack + } + latest, err := clientupdate.LatestPackages(track) + if err != nil { + return "", "", "", cleanup, fmt.Errorf("fetching package manifest: %w", err) + } + if len(latest.GAFs) == 0 { + return "", "", "", cleanup, fmt.Errorf("no appliance GAFs published on %q track", track) + } + + variant, err = pickVariant(latest.GAFs, args.variant) + if err != nil { + return "", "", "", cleanup, err + } + gafName := latest.GAFs[variant] + + gafURL, err := url.JoinPath("https://pkgs.tailscale.com", track, gafName) + if err != nil { + return "", "", "", cleanup, err + } + + tmp, err := os.CreateTemp("", "tailscale-flash-*.gaf") + if err != nil { + return "", "", "", cleanup, err + } + tmpName := tmp.Name() + tmp.Close() + cleanup = func() { os.Remove(tmpName) } + + printf("Downloading %s (version %s)\n", gafURL, latest.GAFsVersion) + logf := func(format string, args ...any) { fmt.Fprintf(Stderr, format+"\n", args...) } + if err := distsign.DownloadVerified(ctx, logf, gafURL, tmpName); err != nil { + cleanup() + return "", "", "", func() {}, fmt.Errorf("download GAF: %w", err) + } + return tmpName, fmt.Sprintf("%s (%s)", gafName, latest.GAFsVersion), variant, cleanup, nil +} + +// pickVariant returns the variant key from gafs the user wants. If +// chosen is non-empty, it's validated against the available keys. +// Otherwise the caller is prompted to choose one on the next invocation. +func pickVariant(gafs map[string]string, chosen string) (string, error) { + variants := make([]string, 0, len(gafs)) + for k := range gafs { + variants = append(variants, k) + } + sort.Strings(variants) + + if chosen != "" { + if !slices.Contains(variants, chosen) { + return "", fmt.Errorf("variant %q not published; available: %s", chosen, strings.Join(variants, ", ")) + } + return chosen, nil + } + + printf("Available appliance variants:\n") + for i, v := range variants { + printf(" %d) %s\n", i+1, v) + } + return "", fmt.Errorf("pass --variant=", strings.Join(variants, "|")) +} + +// readGAFMember returns the contents of a named member of the GAF zip. +// It returns an error if the member is missing or larger than maxBytes. +func readGAFMember(files []*zip.File, name string, maxBytes int64) ([]byte, error) { + for _, f := range files { + if f.Name != name { + continue + } + if int64(f.UncompressedSize64) > maxBytes { + return nil, fmt.Errorf("%s is %d bytes; refusing to read more than %d", name, f.UncompressedSize64, maxBytes) + } + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() + return io.ReadAll(rc) + } + return nil, fmt.Errorf("GAF is missing %s", name) +} + +// writeGAFToDisk writes a fresh gokrazy install to diskPath: the +// protective MBR (with bootCode in the first 446 bytes), the primary +// and secondary GPT, then boot.img at the boot partition's offset and +// root.img at root A's offset. Root B and perm are left untouched — the +// appliance populates root B on first boot, and the caller formats +// perm with mkfs.ext4. +func writeGAFToDisk(files []*zip.File, diskPath string, bootCode []byte, variant string) error { + f, err := openBlockDevice(diskPath) + if err != nil { + return err + } + defer f.Close() + + devsize, err := blockDeviceSize(f) + if err != nil { + return fmt.Errorf("sizing %s: %w", diskPath, err) + } + if devsize <= 0 { + return fmt.Errorf("could not determine size of %s", diskPath) + } + + if err := writeApplianceImage(f, devsize, files, bootCode, variant); err != nil { + return err + } + + if err := syncBlockDevice(f); err != nil { + return fmt.Errorf("fsync %s: %w", diskPath, err) + } + if err := rereadPartitionTable(f); err != nil { + return fmt.Errorf("reread partition table: %w", err) + } + return nil +} + +// writeApplianceImage writes the gokrazy install (protective MBR + GPT, +// boot.img, root.img) to f, whose usable size is devsize bytes. It does +// not fsync or reread the partition table; those are the caller's +// responsibility if targeting a block device. +// +// f is used for random-access seeks and writes and must already be sized +// to devsize (a fresh block device, or a regular file that the caller +// has truncated to devsize). +func writeApplianceImage(f *os.File, devsize int64, files []*zip.File, bootCode []byte, variant string) error { + if len(bootCode) > 446 { + return fmt.Errorf("mbr.img is %d bytes; expected at most 446", len(bootCode)) + } + + if err := checkPartitionFits(files, "boot.img", int64(disklayout.BootPartitionSizeMB)<<20); err != nil { + return err + } + if err := checkPartitionFits(files, "root.img", int64(disklayout.RootPartitionSizeMB)<<20); err != nil { + return err + } + + bootImg, err := readGAFMember(files, "boot.img", int64(disklayout.BootPartitionSizeMB)<<20) + if err != nil { + return err + } + partUUID, err := partUUIDFromBootImg(bootImg) + if err != nil { + return fmt.Errorf("locating gokrazy partuuid in boot.img: %w", err) + } + + printf("Writing protective MBR + GPT (partuuid=%08x, arch=%s)\n", partUUID, rootArchForVariant(variant)) + if err := disklayout.WriteGPT(f, uint64(devsize), disklayout.DefaultBootPartitionStartLBA, bootCode, partUUID, rootArchForVariant(variant)); err != nil { + return fmt.Errorf("writing GPT: %w", err) + } + + writes := []struct { + member string + offsetLBA uint32 + }{ + {"boot.img", disklayout.BootStartLBA(disklayout.DefaultBootPartitionStartLBA)}, + {"root.img", disklayout.RootAStartLBA(disklayout.DefaultBootPartitionStartLBA)}, + } + for _, w := range writes { + zf := findZipMember(files, w.member) + if zf == nil { + return fmt.Errorf("GAF is missing %s", w.member) + } + printf("Writing %s (%d bytes) at sector %d\n", w.member, zf.UncompressedSize64, w.offsetLBA) + if err := writeZipMemberAt(f, zf, int64(w.offsetLBA)*512); err != nil { + return fmt.Errorf("writing %s: %w", w.member, err) + } + } + return nil +} + +// rootArchForVariant picks the GPT root partition type architecture +// based on the GAF variant key (e.g. "pi-arm64" → arm64). +func rootArchForVariant(variant string) disklayout.RootArch { + switch { + case strings.HasSuffix(variant, "-amd64"): + return disklayout.ArchAMD64 + default: + // pi-arm64, vm-arm64, or empty (--gaf path): arm64 is the + // default for tailscale appliance images. + return disklayout.ArchARM64 + } +} + +// partUUIDFromBootImg returns the gokrazy per-disk partuuid embedded in +// boot.img's cmdline.txt. We byte-search the FAT image for the +// "PARTUUID=60c24cc1-..." pattern rather than parsing FAT, which is +// good enough since the only thing on disk with that prefix is +// cmdline.txt. +func partUUIDFromBootImg(boot []byte) (uint32, error) { + return disklayout.ParseCmdlinePartUUID(string(boot)) +} + +// checkPartitionFits returns an error if the named GAF member is too +// large to fit in a partition of maxBytes. +func checkPartitionFits(files []*zip.File, name string, maxBytes int64) error { + zf := findZipMember(files, name) + if zf == nil { + return fmt.Errorf("GAF is missing %s", name) + } + if got := int64(zf.UncompressedSize64); got > maxBytes { + return fmt.Errorf("%s is %d bytes; gokrazy layout allows up to %d", name, got, maxBytes) + } + return nil +} + +func findZipMember(files []*zip.File, name string) *zip.File { + for _, f := range files { + if f.Name == name { + return f + } + } + return nil +} + +func writeZipMemberAt(f *os.File, zf *zip.File, offset int64) error { + rc, err := zf.Open() + if err != nil { + return err + } + defer rc.Close() + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return err + } + total := int64(zf.UncompressedSize64) + cw := &progresstracking.CountingWriter{W: f} + stop := progresstracking.Ticker(cw.Count, total, func(d, t int64) { + pct := 0.0 + if t > 0 { + pct = float64(d) * 100 / float64(t) + } + fmt.Fprintf(Stderr, " %s: %s / %s (%.1f%%)\n", zf.Name, humanBytes(d), humanBytes(t), pct) + }) + defer stop() + _, err = io.Copy(cw, rc) + return err +} + +// humanBytes returns a friendly approximation of n bytes, e.g. "62.5 GB". +func humanBytes(n int64) string { + const ( + gb = 1 << 30 + mb = 1 << 20 + kb = 1 << 10 + ) + switch { + case n >= gb: + return fmt.Sprintf("%.1f GB", float64(n)/float64(gb)) + case n >= mb: + return fmt.Sprintf("%.1f MB", float64(n)/float64(mb)) + case n >= kb: + return fmt.Sprintf("%.1f KB", float64(n)/float64(kb)) + default: + return fmt.Sprintf("%d B", n) + } +} diff --git a/cmd/tailscale/cli/configure-flash-appliance_darwin.go b/cmd/tailscale/cli/configure-flash-appliance_darwin.go new file mode 100644 index 000000000..254fd6de6 --- /dev/null +++ b/cmd/tailscale/cli/configure-flash-appliance_darwin.go @@ -0,0 +1,430 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_flashappliance + +package cli + +import ( + "context" + "encoding/xml" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + + "golang.org/x/sys/unix" +) + +// maxAutoDetectDiskBytes is the upper size limit for a disk that +// flash-appliance auto-discovers. Anything larger is reported to the +// user but skipped from the candidate list, so it's harder to wipe an +// unmounted internal SSD or a backup drive by accident; the user can +// still target it with --disk explicitly. +const maxAutoDetectDiskBytes = 256 << 30 + +// discoverExternalDisks returns the physical disks suitable for flashing. +// We pass just "physical" (not "external physical") to diskutil because +// macOS reports built-in SD card readers as internal; instead we exclude +// whichever whole disks back the running root. +func discoverExternalDisks(ctx context.Context) ([]diskCandidate, error) { + out, err := exec.CommandContext(ctx, "diskutil", "list", "-plist", "physical").Output() + if err != nil { + return nil, fmt.Errorf("diskutil list: %w", err) + } + ids, err := parseDiskutilListPlist(out) + if err != nil { + return nil, fmt.Errorf("parse diskutil list output: %w", err) + } + boot, err := bootWholeDisks(ctx) + if err != nil { + return nil, fmt.Errorf("locating boot disk: %w", err) + } + disks := make([]diskCandidate, 0, len(ids)) + for _, id := range ids { + if boot[id] { + continue + } + d, err := diskutilInfo(ctx, id) + if err != nil { + return nil, err + } + if d.SizeBytes > maxAutoDetectDiskBytes { + printf("Skipping %s (%s) from auto-detection: looks suspiciously large.\n", d.Path, humanBytes(d.SizeBytes)) + printf(" To flash it anyway, pass --disk=%s explicitly.\n", d.Path) + continue + } + disks = append(disks, d) + } + return disks, nil +} + +var darwinWholeDiskRe = regexp.MustCompile(`^(disk\d+)`) + +// bootWholeDisks returns the set of whole-disk identifiers (e.g. "disk0") +// that back the running root filesystem. It seeds the walk from `df -P /` +// (which on Apple Silicon points to the sealed snapshot, e.g. +// disk3s1s1) and follows ParentWholeDisk and APFSPhysicalStores so that +// both the synthesized APFS container (disk3) and the physical disk +// behind it (disk0) get excluded from flash candidates. +func bootWholeDisks(ctx context.Context) (map[string]bool, error) { + rootDev, err := dfRootDevice(ctx) + if err != nil { + return nil, fmt.Errorf("locating root device: %w", err) + } + + boot := map[string]bool{} + seen := map[string]bool{} + queue := []string{rootDev} + for len(queue) > 0 { + id := queue[0] + queue = queue[1:] + if seen[id] { + continue + } + seen[id] = true + + if m := darwinWholeDiskRe.FindString(id); m != "" { + boot[m] = true + } + + out, err := exec.CommandContext(ctx, "diskutil", "info", "-plist", id).Output() + if err != nil { + // Skip identifiers diskutil can't resolve (e.g. a physical + // store on a disk that was unplugged); anything already + // collected stays excluded. + continue + } + info, err := parseDiskutilInfoPlist(out) + if err != nil { + continue + } + if d := info.ParentWholeDisk; d != "" { + queue = append(queue, d) + } + queue = append(queue, info.APFSPhysicalStores...) + } + return boot, nil +} + +// dfRootDevice returns the device identifier (e.g. "disk3s1s1") that +// backs the root mount, by parsing the second line of `df -P /`. +func dfRootDevice(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, "df", "-P", "/").Output() + if err != nil { + return "", err + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + if len(lines) < 2 { + return "", fmt.Errorf("unexpected df output: %q", out) + } + fields := strings.Fields(lines[1]) + if len(fields) == 0 { + return "", fmt.Errorf("unexpected df line: %q", lines[1]) + } + return strings.TrimPrefix(fields[0], "/dev/"), nil +} + +func diskutilInfo(ctx context.Context, id string) (diskCandidate, error) { + out, err := exec.CommandContext(ctx, "diskutil", "info", "-plist", id).Output() + if err != nil { + return diskCandidate{}, fmt.Errorf("diskutil info %s: %w", id, err) + } + info, err := parseDiskutilInfoPlist(out) + if err != nil { + return diskCandidate{}, fmt.Errorf("parse diskutil info %s: %w", id, err) + } + desc := info.Model + if desc == "" { + desc = info.MediaName + } + if info.Size > 0 { + desc = strings.TrimSpace(fmt.Sprintf("%s (%s)", desc, humanBytes(info.Size))) + } + return diskCandidate{ + Path: "/dev/" + id, + SizeBytes: info.Size, + Description: desc, + }, nil +} + +// validateDiskPath checks that the user-provided disk path looks sane to +// flash on macOS. We trust the user more than on Linux since they had to +// type a /dev/disk path explicitly. +func validateDiskPath(path string) error { + if !strings.HasPrefix(path, "/dev/disk") { + return fmt.Errorf("disk path %q does not look like a macOS whole-disk device (/dev/diskN)", path) + } + if strings.Contains(path, "s") && strings.IndexByte(path, 's') > len("/dev/disk") { + return fmt.Errorf("disk path %q looks like a partition (/dev/diskNsP); pass the whole disk", path) + } + return nil +} + +// unmountDisk uses `diskutil unmountDisk` to release all partitions on the +// target disk. +func unmountDisk(ctx context.Context, path string) error { + cmd := exec.CommandContext(ctx, "diskutil", "unmountDisk", path) + cmd.Stdout = Stderr + cmd.Stderr = Stderr + return cmd.Run() +} + +// ejectDisk runs `diskutil eject` so the user can pull the SD card or +// USB drive without macOS complaining about an improper eject. Returns +// true if the eject command ran successfully. +func ejectDisk(ctx context.Context, path string) (bool, error) { + cmd := exec.CommandContext(ctx, "diskutil", "eject", path) + cmd.Stdout = Stderr + cmd.Stderr = Stderr + if err := cmd.Run(); err != nil { + return false, err + } + return true, nil +} + +// openBlockDevice opens the whole-disk device for writing. On macOS we use +// the raw "rdiskN" alias because the buffered "diskN" path is much slower +// for large writes. +func openBlockDevice(path string) (*os.File, error) { + raw := strings.Replace(path, "/dev/disk", "/dev/rdisk", 1) + return os.OpenFile(raw, os.O_WRONLY, 0) +} + +// rereadPartitionTable is a no-op on macOS; diskutil and the kernel pick up +// partition changes when the device is closed and re-opened. +func rereadPartitionTable(_ *os.File) error { return nil } + +// macOS ioctls from . lseek(SEEK_END) returns 0 on raw +// (/dev/rdiskN) devices, so we have to compute the size from the block +// size and block count. +const ( + dkiocGetBlockSize = 0x40046418 // _IOR('d', 24, uint32_t) + dkiocGetBlockCount = 0x40086419 // _IOR('d', 25, uint64_t) +) + +// syncBlockDevice asks the kernel to flush in-flight writes to disk. On +// macOS, /dev/rdiskN is the unbuffered raw device, so its writes are +// already synchronous and fsync returns ENOTTY ("inappropriate ioctl +// for device"). We try F_FULLFSYNC for completeness and tolerate the +// same ENOTTY there. +func syncBlockDevice(f *os.File) error { + _, err := unix.FcntlInt(f.Fd(), unix.F_FULLFSYNC, 0) + if err == nil || err == unix.ENOTTY { + return nil + } + return err +} + +// blockDeviceSize returns the size in bytes of the open block device f. +// On little-endian darwin, IoctlGetInt's 8-byte int safely receives +// both a 4-byte uint32 (block size) and an 8-byte uint64 (block count). +func blockDeviceSize(f *os.File) (int64, error) { + blockSize, err := unix.IoctlGetInt(int(f.Fd()), dkiocGetBlockSize) + if err != nil { + return 0, fmt.Errorf("DKIOCGETBLOCKSIZE: %w", err) + } + blockCount, err := unix.IoctlGetInt(int(f.Fd()), dkiocGetBlockCount) + if err != nil { + return 0, fmt.Errorf("DKIOCGETBLOCKCOUNT: %w", err) + } + return int64(blockSize) * int64(blockCount), nil +} + +// diskutilInfoFields are the fields we care about from `diskutil info -plist`. +type diskutilInfoFields struct { + Model string + MediaName string + Size int64 + ParentWholeDisk string // e.g. "disk3" for "/" on APFS + APFSPhysicalStores []string // e.g. ["disk0s2"] for "/" on APFS +} + +// parseDiskutilListPlist returns the WholeDisk device identifiers from the +// output of `diskutil list -plist external physical`. +func parseDiskutilListPlist(data []byte) ([]string, error) { + type listPlist struct { + Dict plistDict `xml:"dict"` + } + var p listPlist + if err := xml.Unmarshal(data, &p); err != nil { + return nil, err + } + arr, ok := p.Dict.Get("WholeDisks").(plistArray) + if !ok { + return nil, nil + } + var out []string + for _, v := range arr { + if s, ok := v.(string); ok { + out = append(out, s) + } + } + return out, nil +} + +// parseDiskutilInfoPlist returns the fields we care about from `diskutil +// info -plist `. +func parseDiskutilInfoPlist(data []byte) (diskutilInfoFields, error) { + type infoPlist struct { + Dict plistDict `xml:"dict"` + } + var p infoPlist + if err := xml.Unmarshal(data, &p); err != nil { + return diskutilInfoFields{}, err + } + var out diskutilInfoFields + if s, ok := p.Dict.Get("MediaName").(string); ok { + out.MediaName = s + } + if s, ok := p.Dict.Get("DeviceModel").(string); ok { + out.Model = s + } else if s, ok := p.Dict.Get("IORegistryEntryName").(string); ok { + out.Model = s + } + if i, ok := p.Dict.Get("Size").(int64); ok { + out.Size = i + } else if i, ok := p.Dict.Get("TotalSize").(int64); ok { + out.Size = i + } + if s, ok := p.Dict.Get("ParentWholeDisk").(string); ok { + out.ParentWholeDisk = s + } + if arr, ok := p.Dict.Get("APFSPhysicalStores").(plistArray); ok { + // The key inside each entry is APFSPhysicalStore (singular) on + // macOS 14+; older releases may use DeviceIdentifier. Accept + // either. + for _, v := range arr { + d, ok := v.(plistDict) + if !ok { + continue + } + if id, ok := d.Get("APFSPhysicalStore").(string); ok { + out.APFSPhysicalStores = append(out.APFSPhysicalStores, id) + } else if id, ok := d.Get("DeviceIdentifier").(string); ok { + out.APFSPhysicalStores = append(out.APFSPhysicalStores, id) + } + } + } + return out, nil +} + +// plistDict and plistArray support unmarshaling a small subset of Apple +// XML plists. They preserve key order and decode , , +// , , , and nested elements. +type plistDict []plistEntry + +type plistEntry struct { + Key string + Value any +} + +type plistArray []any + +// Get returns the value for a top-level key, or nil if absent. +func (d plistDict) Get(key string) any { + for _, e := range d { + if e.Key == key { + return e.Value + } + } + return nil +} + +// UnmarshalXML decodes the children of a element as alternating +// ... and value elements. +func (d *plistDict) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error { + for { + tok, err := dec.Token() + if err != nil { + return err + } + switch t := tok.(type) { + case xml.EndElement: + if t.Name == start.Name { + return nil + } + case xml.StartElement: + if t.Name.Local != "key" { + return fmt.Errorf("dict child %q is not ", t.Name.Local) + } + var key string + if err := dec.DecodeElement(&key, &t); err != nil { + return err + } + vtok, err := nextStart(dec) + if err != nil { + return err + } + v, err := decodePlistValue(dec, vtok) + if err != nil { + return err + } + *d = append(*d, plistEntry{Key: key, Value: v}) + } + } +} + +func nextStart(dec *xml.Decoder) (xml.StartElement, error) { + for { + tok, err := dec.Token() + if err != nil { + return xml.StartElement{}, err + } + if s, ok := tok.(xml.StartElement); ok { + return s, nil + } + } +} + +func decodePlistValue(dec *xml.Decoder, start xml.StartElement) (any, error) { + switch start.Name.Local { + case "string": + var s string + if err := dec.DecodeElement(&s, &start); err != nil { + return nil, err + } + return s, nil + case "integer": + var s string + if err := dec.DecodeElement(&s, &start); err != nil { + return nil, err + } + var i int64 + fmt.Sscan(strings.TrimSpace(s), &i) + return i, nil + case "true": + return true, dec.Skip() + case "false": + return false, dec.Skip() + case "array": + var arr plistArray + for { + tok, err := dec.Token() + if err != nil { + return nil, err + } + switch t := tok.(type) { + case xml.EndElement: + if t.Name == start.Name { + return arr, nil + } + case xml.StartElement: + v, err := decodePlistValue(dec, t) + if err != nil { + return nil, err + } + arr = append(arr, v) + } + } + case "dict": + var d plistDict + if err := d.UnmarshalXML(dec, start); err != nil { + return nil, err + } + return d, nil + default: + return nil, dec.Skip() + } +} diff --git a/cmd/tailscale/cli/configure-flash-appliance_darwin_test.go b/cmd/tailscale/cli/configure-flash-appliance_darwin_test.go new file mode 100644 index 000000000..c82754dff --- /dev/null +++ b/cmd/tailscale/cli/configure-flash-appliance_darwin_test.go @@ -0,0 +1,98 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_flashappliance + +package cli + +import "testing" + +const diskutilListSample = ` + + + + AllDisks + + disk4 + disk4s1 + + WholeDisks + + disk4 + + +` + +const diskutilInfoSample = ` + + + + DeviceIdentifier + disk4 + DeviceModel + Generic STORAGE DEVICE + MediaName + Generic STORAGE DEVICE Media + Size + 62512365568 + Removable + + +` + +const diskutilInfoRootSample = ` + + + + DeviceIdentifier + disk3s1s1 + ParentWholeDisk + disk3 + APFSPhysicalStores + + + APFSPhysicalStore + disk0s2 + + + +` + +func TestParseDiskutilListPlist(t *testing.T) { + ids, err := parseDiskutilListPlist([]byte(diskutilListSample)) + if err != nil { + t.Fatalf("parseDiskutilListPlist: %v", err) + } + if len(ids) != 1 || ids[0] != "disk4" { + t.Errorf("ids = %v; want [disk4]", ids) + } +} + +func TestParseDiskutilInfoPlist(t *testing.T) { + info, err := parseDiskutilInfoPlist([]byte(diskutilInfoSample)) + if err != nil { + t.Fatalf("parseDiskutilInfoPlist: %v", err) + } + if info.Model != "Generic STORAGE DEVICE" { + t.Errorf("Model = %q; want %q", info.Model, "Generic STORAGE DEVICE") + } + if info.MediaName != "Generic STORAGE DEVICE Media" { + t.Errorf("MediaName = %q", info.MediaName) + } + if info.Size != 62512365568 { + t.Errorf("Size = %d; want 62512365568", info.Size) + } +} + +func TestParseDiskutilInfoPlistRoot(t *testing.T) { + info, err := parseDiskutilInfoPlist([]byte(diskutilInfoRootSample)) + if err != nil { + t.Fatalf("parseDiskutilInfoPlist: %v", err) + } + if info.ParentWholeDisk != "disk3" { + t.Errorf("ParentWholeDisk = %q; want disk3", info.ParentWholeDisk) + } + if len(info.APFSPhysicalStores) != 1 || info.APFSPhysicalStores[0] != "disk0s2" { + t.Errorf("APFSPhysicalStores = %v; want [disk0s2]", info.APFSPhysicalStores) + } +} diff --git a/cmd/tailscale/cli/configure-flash-appliance_linux.go b/cmd/tailscale/cli/configure-flash-appliance_linux.go new file mode 100644 index 000000000..987447ea6 --- /dev/null +++ b/cmd/tailscale/cli/configure-flash-appliance_linux.go @@ -0,0 +1,150 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_flashappliance + +package cli + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + + "golang.org/x/sys/unix" +) + +// discoverExternalDisks returns no disks on Linux: the user must pass +// --disk=/dev/sdX. We don't try to enumerate removable disks here because +// the right answer depends heavily on the host (servers don't have +// removable media; Pi-on-Pi flashing has no notion of "external"; LVM +// setups have arbitrary names). +func discoverExternalDisks(_ context.Context) ([]diskCandidate, error) { + return nil, errors.New("on Linux, pass --disk=/dev/sdX (auto-discovery is macOS-only)") +} + +// validateDiskPath rejects partition paths, the running root disk, and +// disks with any partition currently mounted. +func validateDiskPath(path string) error { + if !strings.HasPrefix(path, "/dev/") { + return fmt.Errorf("disk path %q must start with /dev/", path) + } + if isPartitionPath(path) { + return fmt.Errorf("disk path %q looks like a partition; pass the whole disk", path) + } + fi, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat %s: %w", path, err) + } + if fi.Mode()&os.ModeDevice == 0 { + return fmt.Errorf("%s is not a device file", path) + } + mounts, err := mountedSources() + if err != nil { + return err + } + for _, m := range mounts { + if m == path || strings.HasPrefix(m, path) { + return fmt.Errorf("%s (or one of its partitions) is currently mounted; unmount it first", path) + } + } + return nil +} + +// isPartitionPath reports whether path looks like a partition (e.g. +// /dev/sda1, /dev/nvme0n1p2, /dev/mmcblk0p1) rather than a whole disk. +func isPartitionPath(path string) bool { + base := strings.TrimPrefix(path, "/dev/") + switch { + case strings.HasPrefix(base, "sd"), strings.HasPrefix(base, "hd"), strings.HasPrefix(base, "vd"): + // /dev/sdaN — partition. + if len(base) >= 4 && base[len(base)-1] >= '0' && base[len(base)-1] <= '9' { + return true + } + case strings.HasPrefix(base, "nvme"), strings.HasPrefix(base, "mmcblk"), strings.HasPrefix(base, "loop"): + // /dev/nvme0n1p1 — partition is "p". The 'p' + // must follow a digit (to distinguish loop0 from loop0p1). + i := strings.LastIndexByte(base, 'p') + if i <= 0 || i >= len(base)-1 || base[i-1] < '0' || base[i-1] > '9' { + return false + } + for _, r := range base[i+1:] { + if r < '0' || r > '9' { + return false + } + } + return true + } + return false +} + +// mountedSources returns the source device paths from /proc/mounts. +func mountedSources() ([]string, error) { + f, err := os.Open("/proc/mounts") + if err != nil { + return nil, err + } + defer f.Close() + var out []string + sc := bufio.NewScanner(f) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) > 0 { + out = append(out, fields[0]) + } + } + return out, sc.Err() +} + +// unmountDisk unmounts every entry in /proc/mounts whose source starts with +// path (covers /dev/sdb plus /dev/sdb1, /dev/sdb2, ...). +func unmountDisk(ctx context.Context, path string) error { + mounts, err := mountedSources() + if err != nil { + return err + } + for _, m := range mounts { + if m == path || strings.HasPrefix(m, path) { + cmd := exec.CommandContext(ctx, "umount", m) + cmd.Stdout = Stderr + cmd.Stderr = Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("umount %s: %w", m, err) + } + } + } + return nil +} + +func openBlockDevice(path string) (*os.File, error) { + return os.OpenFile(path, os.O_WRONLY|unix.O_SYNC, 0) +} + +// rereadPartitionTable asks the kernel to re-scan the partition table on +// the open block device. Required on Linux before we can mkfs the perm +// partition we just wrote. +func rereadPartitionTable(f *os.File) error { + return unix.IoctlSetInt(int(f.Fd()), unix.BLKRRPART, 0) +} + +// syncBlockDevice flushes pending writes to disk. +func syncBlockDevice(f *os.File) error { return f.Sync() } + +// ejectDisk is a no-op on Linux; the user just pulls the disk after +// the sync at the end of writeGAFToDisk. Returns false so the success +// message instructs the user to eject themselves. +func ejectDisk(_ context.Context, _ string) (bool, error) { return false, nil } + +// blockDeviceSize returns the size in bytes of the open block device f. +// BLKGETSIZE64 returns a uint64; on 64-bit linux IoctlGetInt's int is wide +// enough to receive it without needing an unsafe.Pointer. +func blockDeviceSize(f *os.File) (int64, error) { + size, err := unix.IoctlGetInt(int(f.Fd()), unix.BLKGETSIZE64) + if err != nil { + return 0, fmt.Errorf("BLKGETSIZE64: %w", err) + } + return int64(size), nil +} diff --git a/cmd/tailscale/cli/configure-flash-appliance_linux_test.go b/cmd/tailscale/cli/configure-flash-appliance_linux_test.go new file mode 100644 index 000000000..7da5c064b --- /dev/null +++ b/cmd/tailscale/cli/configure-flash-appliance_linux_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_flashappliance + +package cli + +import "testing" + +func TestIsPartitionPath(t *testing.T) { + tests := []struct { + path string + want bool + }{ + {"/dev/sda", false}, + {"/dev/sda1", true}, + {"/dev/sdb", false}, + {"/dev/sdb4", true}, + {"/dev/sdz9", true}, + {"/dev/vdb", false}, + {"/dev/vdb1", true}, + {"/dev/nvme0n1", false}, + {"/dev/nvme0n1p1", true}, + {"/dev/nvme0n1p4", true}, + {"/dev/mmcblk0", false}, + {"/dev/mmcblk0p1", true}, + {"/dev/loop0", false}, + {"/dev/loop0p1", true}, + } + for _, tt := range tests { + if got := isPartitionPath(tt.path); got != tt.want { + t.Errorf("isPartitionPath(%q) = %v; want %v", tt.path, got, tt.want) + } + } +} diff --git a/cmd/tailscale/cli/configure-flash-appliance_omit.go b/cmd/tailscale/cli/configure-flash-appliance_omit.go new file mode 100644 index 000000000..903fe2ac4 --- /dev/null +++ b/cmd/tailscale/cli/configure-flash-appliance_omit.go @@ -0,0 +1,13 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ts_omit_flashappliance + +package cli + +import "github.com/peterbourgon/ff/v3/ffcli" + +func flashApplianceCmd() *ffcli.Command { + // Omitted from the build when the ts_omit_flashappliance build tag is set. + return nil +} diff --git a/cmd/tailscale/cli/configure-flash-appliance_other.go b/cmd/tailscale/cli/configure-flash-appliance_other.go new file mode 100644 index 000000000..2253f69e7 --- /dev/null +++ b/cmd/tailscale/cli/configure-flash-appliance_other.go @@ -0,0 +1,39 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_flashappliance && !linux && !darwin + +package cli + +import ( + "context" + "errors" + "os" + "runtime" +) + +var errFlashUnsupported = errors.New("flash-appliance is only supported on linux and darwin (got " + runtime.GOOS + ")") + +func discoverExternalDisks(_ context.Context) ([]diskCandidate, error) { + return nil, errFlashUnsupported +} + +func validateDiskPath(_ string) error { + return errFlashUnsupported +} + +func unmountDisk(_ context.Context, _ string) error { + return errFlashUnsupported +} + +func openBlockDevice(_ string) (*os.File, error) { + return nil, errFlashUnsupported +} + +func rereadPartitionTable(_ *os.File) error { return nil } + +func blockDeviceSize(_ *os.File) (int64, error) { return 0, errFlashUnsupported } + +func syncBlockDevice(_ *os.File) error { return errFlashUnsupported } + +func ejectDisk(_ context.Context, _ string) (bool, error) { return false, errFlashUnsupported } diff --git a/cmd/tailscale/cli/configure-flash-appliance_test.go b/cmd/tailscale/cli/configure-flash-appliance_test.go new file mode 100644 index 000000000..9a1fd9a7b --- /dev/null +++ b/cmd/tailscale/cli/configure-flash-appliance_test.go @@ -0,0 +1,54 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_flashappliance + +package cli + +import ( + "archive/zip" + "bytes" + "testing" +) + +func TestCheckPartitionFits(t *testing.T) { + files := buildZip(t, map[string][]byte{ + "boot.img": bytes.Repeat([]byte{0xAB}, 1<<20), + "root.img": bytes.Repeat([]byte{0xCD}, 4<<20), + }) + + if err := checkPartitionFits(files, "boot.img", 2<<20); err != nil { + t.Errorf("boot.img within limit: %v", err) + } + if err := checkPartitionFits(files, "root.img", 1<<20); err == nil { + t.Errorf("root.img over limit: expected error") + } + if err := checkPartitionFits(files, "missing.img", 1<<20); err == nil { + t.Errorf("missing file: expected error") + } +} + +// buildZip returns the *zip.File entries for an in-memory zip containing +// the given members. +func buildZip(t *testing.T, members map[string][]byte) []*zip.File { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, data := range members { + w, err := zw.Create(name) + if err != nil { + t.Fatalf("zip.Create %s: %v", name, err) + } + if _, err := w.Write(data); err != nil { + t.Fatalf("zip.Write %s: %v", name, err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("zip.Close: %v", err) + } + zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + if err != nil { + t.Fatalf("zip.NewReader: %v", err) + } + return zr.File +} diff --git a/cmd/tailscale/cli/configure-kube.go b/cmd/tailscale/cli/configure-kube.go index 3dcec250f..e2ef6becf 100644 --- a/cmd/tailscale/cli/configure-kube.go +++ b/cmd/tailscale/cli/configure-kube.go @@ -20,10 +20,8 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "k8s.io/client-go/util/homedir" "sigs.k8s.io/yaml" - "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" - "tailscale.com/types/netmap" "tailscale.com/util/dnsname" "tailscale.com/version" ) @@ -46,7 +44,7 @@ See: https://tailscale.com/s/k8s-auth-proxy `), FlagSet: (func() *flag.FlagSet { fs := newFlagSet("kubeconfig") - fs.BoolVar(&configureKubeconfigArgs.http, "http", false, "Use HTTP instead of HTTPS to connect to the auth proxy. Ignored if you include a scheme in the hostname argument.") + fs.BoolVar(&configureKubeconfigArgs.http, "http", false, "use HTTP instead of HTTPS to connect to the auth proxy. Ignored if you include a scheme in the hostname argument.") return fs })(), Exec: runConfigureKubeconfig, @@ -54,18 +52,15 @@ See: https://tailscale.com/s/k8s-auth-proxy } // kubeconfigPath returns the path to the kubeconfig file for the current user. -func kubeconfigPath() (string, error) { +func kubeconfigPath() string { if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { - if version.IsSandboxedMacOS() { - return "", errors.New("cannot read $KUBECONFIG on GUI builds of the macOS client: this requires the open-source tailscaled distribution") - } var out string for _, out = range filepath.SplitList(kubeconfig) { if info, err := os.Stat(out); !os.IsNotExist(err) && !info.IsDir() { break } } - return out, nil + return out } var dir string @@ -79,7 +74,64 @@ func kubeconfigPath() (string, error) { } else { dir = homedir.HomeDir() } - return filepath.Join(dir, ".kube", "config"), nil + return filepath.Join(dir, ".kube", "config") +} + +// checkKubeconfigWritable returns nil if the kubeconfig at path can be written, +// or an error explaining why it can't. A not-yet-created file or .kube +// directory is fine as long as the nearest existing ancestor is writable. +// +// On sandboxed macOS builds, kubeconfigPath resolves path to the user's real +// ~/.kube/config, which we can only write via the home-relative-path +// entitlement. If that write would fail (e.g. because $KUBECONFIG points +// somewhere the sandbox can't reach), we want to surface a clear error pointing +// at the open-source tailscaled distribution rather than silently writing a +// config the user's kubectl will never read into the sandbox container. +func checkKubeconfigWritable(path string) error { + for try := path; ; try = filepath.Dir(try) { + if _, err := os.Stat(try); err == nil { + if err := isWritable(try); err != nil { + return kubeconfigAccessErr(path, err) + } + return nil + } else if !os.IsNotExist(err) { + return kubeconfigAccessErr(path, err) + } + if parent := filepath.Dir(try); parent == try { + return nil // reached the filesystem root + } + } +} + +// isWritable reports whether path can be opened or created for writing. For a +// directory it probes by creating and removing a temporary file. +func isWritable(path string) error { + fi, err := os.Stat(path) + if err != nil { + return err + } + if fi.IsDir() { + f, err := os.CreateTemp(path, ".tailscale-kubeconfig-*") + if err != nil { + return err + } + f.Close() + return os.Remove(f.Name()) + } + f, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return err + } + return f.Close() +} + +// kubeconfigAccessErr wraps err with context about path, adding macOS sandbox +// guidance when the process is sandboxed. +func kubeconfigAccessErr(path string, err error) error { + if version.IsSandboxedMacOS() { + return fmt.Errorf("cannot write kubeconfig at %q: %w; GUI builds of the macOS client run in a sandbox and can only access files under your home directory, use the open-source tailscaled distribution for other locations", path, err) + } + return fmt.Errorf("cannot write kubeconfig at %q: %w", path, err) } func runConfigureKubeconfig(ctx context.Context, args []string) error { @@ -98,18 +150,18 @@ func runConfigureKubeconfig(ctx context.Context, args []string) error { if st.BackendState != "Running" { return errors.New("Tailscale is not running") } - nm, err := getNetMap(ctx) + dnsCfg, err := getDNSConfig(ctx) if err != nil { return err } - targetFQDN, err := nodeOrServiceDNSNameFromArg(st, nm, hostOrFQDNOrIP) + targetFQDN, err := nodeOrServiceDNSNameFromArg(st, dnsCfg, hostOrFQDNOrIP) if err != nil { return err } targetFQDN = strings.TrimSuffix(targetFQDN, ".") - var kubeconfig string - if kubeconfig, err = kubeconfigPath(); err != nil { + kubeconfig := kubeconfigPath() + if err := checkKubeconfigWritable(kubeconfig); err != nil { return err } scheme := "https://" @@ -217,13 +269,8 @@ func setKubeconfigForPeer(scheme, fqdn, filePath string) error { if !os.IsNotExist(err) { return err } - if err := os.Mkdir(dir, 0755); err != nil { - if version.IsSandboxedMacOS() && errors.Is(err, os.ErrPermission) { - // macOS sandboxing prevents us from creating the .kube directory - // in the home directory. - return errors.New("unable to create .kube directory in home directory, please create it manually (e.g. mkdir ~/.kube") - } - return err + if err := os.MkdirAll(dir, 0755); err != nil { + return kubeconfigAccessErr(filePath, err) } } b, err := os.ReadFile(filePath) @@ -240,14 +287,14 @@ func setKubeconfigForPeer(scheme, fqdn, filePath string) error { // nodeOrServiceDNSNameFromArg returns the PeerStatus.DNSName value from a peer // in st that matches the input arg which can be a base name, full DNS name, or // an IP. If none is found, it looks for a Tailscale Service -func nodeOrServiceDNSNameFromArg(st *ipnstate.Status, nm *netmap.NetworkMap, arg string) (string, error) { +func nodeOrServiceDNSNameFromArg(st *ipnstate.Status, dns *tailcfg.DNSConfig, arg string) (string, error) { // First check for a node DNS name. if dnsName, ok := nodeDNSNameFromArg(st, arg); ok { return dnsName, nil } // If not found, check for a Tailscale Service DNS name. - rec, ok := serviceDNSRecordFromNetMap(nm, arg) + rec, ok := serviceDNSRecordFromDNSConfig(dns, arg) if !ok { return "", fmt.Errorf("no peer found for %q", arg) } @@ -259,6 +306,11 @@ func nodeOrServiceDNSNameFromArg(st *ipnstate.Status, nm *netmap.NetworkMap, arg } ipPrefix := netip.PrefixFrom(ip, ip.BitLen()) for _, ps := range st.Peer { + if ps.AllowedIPs == nil { + // Peer with no addresses visible in the tailnet, e.g. a ProxyGroup + // whose backing nodes are offline or not yet approved (#20255). + continue + } for _, allowedIP := range ps.AllowedIPs.All() { if allowedIP == ipPrefix { return rec.Name, nil @@ -269,25 +321,13 @@ func nodeOrServiceDNSNameFromArg(st *ipnstate.Status, nm *netmap.NetworkMap, arg return "", fmt.Errorf("%q is in MagicDNS, but is not currently reachable on any known peer", arg) } -func getNetMap(ctx context.Context) (*netmap.NetworkMap, error) { +func getDNSConfig(ctx context.Context) (*tailcfg.DNSConfig, error) { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - - watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) - if err != nil { - return nil, err - } - defer watcher.Close() - - n, err := watcher.Next() - if err != nil { - return nil, err - } - - return n.NetMap, nil + return localClient.DNSConfig(ctx) } -func serviceDNSRecordFromNetMap(nm *netmap.NetworkMap, arg string) (rec tailcfg.DNSRecord, ok bool) { +func serviceDNSRecordFromDNSConfig(dns *tailcfg.DNSConfig, arg string) (rec tailcfg.DNSRecord, ok bool) { argIP, _ := netip.ParseAddr(arg) argFQDN, err := dnsname.ToFQDN(arg) argFQDNValid := err == nil @@ -295,7 +335,7 @@ func serviceDNSRecordFromNetMap(nm *netmap.NetworkMap, arg string) (rec tailcfg. return rec, false } - for _, rec := range nm.DNS.ExtraRecords { + for _, rec := range dns.ExtraRecords { if argIP.IsValid() { recIP, _ := netip.ParseAddr(rec.Value) if recIP == argIP { diff --git a/cmd/tailscale/cli/configure-kube_test.go b/cmd/tailscale/cli/configure-kube_test.go index 2c2a05ac0..45ab7f542 100644 --- a/cmd/tailscale/cli/configure-kube_test.go +++ b/cmd/tailscale/cli/configure-kube_test.go @@ -7,10 +7,18 @@ package cli import ( "bytes" "fmt" + "net/netip" + "os" + "path/filepath" + "runtime" "strings" "testing" "github.com/google/go-cmp/cmp" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/key" + "tailscale.com/types/views" ) func TestKubeconfig(t *testing.T) { @@ -247,6 +255,69 @@ users: } } +func TestCheckKubeconfigWritable(t *testing.T) { + t.Run("nonexistent-file-in-writable-dir", func(t *testing.T) { + dir := t.TempDir() + if err := checkKubeconfigWritable(filepath.Join(dir, "config")); err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + + t.Run("nonexistent-file-and-dir-in-writable-parent", func(t *testing.T) { + dir := t.TempDir() + // The .kube directory does not exist yet, but its parent does and is + // writable, so this should be fine. + if err := checkKubeconfigWritable(filepath.Join(dir, ".kube", "config")); err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + + t.Run("existing-writable-file", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config") + if err := os.WriteFile(path, []byte("apiVersion: v1\nkind: Config\n"), 0600); err != nil { + t.Fatal(err) + } + if err := checkKubeconfigWritable(path); err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + + t.Run("unwritable-existing-file", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("file mode permissions are not enforced the same way on Windows") + } + if os.Getuid() == 0 { + t.Skip("root bypasses file permission checks") + } + dir := t.TempDir() + path := filepath.Join(dir, "config") + if err := os.WriteFile(path, []byte("x"), 0400); err != nil { + t.Fatal(err) + } + if err := checkKubeconfigWritable(path); err == nil { + t.Error("expected error for read-only file, got nil") + } + }) + + t.Run("unwritable-dir", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory mode permissions are not enforced the same way on Windows") + } + if os.Getuid() == 0 { + t.Skip("root bypasses directory permission checks") + } + dir := t.TempDir() + sub := filepath.Join(dir, "ro") + if err := os.Mkdir(sub, 0500); err != nil { + t.Fatal(err) + } + if err := checkKubeconfigWritable(filepath.Join(sub, "config")); err == nil { + t.Error("expected error for unwritable dir, got nil") + } + }) +} + func TestGetInputs(t *testing.T) { for _, arg := range []string{ "foo.tail-scale.ts.net", @@ -273,3 +344,68 @@ func TestGetInputs(t *testing.T) { } } } + +func TestNodeOrServiceDNSNameFromArg(t *testing.T) { + svcIP := netip.MustParseAddr("100.100.100.100") + dnsCfg := &tailcfg.DNSConfig{ + ExtraRecords: []tailcfg.DNSRecord{ + {Name: "svc.example.ts.net", Value: svcIP.String()}, + }, + } + + peerWithService := &ipnstate.PeerStatus{DNSName: "node-a.example.ts.net."} + allowed := views.SliceOf([]netip.Prefix{netip.PrefixFrom(svcIP, svcIP.BitLen())}) + peerWithService.AllowedIPs = &allowed + + // A peer with no AllowedIPs, as reported for a ProxyGroup whose backing + // nodes are offline or not yet approved (issue #20255). + peerNoAddrs := &ipnstate.PeerStatus{DNSName: "node-b.example.ts.net."} + + tests := []struct { + name string + peers []*ipnstate.PeerStatus + arg string + want string + wantErr string + }{ + { + name: "service_with_no_reachable_peer", + peers: []*ipnstate.PeerStatus{peerNoAddrs}, + arg: "svc", + wantErr: "not currently reachable", + }, + { + name: "service_advertised_by_peer", + peers: []*ipnstate.PeerStatus{peerNoAddrs, peerWithService}, + arg: "svc", + want: "svc.example.ts.net", + }, + { + name: "node_dns_name", + peers: []*ipnstate.PeerStatus{peerNoAddrs}, + arg: "node-b", + want: "node-b.example.ts.net.", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := &ipnstate.Status{Peer: map[key.NodePublic]*ipnstate.PeerStatus{}} + for _, ps := range tt.peers { + st.Peer[key.NewNode().Public()] = ps + } + got, err := nodeOrServiceDNSNameFromArg(st, dnsCfg, tt.arg) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v, want error containing %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} diff --git a/cmd/tailscale/cli/configure-pve-appliance.go b/cmd/tailscale/cli/configure-pve-appliance.go new file mode 100644 index 000000000..17656ebb1 --- /dev/null +++ b/cmd/tailscale/cli/configure-pve-appliance.go @@ -0,0 +1,413 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_flashappliance + +package cli + +import ( + "archive/zip" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/clientupdate" + "tailscale.com/gokrazy/mkfs" + "tailscale.com/util/prompt" +) + +var pveApplianceArgs struct { + vmid int + name string + storage string + diskSize string + cores int + memory int + bridge string + variant string + track string + gaf string + addSSHAuthorizedKeys string + start bool + yes bool +} + +func pveApplianceCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "pve-appliance", + ShortUsage: "tailscale configure pve-appliance --storage= [flags]", + ShortHelp: "Create a Proxmox VE VM running the Tailscale appliance image [experimental]", + LongHelp: hidden + strings.TrimSpace(` +This experimental command downloads a signed Tailscale appliance GAF from +pkgs.tailscale.com, builds a raw disk image in /var/tmp, then invokes +'qm create' / 'qm disk import' / 'qm set' on the local Proxmox VE host to +create a new VM backed by that image. It must be run on the PVE host +itself (where the 'qm' CLI is available). + +The imported disk is attached as scsi0 on a virtio-scsi-single controller +with iothread, the network attaches to the given bridge (default vmbr0), +and the guest agent is enabled — pair with the appliance's built-in +qemu-guest-kragent so PVE can see the guest's IPs. + +The VM is created with a virtio-serial console (--serial0 socket). Once +the VM is running, 'qm terminal ' on the PVE host — then press +Enter — drops you into a busybox shell inside the appliance without +needing an SSH key. + +Defaults are chosen so a bare invocation like: + + tailscale configure pve-appliance --storage=local-lvm + +is enough to produce a bootable Tailscale appliance VM. +`), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("pve-appliance") + fs.IntVar(&pveApplianceArgs.vmid, "vmid", 0, "target VM ID; 0 asks Proxmox for the next available ID") + fs.StringVar(&pveApplianceArgs.name, "name", "", `VM name; defaults to "tsapp-"`) + fs.StringVar(&pveApplianceArgs.storage, "storage", "", "PVE storage to import the disk into (e.g. local-lvm, ssd2); required") + fs.StringVar(&pveApplianceArgs.diskSize, "disk-size", "4G", "raw image size (accepts K/M/G suffixes, e.g. 4G, 8192M)") + fs.IntVar(&pveApplianceArgs.cores, "cores", 2, "vCPU cores") + fs.IntVar(&pveApplianceArgs.memory, "memory", 1024, "memory in MiB") + fs.StringVar(&pveApplianceArgs.bridge, "bridge", "vmbr0", "network bridge to attach virtio net0 to") + fs.StringVar(&pveApplianceArgs.variant, "variant", "vm-amd64", `appliance variant: "vm-amd64" or "vm-arm64"`) + fs.StringVar(&pveApplianceArgs.track, "track", "", `which track to download from; defaults to "`+clientupdate.CurrentTrack+`"`) + fs.StringVar(&pveApplianceArgs.gaf, "gaf", "", "use a local GAF file instead of downloading (skips signature verification)") + fs.StringVar(&pveApplianceArgs.addSSHAuthorizedKeys, "add-ssh-authorized-keys", "", "path to an authorized_keys file to include on the appliance for breakglass SSH access") + fs.BoolVar(&pveApplianceArgs.start, "start", true, "start the VM after import") + fs.BoolVar(&pveApplianceArgs.yes, "yes", false, "skip the confirmation prompt") + return fs + })(), + Exec: runPVEAppliance, + } +} + +func runPVEAppliance(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unknown arguments") + } + if runtime.GOOS != "linux" { + return errors.New("the pve-appliance subcommand is only available on Linux; for use on Proxmox PVE hosts") + } + if fi, err := os.Stat("/etc/pve"); err != nil || !fi.IsDir() { + return errors.New("/etc/pve is not a directory: run this on a Proxmox VE host") + } + if _, err := exec.LookPath("qm"); err != nil { + return errors.New("`qm` not found in $PATH: run this on the Proxmox VE host") + } + if pveApplianceArgs.storage == "" { + return errors.New("--storage is required (e.g. --storage=local-lvm)") + } + + diskBytes, err := parseSizeBytes(pveApplianceArgs.diskSize) + if err != nil { + return fmt.Errorf("parsing --disk-size: %w", err) + } + + vmid := pveApplianceArgs.vmid + if vmid == 0 { + vmid, err = pveNextID(ctx) + if err != nil { + return fmt.Errorf("fetching next VMID: %w", err) + } + } + name := pveApplianceArgs.name + if name == "" { + name = fmt.Sprintf("tsapp-%d", vmid) + } + + gafPath, gafLabel, variant, cleanup, err := obtainGAF(ctx, gafDownloadArgs{ + localGAF: pveApplianceArgs.gaf, + track: pveApplianceArgs.track, + variant: pveApplianceArgs.variant, + }) + if err != nil { + return err + } + defer cleanup() + + if !pveApplianceArgs.yes { + printf("About to create Proxmox VM %d (%q) on storage %q from %s\n", + vmid, name, pveApplianceArgs.storage, gafLabel) + printf(" cores=%d memory=%dMiB bridge=%s disk=%s start=%v\n", + pveApplianceArgs.cores, pveApplianceArgs.memory, + pveApplianceArgs.bridge, pveApplianceArgs.diskSize, pveApplianceArgs.start) + if !prompt.YesNo("Proceed?", false) { + return errors.New("aborted") + } + } + + imgPath, err := buildPVERawImage(gafPath, diskBytes, variant) + if err != nil { + return err + } + defer os.Remove(imgPath) + + if err := createPVEVM(ctx, vmid, name); err != nil { + return fmt.Errorf("qm create: %w", err) + } + diskRef, err := importPVEDisk(ctx, vmid, pveApplianceArgs.storage, imgPath) + if err != nil { + return fmt.Errorf("qm disk import: %w", err) + } + if err := attachPVEDisk(ctx, vmid, diskRef); err != nil { + return fmt.Errorf("qm set: %w", err) + } + + if pveApplianceArgs.start { + if err := runQM(ctx, "start", strconv.Itoa(vmid)); err != nil { + return fmt.Errorf("qm start: %w", err) + } + printf("VM %d started.\n", vmid) + } else { + printf("VM %d created; not started (pass --start to auto-start).\n", vmid) + } + return nil +} + +// buildPVERawImage creates a sparse raw disk image in /var/tmp and +// writes the GAF's boot + root images to it plus a fresh /perm ext4 +// filesystem. The returned path is the caller's to remove. +func buildPVERawImage(gafPath string, devsize int64, variant string) (string, error) { + zr, err := zip.OpenReader(gafPath) + if err != nil { + return "", fmt.Errorf("open GAF: %w", err) + } + defer zr.Close() + + bootCode, err := readGAFMember(zr.File, "mbr.img", 1<<20) + if err != nil { + return "", err + } + + tmp, err := os.CreateTemp("/var/tmp", "tsapp-pve-*.raw") + if err != nil { + return "", err + } + imgPath := tmp.Name() + if err := tmp.Truncate(devsize); err != nil { + tmp.Close() + os.Remove(imgPath) + return "", fmt.Errorf("truncate: %w", err) + } + + if err := writeApplianceImage(tmp, devsize, zr.File, bootCode, variant); err != nil { + tmp.Close() + os.Remove(imgPath) + return "", err + } + + var permFiles []mkfs.PermFile + if k := pveApplianceArgs.addSSHAuthorizedKeys; k != "" { + keys, err := os.ReadFile(k) + if err != nil { + tmp.Close() + os.Remove(imgPath) + return "", fmt.Errorf("reading --add-ssh-authorized-keys: %w", err) + } + permFiles = append(permFiles, mkfs.PermFile{ + Path: "breakglass.authorized_keys", + Content: keys, + }) + printf("Including SSH authorized_keys for breakglass access.\n") + } + if err := mkfs.Perm(tmp, devsize, permFiles...); err != nil { + tmp.Close() + os.Remove(imgPath) + return "", fmt.Errorf("formatting perm: %w", err) + } + + if err := tmp.Sync(); err != nil { + tmp.Close() + os.Remove(imgPath) + return "", err + } + if err := tmp.Close(); err != nil { + os.Remove(imgPath) + return "", err + } + return imgPath, nil +} + +// pveNextID asks Proxmox for the next unused VMID via pvesh. +func pveNextID(ctx context.Context) (int, error) { + out, err := exec.CommandContext(ctx, "pvesh", "get", "/cluster/nextid").Output() + if err != nil { + return 0, err + } + s := strings.TrimSpace(string(out)) + n, err := strconv.Atoi(s) + if err != nil { + return 0, fmt.Errorf("parsing pvesh output %q: %w", s, err) + } + return n, nil +} + +func createPVEVM(ctx context.Context, vmid int, name string) error { + return runQM(ctx, "create", strconv.Itoa(vmid), + "--name", name, + "--memory", strconv.Itoa(pveApplianceArgs.memory), + "--cores", strconv.Itoa(pveApplianceArgs.cores), + "--net0", "virtio,bridge="+pveApplianceArgs.bridge, + "--scsihw", "virtio-scsi-single", + "--serial0", "socket", + "--agent", "1", + "--ostype", "l26", + "--tablet", "0", + "--vga", "virtio", + "--description", vmNotes(vmid), + ) +} + +// vmNotes is the description shown in the PVE web UI's "Notes" panel +// for VMs we create. It tells admins the two ways to get into the +// appliance without needing an SSH key: the framebuffer console (press +// Esc for a shell) and the serial console (qm terminal). +func vmNotes(vmid int) string { + return fmt.Sprintf(`# Tailscale appliance [experimental] + +Admin access to this VM (no SSH key required): + +- **Framebuffer / NoVNC console**: press **Esc** on the enrollment screen + to drop into a busybox shell. Type `+"`exit`"+` to return to the + Tailscale status display. + +- **Serial console**: on this Proxmox host, run + + qm terminal %d + + then press **Enter** to get a busybox shell. Press **Ctrl+O** to + detach from `+"`qm terminal`"+` (leaves the guest shell alive). + +Inside either shell, run `+"`tailscale`"+` commands as usual +(`+"`tailscale up`"+`, `+"`tailscale status`"+`, etc.). +`, vmid) +} + +// importPVEDisk imports src into storage on vmid and returns the +// Proxmox volume reference of the newly-imported disk (e.g. +// "local-lvm:vm-102-disk-0"). +// +// The stdout of "qm disk import" isn't a stable API across Proxmox +// versions — historically it has printed variants like +// "vm--disk-" and "importing disk … as ", and the +// allocated volume name isn't always vm--disk-0 (e.g. if a +// stale volume with that name exists on the storage from a prior +// VM). We instead diff the VM config before and after the import +// and pick out the newly-added unusedN entry, which holds the real +// volume ref. +func importPVEDisk(ctx context.Context, vmid int, storage, src string) (volume string, err error) { + before, err := pveVMConfig(ctx, vmid) + if err != nil { + return "", fmt.Errorf("reading VM %d config: %w", vmid, err) + } + if err := runQM(ctx, "disk", "import", strconv.Itoa(vmid), src, storage, "--format", "raw"); err != nil { + return "", err + } + after, err := pveVMConfig(ctx, vmid) + if err != nil { + return "", fmt.Errorf("reading VM %d config after import: %w", vmid, err) + } + for k, v := range after { + if !strings.HasPrefix(k, "unused") { + continue + } + if before[k] == v { + continue + } + return v, nil + } + return "", fmt.Errorf("qm disk import didn't add an unusedN entry to VM %d config", vmid) +} + +// pveVMConfig returns the current runtime config for vmid on the local +// PVE node, as a flat key → string map. We ask pvesh for JSON since +// "qm config" text output has evolved over releases. +func pveVMConfig(ctx context.Context, vmid int) (map[string]string, error) { + node, err := os.Hostname() + if err != nil { + return nil, err + } + out, err := exec.CommandContext(ctx, "pvesh", "get", + fmt.Sprintf("/nodes/%s/qemu/%d/config", node, vmid), + "--output-format", "json").Output() + if err != nil { + return nil, err + } + var raw map[string]any + if err := json.Unmarshal(out, &raw); err != nil { + return nil, fmt.Errorf("parsing pvesh JSON: %w", err) + } + m := make(map[string]string, len(raw)) + for k, v := range raw { + switch v := v.(type) { + case string: + m[k] = v + case float64: + m[k] = strconv.FormatFloat(v, 'f', -1, 64) + case bool: + m[k] = strconv.FormatBool(v) + } + } + return m, nil +} + +func attachPVEDisk(ctx context.Context, vmid int, diskRef string) error { + return runQM(ctx, "set", strconv.Itoa(vmid), + "--scsi0", diskRef+",iothread=1", + "--boot", "order=scsi0", + ) +} + +// runQM invokes `qm` with args, streaming its output to Stderr so the +// caller can see disk-import progress and any error messages. +func runQM(ctx context.Context, args ...string) error { + printf("$ qm %s\n", strings.Join(args, " ")) + cmd := exec.CommandContext(ctx, "qm", args...) + cmd.Stdout = Stderr + cmd.Stderr = Stderr + return cmd.Run() +} + +// parseSizeBytes parses a size string like "4G", "8192M", "1024K", or +// "12345" (bytes) into a byte count. Empty string returns an error. +func parseSizeBytes(s string) (int64, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, errors.New("empty size") + } + mult := int64(1) + switch last := s[len(s)-1]; { + case last >= '0' && last <= '9': + // no suffix + default: + switch last { + case 'K', 'k': + mult = 1 << 10 + case 'M', 'm': + mult = 1 << 20 + case 'G', 'g': + mult = 1 << 30 + case 'T', 't': + mult = 1 << 40 + default: + return 0, fmt.Errorf("unknown size suffix %q", string(last)) + } + s = s[:len(s)-1] + } + n, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, err + } + if n <= 0 { + return 0, fmt.Errorf("size %d must be positive", n) + } + return n * mult, nil +} diff --git a/cmd/tailscale/cli/configure-pve-appliance_omit.go b/cmd/tailscale/cli/configure-pve-appliance_omit.go new file mode 100644 index 000000000..81af1f31d --- /dev/null +++ b/cmd/tailscale/cli/configure-pve-appliance_omit.go @@ -0,0 +1,13 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ts_omit_flashappliance + +package cli + +import "github.com/peterbourgon/ff/v3/ffcli" + +func pveApplianceCmd() *ffcli.Command { + // Omitted from the build when the ts_omit_flashappliance build tag is set. + return nil +} diff --git a/cmd/tailscale/cli/configure.go b/cmd/tailscale/cli/configure.go index e7a6448e7..863f0fb6b 100644 --- a/cmd/tailscale/cli/configure.go +++ b/cmd/tailscale/cli/configure.go @@ -32,6 +32,8 @@ services on the host to use Tailscale in more ways. Subcommands: nonNilCmds( configureKubeconfigCmd(), synologyConfigureCmd(), + flashApplianceCmd(), + pveApplianceCmd(), ccall(maybeConfigSynologyCertCmd), ccall(maybeSysExtCmd), ccall(maybeVPNConfigCmd), diff --git a/cmd/tailscale/cli/configure_linux.go b/cmd/tailscale/cli/configure_linux.go index 9ba3b8e87..e21af3c90 100644 --- a/cmd/tailscale/cli/configure_linux.go +++ b/cmd/tailscale/cli/configure_linux.go @@ -18,7 +18,7 @@ func init() { maybeSystrayCmd = systrayConfigCmd } -var systrayArgs struct { +var configSystrayArgs struct { initSystem string installStartup bool } @@ -32,16 +32,16 @@ func systrayConfigCmd() *ffcli.Command { Exec: configureSystray, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("systray") - fs.StringVar(&systrayArgs.initSystem, "enable-startup", "", - "Install startup script for init system. Currently supported systems are [systemd, freedesktop].") + fs.StringVar(&configSystrayArgs.initSystem, "enable-startup", "", + "install startup script for init system. Currently supported systems are [systemd, freedesktop].") return fs })(), } } func configureSystray(_ context.Context, _ []string) error { - if systrayArgs.initSystem != "" { - if err := systray.InstallStartupScript(systrayArgs.initSystem); err != nil { + if configSystrayArgs.initSystem != "" { + if err := systray.InstallStartupScript(configSystrayArgs.initSystem); err != nil { fmt.Printf("%s\n\n", err.Error()) return flag.ErrHelp } diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index 944f99f91..8e25c81f1 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -39,6 +39,7 @@ import ( "tailscale.com/net/ace" "tailscale.com/net/dnscache" "tailscale.com/net/netmon" + "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" "tailscale.com/net/tsdial" "tailscale.com/paths" @@ -257,7 +258,7 @@ func debugCmd() *ffcli.Command { ShortHelp: "Print prefs", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("prefs") - fs.BoolVar(&prefsArgs.pretty, "pretty", false, "If true, pretty-print output") + fs.BoolVar(&prefsArgs.pretty, "pretty", false, "if true, pretty-print output") return fs })(), }, @@ -268,10 +269,19 @@ func debugCmd() *ffcli.Command { ShortHelp: "Subscribe to IPN message bus", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("watch-ipn") - fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages") - fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include initial status") - fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messages") + fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include the initial backend State and Prefs in the first message") fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever") + fs.BoolVar(&watchIPNArgs.engineUpdates, "engine-updates", false, "set NotifyWatchEngineUpdates: send Engine updates") + fs.BoolVar(&watchIPNArgs.initialDriveShares, "initial-drive-shares", false, "set NotifyInitialDriveShares: send current Taildrive Shares in first message") + fs.BoolVar(&watchIPNArgs.initialOutgoingFiles, "initial-outgoing-files", false, "set NotifyInitialOutgoingFiles: send current Taildrop OutgoingFiles in first message") + fs.BoolVar(&watchIPNArgs.initialHealthState, "initial-health", false, "set NotifyInitialHealthState: send current health.State in first message") + fs.BoolVar(&watchIPNArgs.healthActions, "health-actions", false, "set NotifyHealthActions: include PrimaryActions in health.State") + fs.BoolVar(&watchIPNArgs.initialSuggestedExitNode, "initial-suggested-exit-node", false, "set NotifyInitialSuggestedExitNode: send current SuggestedExitNode in first message") + fs.BoolVar(&watchIPNArgs.initialClientVersion, "initial-client-version", false, "set NotifyInitialClientVersion: send current ClientVersion in first message") + fs.BoolVar(&watchIPNArgs.peerChanges, "peer-changes", true, "set NotifyPeerChanges: send PeersChanged and PeersRemoved updates") + fs.BoolVar(&watchIPNArgs.initialStatus, "initial-status", false, "set NotifyInitialStatus: send current ipnstate.Status in first message") + fs.BoolVar(&watchIPNArgs.peerPatches, "peer-patches", true, "set NotifyPeerPatches: send narrow per-field peer patches") + fs.BoolVar(&watchIPNArgs.peerWireGuardState, "peer-wireguard-state", false, "set NotifyPeerWireGuardState: send WireGuard session state notifications") return fs })(), }, @@ -632,19 +642,59 @@ func runPrefs(ctx context.Context, args []string) error { } var watchIPNArgs struct { - netmap bool - initial bool - rateLimit bool - count int + initial bool + count int + + engineUpdates bool + initialDriveShares bool + initialOutgoingFiles bool + initialHealthState bool + healthActions bool + initialSuggestedExitNode bool + initialClientVersion bool + peerChanges bool + initialStatus bool + peerPatches bool + peerWireGuardState bool } func runWatchIPN(ctx context.Context, args []string) error { - var mask ipn.NotifyWatchOpt + mask := ipn.NotifyNoNetMap if watchIPNArgs.initial { - mask = ipn.NotifyInitialState | ipn.NotifyInitialPrefs | ipn.NotifyInitialNetMap + mask |= ipn.NotifyInitialState | ipn.NotifyInitialPrefs } - if watchIPNArgs.rateLimit { - mask |= ipn.NotifyRateLimit + if watchIPNArgs.engineUpdates { + mask |= ipn.NotifyWatchEngineUpdates + } + if watchIPNArgs.initialDriveShares { + mask |= ipn.NotifyInitialDriveShares + } + if watchIPNArgs.initialOutgoingFiles { + mask |= ipn.NotifyInitialOutgoingFiles + } + if watchIPNArgs.initialHealthState { + mask |= ipn.NotifyInitialHealthState + } + if watchIPNArgs.healthActions { + mask |= ipn.NotifyHealthActions + } + if watchIPNArgs.initialSuggestedExitNode { + mask |= ipn.NotifyInitialSuggestedExitNode + } + if watchIPNArgs.initialClientVersion { + mask |= ipn.NotifyInitialClientVersion + } + if watchIPNArgs.peerChanges { + mask |= ipn.NotifyPeerChanges + } + if watchIPNArgs.initialStatus { + mask |= ipn.NotifyInitialStatus + } + if watchIPNArgs.peerPatches { + mask |= ipn.NotifyPeerPatches + } + if watchIPNArgs.peerWireGuardState { + mask |= ipn.NotifyPeerWireGuardState } watcher, err := localClient.WatchIPNBus(ctx, mask) if err != nil { @@ -657,9 +707,6 @@ func runWatchIPN(ctx context.Context, args []string) error { if err != nil { return err } - if !watchIPNArgs.netmap { - n.NetMap = nil - } j, _ := json.MarshalIndent(n, "", "\t") fmt.Printf("%s\n", j) } @@ -670,18 +717,11 @@ func runNetmap(ctx context.Context, args []string) error { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - var mask ipn.NotifyWatchOpt = ipn.NotifyInitialNetMap - watcher, err := localClient.WatchIPNBus(ctx, mask) + raw, err := localClient.DebugResultJSON(ctx, "current-netmap") if err != nil { return err } - defer watcher.Close() - - n, err := watcher.Next() - if err != nil { - return err - } - j, _ := json.MarshalIndent(n.NetMap, "", "\t") + j, _ := json.MarshalIndent(raw, "", "\t") fmt.Printf("%s\n", j) return nil } @@ -798,10 +838,13 @@ func runDaemonLogs(ctx context.Context, args []string) error { } d := json.NewDecoder(logs) for { + type logtail struct { + Time string `json:"client_time"` + } var line struct { - Text string `json:"text"` - Verbose int `json:"v"` - Time string `json:"client_time"` + Text string `json:"text"` + Verbose int `json:"v"` + Logtail logtail `json:"logtail"` } err := d.Decode(&line) if err != nil { @@ -811,8 +854,8 @@ func runDaemonLogs(ctx context.Context, args []string) error { if line.Text == "" || line.Verbose > daemonLogsArgs.verbose { continue } - if daemonLogsArgs.time { - fmt.Printf("%s %s\n", line.Time, line.Text) + if daemonLogsArgs.time && line.Logtail.Time != "" { + fmt.Printf("%s %s\n", line.Logtail.Time, line.Text) } else { fmt.Println(line.Text) } @@ -998,7 +1041,7 @@ func runTS2021(ctx context.Context, args []string) error { keysURL := "https://" + ts2021Args.host + "/key?v=" + strconv.Itoa(ts2021Args.version) - keyTransport := http.DefaultTransport.(*http.Transport).Clone() + keyTransport := netutil.NewDefaultTransport() if ts2021Args.aceHost != "" { log.Printf("using ACE server %q", ts2021Args.aceHost) keyTransport.Proxy = nil diff --git a/cmd/tailscale/cli/dns-query.go b/cmd/tailscale/cli/dns-query.go index 2993441b3..6336b159c 100644 --- a/cmd/tailscale/cli/dns-query.go +++ b/cmd/tailscale/cli/dns-query.go @@ -15,7 +15,8 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "golang.org/x/net/dns/dnsmessage" - "tailscale.com/cmd/tailscale/cli/jsonoutput" + + "tailscale.com/cmd/tailscale/tsdnsjsonv0" ) var dnsQueryArgs struct { @@ -73,7 +74,7 @@ func runDNSQuery(ctx context.Context, args []string) error { return fmt.Errorf("failed to query DNS: %w", err) } - data := &jsonoutput.DNSQueryResult{ + data := &tsdnsjsonv0.QueryResponse{ Name: name, QueryType: queryType, } @@ -96,9 +97,9 @@ func runDNSQuery(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("failed to parse DNS answers: %w", err) } - data.Answers = make([]jsonoutput.DNSAnswer, 0, len(answers)) + data.Answers = make([]tsdnsjsonv0.Answer, 0, len(answers)) for _, a := range answers { - data.Answers = append(data.Answers, jsonoutput.DNSAnswer{ + data.Answers = append(data.Answers, tsdnsjsonv0.Answer{ Name: a.Header.Name.String(), TTL: a.Header.TTL, Class: a.Header.Class.String(), @@ -120,7 +121,7 @@ func runDNSQuery(ctx context.Context, args []string) error { return nil } -func formatDNSQueryText(data *jsonoutput.DNSQueryResult) string { +func formatDNSQueryText(data *tsdnsjsonv0.QueryResponse) string { var sb strings.Builder fmt.Fprintf(&sb, "DNS query for %q (%s) using internal resolver:\n", data.Name, data.QueryType) @@ -158,8 +159,8 @@ func formatDNSQueryText(data *jsonoutput.DNSQueryResult) string { return sb.String() } -// formatResolverString formats a jsonoutput.DNSResolverInfo for human-readable text output. -func formatResolverString(r jsonoutput.DNSResolverInfo) string { +// formatResolverString formats a [tsdnsjsonv0.ResolverInfo] for human-readable text output. +func formatResolverString(r tsdnsjsonv0.ResolverInfo) string { if len(r.BootstrapResolution) > 0 { return fmt.Sprintf("%s (bootstrap: %v)", r.Addr, r.BootstrapResolution) } diff --git a/cmd/tailscale/cli/dns-status.go b/cmd/tailscale/cli/dns-status.go index 66a5e21d8..2b2e64871 100644 --- a/cmd/tailscale/cli/dns-status.go +++ b/cmd/tailscale/cli/dns-status.go @@ -13,10 +13,9 @@ import ( "strings" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/cmd/tailscale/cli/jsonoutput" - "tailscale.com/ipn" + + "tailscale.com/cmd/tailscale/tsdnsjsonv0" "tailscale.com/types/dnstype" - "tailscale.com/types/netmap" ) var dnsStatusCmd = &ffcli.Command{ @@ -86,9 +85,9 @@ var dnsStatusArgs struct { json bool } -// makeDNSResolverInfo converts a dnstype.Resolver to a jsonoutput.DNSResolverInfo. -func makeDNSResolverInfo(r *dnstype.Resolver) jsonoutput.DNSResolverInfo { - info := jsonoutput.DNSResolverInfo{Addr: r.Addr} +// makeDNSResolverInfo converts a [dnstype.Resolver] to a [tsdnsjsonv0.ResolverInfo]. +func makeDNSResolverInfo(r *dnstype.Resolver) tsdnsjsonv0.ResolverInfo { + info := tsdnsjsonv0.ResolverInfo{Addr: r.Addr} if r.BootstrapResolution != nil { info.BootstrapResolution = make([]string, 0, len(r.BootstrapResolution)) for _, a := range r.BootstrapResolution { @@ -109,28 +108,27 @@ func runDNSStatus(ctx context.Context, args []string) error { return err } - data := &jsonoutput.DNSStatusResult{ + data := &tsdnsjsonv0.StatusResponse{ TailscaleDNS: prefs.CorpDNS, } if s.CurrentTailnet != nil { - data.CurrentTailnet = &jsonoutput.DNSTailnetInfo{ + data.CurrentTailnet = &tsdnsjsonv0.TailnetInfo{ MagicDNSEnabled: s.CurrentTailnet.MagicDNSEnabled, MagicDNSSuffix: s.CurrentTailnet.MagicDNSSuffix, SelfDNSName: s.Self.DNSName, } - netMap, err := fetchNetMap() + dnsConfig, err := localClient.DNSConfig(ctx) if err != nil { - return fmt.Errorf("failed to fetch network map: %w", err) + return fmt.Errorf("failed to fetch DNS config: %w", err) } - dnsConfig := netMap.DNS for _, r := range dnsConfig.Resolvers { data.Resolvers = append(data.Resolvers, makeDNSResolverInfo(r)) } - data.SplitDNSRoutes = make(map[string][]jsonoutput.DNSResolverInfo) + data.SplitDNSRoutes = make(map[string][]tsdnsjsonv0.ResolverInfo) for k, v := range dnsConfig.Routes { for _, r := range v { data.SplitDNSRoutes[k] = append(data.SplitDNSRoutes[k], makeDNSResolverInfo(r)) @@ -152,7 +150,7 @@ func runDNSStatus(ctx context.Context, args []string) error { data.CertDomains = dnsConfig.CertDomains for _, er := range dnsConfig.ExtraRecords { - data.ExtraRecords = append(data.ExtraRecords, jsonoutput.DNSExtraRecord{ + data.ExtraRecords = append(data.ExtraRecords, tsdnsjsonv0.ExtraRecord{ Name: er.Name, Type: er.Type, Value: er.Value, @@ -169,7 +167,7 @@ func runDNSStatus(ctx context.Context, args []string) error { data.SystemDNSError = err.Error() } } else if osCfg != nil { - data.SystemDNS = &jsonoutput.DNSSystemConfig{ + data.SystemDNS = &tsdnsjsonv0.SystemConfig{ Nameservers: osCfg.Nameservers, SearchDomains: osCfg.SearchDomains, MatchDomains: osCfg.MatchDomains, @@ -189,7 +187,7 @@ func runDNSStatus(ctx context.Context, args []string) error { return nil } -func formatDNSStatusText(data *jsonoutput.DNSStatusResult, all bool) string { +func formatDNSStatusText(data *tsdnsjsonv0.StatusResponse, all bool) string { var sb strings.Builder fmt.Fprintf(&sb, "\n") @@ -357,19 +355,3 @@ func formatDNSStatusText(data *jsonoutput.DNSStatusResult, all bool) string { fmt.Fprintf(&sb, "[this is a preliminary version of this command; the output format may change in the future]\n") return sb.String() } - -func fetchNetMap() (netMap *netmap.NetworkMap, err error) { - w, err := localClient.WatchIPNBus(context.Background(), ipn.NotifyInitialNetMap) - if err != nil { - return nil, err - } - defer w.Close() - notify, err := w.Next() - if err != nil { - return nil, err - } - if notify.NetMap == nil { - return nil, fmt.Errorf("no network map yet available, please try again later") - } - return notify.NetMap, nil -} diff --git a/cmd/tailscale/cli/exitnode.go b/cmd/tailscale/cli/exitnode.go index 7ba4859d7..8b8333121 100644 --- a/cmd/tailscale/cli/exitnode.go +++ b/cmd/tailscale/cli/exitnode.go @@ -16,6 +16,7 @@ import ( "github.com/kballard/go-shellquote" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/envknob" + "tailscale.com/feature/buildfeatures" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" "tailscale.com/util/slicesx" @@ -43,6 +44,13 @@ func exitNodeCmd() *ffcli.Command { ShortUsage: "tailscale exit-node suggest", ShortHelp: "Suggest the best available exit node", Exec: runExitNodeSuggest, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("suggest") + if buildfeatures.HasRouteCheck { + fs.BoolVar(&exitNodeArgs.probe, "force-probe", false, hidden+"perform a routecheck probe before suggesting") + } + return fs + })(), }}, (func() []*ffcli.Command { if !envknob.UseWIPCode() { @@ -68,6 +76,7 @@ func exitNodeCmd() *ffcli.Command { var exitNodeArgs struct { filter string + probe bool } func exitNodeSetUse(wantOn bool) func(ctx context.Context, args []string) error { @@ -148,7 +157,11 @@ func runExitNodeList(ctx context.Context, args []string) error { // runExitNodeSuggest returns a suggested exit node ID to connect to and shows the chosen exit node tailcfg.StableNodeID. // If there are no derp based exit nodes to choose from or there is a failure in finding a suggestion, the command will return an error indicating so. func runExitNodeSuggest(ctx context.Context, args []string) error { - res, err := localClient.SuggestExitNode(ctx) + suggestExitNode := localClient.SuggestExitNode + if exitNodeArgs.probe { + suggestExitNode = localClient.SuggestExitNodeWithProbe + } + res, err := suggestExitNode(ctx) if err != nil { return fmt.Errorf("suggest exit node: %w", err) } diff --git a/cmd/tailscale/cli/file.go b/cmd/tailscale/cli/file.go index e7406bee3..a78caf369 100644 --- a/cmd/tailscale/cli/file.go +++ b/cmd/tailscale/cli/file.go @@ -32,6 +32,7 @@ import ( "tailscale.com/client/tailscale/apitype" "tailscale.com/cmd/tailscale/cli/ffcomplete" "tailscale.com/envknob" + "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/net/tsaddr" "tailscale.com/tailcfg" @@ -78,14 +79,16 @@ var fileCpCmd = &ffcli.Command{ fs.StringVar(&cpArgs.name, "name", "", "alternate filename to use, especially useful when is \"-\" (stdin)") fs.BoolVar(&cpArgs.verbose, "verbose", false, "verbose output") fs.BoolVar(&cpArgs.targets, "targets", false, "list possible file cp targets") + fs.DurationVar(&cpArgs.updateInterval, "update-interval", 250*time.Millisecond, "how often to repaint the progress line; zero or negative disables progress display entirely") return fs })(), } var cpArgs struct { - name string - verbose bool - targets bool + name string + verbose bool + targets bool + updateInterval time.Duration } func runCp(ctx context.Context, args []string) error { @@ -119,9 +122,6 @@ func runCp(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("can't send to %s: %v", target, err) } - if isOffline { - fmt.Fprintf(Stderr, "# warning: %s is offline\n", target) - } if len(files) > 1 { if cpArgs.name != "" { @@ -132,7 +132,51 @@ func runCp(ctx context.Context, args []string) error { } } - for _, fileArg := range files { + // outFiles tracks per-name push state, populated by a goroutine subscribed + // to the IPN bus. tailscaled's OutgoingFile.Sent is the bytes-pulled-toward- + // peerAPI signal; it stays at 0 until the peerAPI request body is actually + // being read, which is what we want both for the progress display and for + // disarming the offline warning. The CLI's local-side bytes counter would + // say "100% sent" the moment net/http buffers a small body into the local + // unix-socket conn to tailscaled, well before the peer has heard a thing. + type pushState struct { + sent atomic.Int64 + warnTimer *time.Timer // disarmed on first byte sent to peerAPI; nil after + } + var ( + outMu sync.Mutex + outFiles = map[string]*pushState{} // keyed by file name + ) + + busCtx, cancelBus := context.WithCancel(ctx) + defer cancelBus() + go watchOutgoingFiles(busCtx, stableID, func(name string, sent int64) { + outMu.Lock() + ps := outFiles[name] + outMu.Unlock() + if ps == nil { + return + } + // Only ever advance ps.sent forward. Bus updates can arrive late + // (after the success path below has already written contentLength + // to ps.sent for an instant final-100% paint), so we'd otherwise + // regress the count and the progress printer would compute a + // negative delta on its next tick. + for { + old := ps.sent.Load() + if sent <= old { + return + } + if ps.sent.CompareAndSwap(old, sent) { + if old == 0 && ps.warnTimer != nil { + ps.warnTimer.Stop() + } + return + } + } + }) + + for i, fileArg := range files { var fileContents *countingReader var name = cpArgs.name var contentLength int64 = -1 @@ -175,16 +219,57 @@ func runCp(ctx context.Context, args []string) error { log.Printf("sending %q to %v/%v/%v ...", name, target, ip, stableID) } + // Register this file with the watcher and, for the first file only, + // arm a timer that warns the user if no bytes have flowed to peerAPI + // after a few seconds. The watcher disarms it on first byte; PushFile + // returning also disarms it (cleanup, below). We don't gate on the + // netmap's Online bit (which can lag reality), but we do use it to + // pick between two warning messages. + ps := &pushState{} + if i == 0 { + ps.warnTimer = time.AfterFunc(3*time.Second, func() { + // vtRestartLine clears whatever (possibly progress) was on + // the current line, then we print the warning + \n so the + // next progress redraw lands on a fresh line below. + const vtRestartLine = "\r\x1b[K" + if isOffline { + fmt.Fprintf(Stderr, "%s# warning: %s is reportedly offline; trying anyway\n", vtRestartLine, target) + } else { + fmt.Fprintf(Stderr, "%s# warning: %s is not replying; trying anyway\n", vtRestartLine, target) + } + }) + } + outMu.Lock() + outFiles[name] = ps + outMu.Unlock() + var group sync.WaitGroup ctxProgress, cancelProgress := context.WithCancel(ctx) defer cancelProgress() - if isatty.IsTerminal(os.Stderr.Fd()) { - group.Go(func() { progressPrinter(ctxProgress, name, fileContents.n.Load, contentLength) }) + if cpArgs.updateInterval > 0 && isatty.IsTerminal(os.Stderr.Fd()) { + group.Go(func() { + progressPrinter(ctxProgress, name, ps.sent.Load, contentLength, cpArgs.updateInterval) + }) } err := localClient.PushFile(ctx, stableID, contentLength, name, fileContents) + if err == nil { + // PushFile can finish faster than the IPN bus delivers a final + // OutgoingFile update, leaving the progress display stuck at 0%. + // Synthesize a "fully done" count before stopping the printer so + // its final paint shows 100%. For stdin (contentLength == -1) we + // don't know the size, so fall back to the local read count. + if contentLength >= 0 { + ps.sent.Store(contentLength) + } else { + ps.sent.Store(fileContents.n.Load()) + } + } cancelProgress() group.Wait() // wait for progress printer to stop before reporting the error + if ps.warnTimer != nil { + ps.warnTimer.Stop() + } if err != nil { return err } @@ -195,15 +280,71 @@ func runCp(ctx context.Context, args []string) error { return nil } -func progressPrinter(ctx context.Context, name string, contentCount func() int64, contentLength int64) { +// watchOutgoingFiles subscribes to the IPN bus and invokes onUpdate once +// per OutgoingFile event for files going to peer. It runs until ctx is +// done (which runCp does on return) and is best-effort: if the bus +// subscription fails for any reason, onUpdate simply isn't called and the +// caller's progress display stays at 0 — exactly the right degradation, +// since the warning timer will then fire on its normal 3-second deadline. +func watchOutgoingFiles(ctx context.Context, peer tailcfg.StableNodeID, onUpdate func(name string, sent int64)) { + // NotifyPeerChanges opts in to per-peer add/remove notifications so the + // bus stays responsive without us also subscribing to the full NetMap, + // which we don't read here. + w, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialOutgoingFiles|ipn.NotifyPeerChanges) + if err != nil { + return + } + defer w.Close() + for { + n, err := w.Next() + if err != nil { + return + } + for _, of := range n.OutgoingFiles { + if of.PeerID != peer { + continue + } + // tailscaled keeps Finished entries in its OutgoingFiles map + // across PushFile calls (see feature/taildrop/ext.go), so a + // re-send of the same filename will see both the old completed + // (Sent == DeclaredSize) entry and the new in-progress one. + // Without this filter the watcher's monotonic CAS would latch + // onto the old entry's max value and the new transfer would + // appear stuck at 100% from the first bus tick. + if of.Finished { + continue + } + onUpdate(of.Name, of.Sent) + } + } +} + +// progressPrinter repaints a single-line transfer progress display every +// interval. interval must be > 0; runCp's caller gates on the +// --update-interval flag and skips invoking us when it's <= 0. +// +// It returns when ctx is done OR when it detects the transfer is stuck — +// "stuck" being: contentCount has equalled contentLength with a near-zero +// rate for >2 seconds. The stuck case prints a final newline so subsequent +// output (e.g. an error from PushFile) lands on a fresh line below the +// frozen progress line, instead of being painted over by it. +func progressPrinter(ctx context.Context, name string, contentCount func() int64, contentLength int64, interval time.Duration) { var rateValueFast, rateValueSlow tsrate.Value - rateValueFast.HalfLife = 1 * time.Second // fast response for rate measurement - rateValueSlow.HalfLife = 10 * time.Second // slow response for ETA measurement + // tailscaled emits OutgoingFile.Sent updates at ~1 Hz, so most printer + // ticks see no delta. With too short a half-life the displayed rate + // roughly halves between updates and doubles back when one arrives, + // looking jumpy. 5s keeps the swing under ~15% while still settling + // within a few seconds of a real change. + rateValueFast.HalfLife = 5 * time.Second // smoothed rate for display + rateValueSlow.HalfLife = 10 * time.Second // even slower, for ETA measurement var prevContentCount int64 print := func() { currContentCount := contentCount() - rateValueFast.Add(float64(currContentCount - prevContentCount)) - rateValueSlow.Add(float64(currContentCount - prevContentCount)) + // Clamp so a regression (which shouldn't happen, but tsrate.Value.Add + // panics on a negative count) can't take down the CLI. + delta := max(currContentCount-prevContentCount, 0) + rateValueFast.Add(float64(delta)) + rateValueSlow.Add(float64(delta)) prevContentCount = currContentCount const vtRestartLine = "\r\x1b[K" @@ -215,16 +356,23 @@ func progressPrinter(ctx context.Context, name string, contentCount func() int64 if contentLength >= 0 { currContentCount = min(currContentCount, contentLength) // cap at 100% ratioRemain := float64(currContentCount) / float64(contentLength) - bytesRemain := float64(contentLength - currContentCount) - secsRemain := bytesRemain / rateValueSlow.Rate() - secs := int(min(max(0, secsRemain), 99*60*60+59+60+59)) + etaStr := "ETA -" + if rate := rateValueSlow.Rate(); rate > 0 { + bytesRemain := float64(contentLength - currContentCount) + secsRemain := bytesRemain / rate + secs := int(min(max(0, secsRemain), 99*60*60+59+60+59)) + etaStr = fmt.Sprintf("ETA %02d:%02d:%02d", secs/60/60, (secs/60)%60, secs%60) + } fmt.Fprintf(os.Stderr, " %s %s", leftPad(fmt.Sprintf("%0.2f%%", 100.0*ratioRemain), len("100.00%")), - fmt.Sprintf("ETA %02d:%02d:%02d", secs/60/60, (secs/60)%60, secs%60)) + etaStr) } } - tc := time.NewTicker(250 * time.Millisecond) + const stuckAfter = 2 * time.Second + var fullStartedAt time.Time // when we first observed currCount==contentLength with ~zero rate + + tc := time.NewTicker(interval) defer tc.Stop() print() for { @@ -235,6 +383,24 @@ func progressPrinter(ctx context.Context, name string, contentCount func() int64 return case <-tc.C: print() + if contentLength < 0 { + continue + } + currCount := contentCount() + rate := rateValueFast.Rate() + if currCount >= contentLength && rate < 1 { + if fullStartedAt.IsZero() { + fullStartedAt = time.Now() + } else if time.Since(fullStartedAt) >= stuckAfter { + // Transfer is stuck at 100% with no movement. Stop + // repainting so we don't keep clobbering anything the + // rest of runCp prints (warnings, errors). + fmt.Fprintln(os.Stderr) + return + } + } else { + fullStartedAt = time.Time{} + } } } } @@ -328,7 +494,10 @@ peerLoop: return "", isOffline, errors.New("cannot send files: missing required Taildrop capability") case ipnstate.TaildropTargetOffline: - return "", isOffline, errors.New("cannot send files: peer is offline") + // Don't gate on the server-reported Online bit (which lags reality + // and isn't always accurate). runCp probes reachability itself with + // TSMP pings. + return foundPeer.ID, isOffline, nil case ipnstate.TaildropTargetNoPeerInfo: return "", isOffline, errors.New("cannot send files: invalid or unrecognized peer") @@ -379,10 +548,7 @@ type slowReader struct { func (r *slowReader) Read(p []byte) (n int, err error) { const burst = 4 << 10 - plen := len(p) - if plen > burst { - plen = burst - } + plen := min(len(p), burst) if r.rl == nil { r.rl = rate.NewLimiter(rate.Limit(1<<10), burst) } @@ -451,10 +617,10 @@ var fileGetCmd = &ffcli.Command{ Exec: runFileGet, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("get") - fs.BoolVar(&getArgs.wait, "wait", false, "wait for a file to arrive if inbox is empty") - fs.BoolVar(&getArgs.loop, "loop", false, "run get in a loop, receiving files as they come in") - fs.BoolVar(&getArgs.verbose, "verbose", false, "verbose output") - fs.Var(&getArgs.conflict, "conflict", "`behavior`"+` when a conflicting (same-named) file already exists in the target directory. + fs.BoolVar(&fileGetArgs.wait, "wait", false, "wait for a file to arrive if inbox is empty") + fs.BoolVar(&fileGetArgs.loop, "loop", false, "run get in a loop, receiving files as they come in") + fs.BoolVar(&fileGetArgs.verbose, "verbose", false, "verbose output") + fs.Var(&fileGetArgs.conflict, "conflict", "`behavior`"+` when a conflicting (same-named) file already exists in the target directory. skip: skip conflicting files: leave them in the taildrop inbox and print an error. get any non-conflicting files overwrite: overwrite existing file rename: write to a new number-suffixed filename`) @@ -463,7 +629,7 @@ var fileGetCmd = &ffcli.Command{ })(), } -var getArgs = struct { +var fileGetArgs = struct { wait bool loop bool verbose bool @@ -525,7 +691,7 @@ func receiveFile(ctx context.Context, wf apitype.WaitingFile, dir string) (targe return "", 0, fmt.Errorf("opening inbox file %q: %w", wf.Name, err) } defer rc.Close() - f, err := openFileOrSubstitute(dir, wf.Name, getArgs.conflict) + f, err := openFileOrSubstitute(dir, wf.Name, fileGetArgs.conflict) if err != nil { return "", 0, err } @@ -551,10 +717,10 @@ func runFileGetOneBatch(ctx context.Context, dir string) []error { errs = append(errs, fmt.Errorf("getting WaitingFiles: %w", err)) break } - if len(wfs) != 0 || !(getArgs.wait || getArgs.loop) { + if len(wfs) != 0 || !(fileGetArgs.wait || fileGetArgs.loop) { break } - if getArgs.verbose { + if fileGetArgs.verbose { printf("waiting for file...") } if err := waitForFile(ctx); err != nil { @@ -575,7 +741,7 @@ func runFileGetOneBatch(ctx context.Context, dir string) []error { errs = append(errs, err) continue } - if getArgs.verbose { + if fileGetArgs.verbose { printf("wrote %v as %v (%d bytes)\n", wf.Name, writtenFile, size) } if err = localClient.DeleteWaitingFile(ctx, wf.Name); err != nil { @@ -587,7 +753,7 @@ func runFileGetOneBatch(ctx context.Context, dir string) []error { if deleted == 0 && len(wfs) > 0 { // persistently stuck files are basically an error errs = append(errs, fmt.Errorf("moved %d/%d files", deleted, len(wfs))) - } else if getArgs.verbose { + } else if fileGetArgs.verbose { printf("moved %d/%d files\n", deleted, len(wfs)) } return errs @@ -607,7 +773,7 @@ func runFileGet(ctx context.Context, args []string) error { if fi, err := os.Stat(dir); err != nil || !fi.IsDir() { return fmt.Errorf("%q is not a directory", dir) } - if getArgs.loop { + if fileGetArgs.loop { for { errs := runFileGetOneBatch(ctx, dir) for _, err := range errs { @@ -639,7 +805,7 @@ func runFileGet(ctx context.Context, args []string) error { } func wipeInbox(ctx context.Context) error { - if getArgs.wait { + if fileGetArgs.wait { return errors.New("can't use --wait with /dev/null target") } wfs, err := localClient.WaitingFiles(ctx) @@ -648,7 +814,7 @@ func wipeInbox(ctx context.Context) error { } deleted := 0 for _, wf := range wfs { - if getArgs.verbose { + if fileGetArgs.verbose { log.Printf("deleting %v ...", wf.Name) } if err := localClient.DeleteWaitingFile(ctx, wf.Name); err != nil { @@ -656,7 +822,7 @@ func wipeInbox(ctx context.Context) error { } deleted++ } - if getArgs.verbose { + if fileGetArgs.verbose { log.Printf("deleted %d files", deleted) } return nil diff --git a/cmd/tailscale/cli/get.go b/cmd/tailscale/cli/get.go new file mode 100644 index 000000000..b896dcb52 --- /dev/null +++ b/cmd/tailscale/cli/get.go @@ -0,0 +1,240 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "strings" + "text/tabwriter" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/net/tsaddr" + "tailscale.com/types/views" +) + +var getCmd = &ffcli.Command{ + Name: "get", + ShortUsage: "tailscale get [flags] [setting-name | all]", + ShortHelp: "Show current preference values", + LongHelp: `"tailscale get" shows the current value of one or all preferences. + +With no argument or "all", all preferences are shown. +With a specific setting name, only that value is shown. + +The setting names are the same flag names accepted by "tailscale set".`, + FlagSet: getFlags, + Exec: runGet, +} + +type getArgsT struct { + json bool + setFlags bool +} + +var getArgs getArgsT + +var getFlags = newGetFlagSet(&getArgs) + +func newGetFlagSet(args *getArgsT) *flag.FlagSet { + fs := newFlagSet("get") + fs.BoolVar(&args.json, "json", false, "output as JSON") + fs.BoolVar(&args.setFlags, "set-flags", false, "output as \"tailscale set\" flag arguments") + return fs +} + +// getSetting is a single preference name-value pair. +type getSetting struct { + name string + value any +} + +func runGet(ctx context.Context, args []string) error { + prefs, err := localClient.GetPrefs(ctx) + if err != nil { + return err + } + st, err := localClient.Status(ctx) + if err != nil { + return err + } + + settings, wantAll, err := selectSettings(prefs, st, effectiveGOOS(), args) + if err != nil { + return err + } + + switch { + case getArgs.json: + return getOutputJSON(settings) + case getArgs.setFlags: + return getOutputSetFlags(settings) + case !wantAll: + // Single value: just print the raw value. + outln(fmt.Sprint(settings[0].value)) + return nil + default: + return getOutputTable(settings) + } +} + +// selectSettings validates args and returns the settings to display. +// wantAll reports whether the caller asked for all settings (no arg or "all"). +func selectSettings(prefs *ipn.Prefs, st *ipnstate.Status, goos string, args []string) (settings []getSetting, wantAll bool, err error) { + if len(args) > 1 { + return nil, false, fmt.Errorf("too many arguments: %q", args) + } + wantAll = len(args) == 0 || args[0] == "all" + if wantAll { + return getSettingsFromPrefs(prefs, st, goos, false), true, nil + } + wantName := args[0] + // When querying a specific name, include hidden flags. + for _, s := range getSettingsFromPrefs(prefs, st, goos, true) { + if s.name == wantName { + return []getSetting{s}, false, nil + } + } + return nil, false, fmt.Errorf("unknown setting %q; see \"tailscale set --help\" for valid settings", wantName) +} + +// getSettingsFromPrefs returns get-able settings derived from prefs, +// using the same flag names as "tailscale set". +// If includeHidden is false, flags with hidden usage strings are omitted. +func getSettingsFromPrefs(prefs *ipn.Prefs, st *ipnstate.Status, goos string, includeHidden bool) []getSetting { + // Use the set command's flag set to get the canonical ordered list + // of flag names and to determine OS applicability. + var dummy setArgsT + fs := newSetFlagSet(goos, &dummy) + + var settings []getSetting + fs.VisitAll(func(f *flag.Flag) { + if preflessFlag(f.Name) { + return + } + if !includeHidden && strings.HasPrefix(f.Usage, hidden) { + return + } + v := prefValue(f.Name, prefs, st) + settings = append(settings, getSetting{name: f.Name, value: v}) + }) + return settings +} + +// prefValue returns the current value of the preference corresponding to +// the given "tailscale set" flag name. +func prefValue(flagName string, prefs *ipn.Prefs, st *ipnstate.Status) any { + switch flagName { + case "accept-routes": + return prefs.RouteAll + case "accept-dns": + return prefs.CorpDNS + case "exit-node": + if prefs.AutoExitNode.IsSet() { + return ipn.AutoExitNodePrefix + string(prefs.AutoExitNode) + } + ip := exitNodeIP(prefs, st) + if ip.IsValid() { + return ip.String() + } + return "" + case "exit-node-allow-lan-access": + return prefs.ExitNodeAllowLANAccess + case "shields-up": + return prefs.ShieldsUp + case "ssh": + return prefs.RunSSH + case "hostname": + return prefs.Hostname + case "advertise-routes": + var sb strings.Builder + for i, r := range tsaddr.WithoutExitRoutes(views.SliceOf(prefs.AdvertiseRoutes)).All() { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(r.String()) + } + return sb.String() + case "advertise-exit-node": + return tsaddr.ContainsExitRoutes(views.SliceOf(prefs.AdvertiseRoutes)) + case "advertise-connector": + return prefs.AppConnector.Advertise + case "nickname": + return prefs.ProfileName + case "update-check": + return prefs.AutoUpdate.Check + case "auto-update": + return prefs.AutoUpdate.Apply.EqualBool(true) + case "report-posture": + return prefs.PostureChecking + case "webclient": + return prefs.RunWebClient + case "operator": + return prefs.OperatorUser + case "snat-subnet-routes": + return !prefs.NoSNAT + case "stateful-filtering": + val, ok := prefs.NoStatefulFiltering.Get() + if ok && val { + return false + } + return true + case "netfilter-mode": + return prefs.NetfilterMode.String() + case "unattended": + return prefs.ForceDaemon + case "sync": + return prefs.Sync.EqualBool(true) + case "remote-config": + return prefs.RemoteConfig + case "relay-server-port": + if prefs.RelayServerPort != nil { + return fmt.Sprint(*prefs.RelayServerPort) + } + return "" + case "relay-server-static-endpoints": + parts := make([]string, len(prefs.RelayServerStaticEndpoints)) + for i, ep := range prefs.RelayServerStaticEndpoints { + parts[i] = ep.String() + } + return strings.Join(parts, ",") + default: + return nil + } +} + +func getOutputTable(settings []getSetting) error { + w := tabwriter.NewWriter(Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintf(w, "NAME\tVALUE\n") + for _, s := range settings { + fmt.Fprintf(w, "%s\t%v\n", s.name, s.value) + } + return w.Flush() +} + +func getOutputJSON(settings []getSetting) error { + m := make(map[string]any, len(settings)) + for _, s := range settings { + m[s.name] = s.value + } + j, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + outln(string(j)) + return nil +} + +func getOutputSetFlags(settings []getSetting) error { + var parts []string + for _, s := range settings { + parts = append(parts, fmtFlagValueArg(s.name, s.value)) + } + outln(strings.Join(parts, " ")) + return nil +} diff --git a/cmd/tailscale/cli/get_test.go b/cmd/tailscale/cli/get_test.go new file mode 100644 index 000000000..7b9f13ea3 --- /dev/null +++ b/cmd/tailscale/cli/get_test.go @@ -0,0 +1,596 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "bytes" + "encoding/json" + "flag" + "io" + "net/netip" + "reflect" + "strings" + "testing" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/safesocket" + "tailscale.com/tailcfg" + "tailscale.com/tstest" + "tailscale.com/types/key" + "tailscale.com/types/opt" + "tailscale.com/types/preftype" +) + +func TestPrefValue(t *testing.T) { + port := uint16(41641) + peerKey := key.NewNode().Public() + exitPeerID := tailcfg.StableNodeID("exit-peer") + exitPeerIP := netip.MustParseAddr("100.64.0.5") + + stWithExitPeer := &ipnstate.Status{ + Peer: map[key.NodePublic]*ipnstate.PeerStatus{ + peerKey: { + ID: exitPeerID, + TailscaleIPs: []netip.Addr{exitPeerIP}, + }, + }, + } + + tests := []struct { + name string + flag string + prefs *ipn.Prefs + st *ipnstate.Status + want any + }{ + // Simple boolean prefs. + { + name: "accept-routes-true", + flag: "accept-routes", + prefs: &ipn.Prefs{RouteAll: true}, + want: true, + }, + { + name: "accept-routes-false", + flag: "accept-routes", + prefs: &ipn.Prefs{}, + want: false, + }, + { + name: "accept-dns", + flag: "accept-dns", + prefs: &ipn.Prefs{CorpDNS: true}, + want: true, + }, + { + name: "exit-node-allow-lan-access", + flag: "exit-node-allow-lan-access", + prefs: &ipn.Prefs{ExitNodeAllowLANAccess: true}, + want: true, + }, + { + name: "shields-up", + flag: "shields-up", + prefs: &ipn.Prefs{ShieldsUp: true}, + want: true, + }, + { + name: "ssh", + flag: "ssh", + prefs: &ipn.Prefs{RunSSH: true}, + want: true, + }, + { + name: "advertise-connector", + flag: "advertise-connector", + prefs: &ipn.Prefs{AppConnector: ipn.AppConnectorPrefs{Advertise: true}}, + want: true, + }, + { + name: "update-check", + flag: "update-check", + prefs: &ipn.Prefs{AutoUpdate: ipn.AutoUpdatePrefs{Check: true}}, + want: true, + }, + { + name: "report-posture", + flag: "report-posture", + prefs: &ipn.Prefs{PostureChecking: true}, + want: true, + }, + { + name: "webclient", + flag: "webclient", + prefs: &ipn.Prefs{RunWebClient: true}, + want: true, + }, + { + name: "unattended", + flag: "unattended", + prefs: &ipn.Prefs{ForceDaemon: true}, + want: true, + }, + + // Simple string prefs. + { + name: "hostname", + flag: "hostname", + prefs: &ipn.Prefs{Hostname: "myhost"}, + want: "myhost", + }, + { + name: "nickname", + flag: "nickname", + prefs: &ipn.Prefs{ProfileName: "work"}, + want: "work", + }, + { + name: "operator", + flag: "operator", + prefs: &ipn.Prefs{OperatorUser: "alice"}, + want: "alice", + }, + + // exit-node has three branches. + { + name: "exit-node/auto", + flag: "exit-node", + prefs: &ipn.Prefs{AutoExitNode: ipn.AnyExitNode}, + want: "auto:any", + }, + { + name: "exit-node/by-ip", + flag: "exit-node", + prefs: &ipn.Prefs{ExitNodeIP: netip.MustParseAddr("100.64.0.1")}, + want: "100.64.0.1", + }, + { + name: "exit-node/by-id-resolves-via-status", + flag: "exit-node", + prefs: &ipn.Prefs{ExitNodeID: exitPeerID}, + st: stWithExitPeer, + want: exitPeerIP.String(), + }, + { + name: "exit-node/empty", + flag: "exit-node", + prefs: &ipn.Prefs{}, + want: "", + }, + + // advertise-routes filters out exit routes, comma-joins. + { + name: "advertise-routes/multiple", + flag: "advertise-routes", + prefs: &ipn.Prefs{AdvertiseRoutes: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/24"), + netip.MustParsePrefix("192.168.0.0/16"), + }}, + want: "10.0.0.0/24,192.168.0.0/16", + }, + { + name: "advertise-routes/excludes-exit-routes", + flag: "advertise-routes", + prefs: &ipn.Prefs{AdvertiseRoutes: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/24"), + netip.MustParsePrefix("0.0.0.0/0"), + netip.MustParsePrefix("::/0"), + }}, + want: "10.0.0.0/24", + }, + { + name: "advertise-routes/empty", + flag: "advertise-routes", + prefs: &ipn.Prefs{}, + want: "", + }, + + // advertise-exit-node derives from AdvertiseRoutes. + { + name: "advertise-exit-node/true", + flag: "advertise-exit-node", + prefs: &ipn.Prefs{AdvertiseRoutes: []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/0"), + netip.MustParsePrefix("::/0"), + }}, + want: true, + }, + { + name: "advertise-exit-node/false-empty", + flag: "advertise-exit-node", + prefs: &ipn.Prefs{}, + want: false, + }, + { + name: "advertise-exit-node/false-only-subnet", + flag: "advertise-exit-node", + prefs: &ipn.Prefs{AdvertiseRoutes: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/24"), + }}, + want: false, + }, + + // auto-update and sync use opt.Bool.EqualBool(true). + { + name: "auto-update/unset-is-false", + flag: "auto-update", + prefs: &ipn.Prefs{}, + want: false, + }, + { + name: "auto-update/explicit-true", + flag: "auto-update", + prefs: &ipn.Prefs{AutoUpdate: ipn.AutoUpdatePrefs{Apply: opt.NewBool(true)}}, + want: true, + }, + { + name: "auto-update/explicit-false", + flag: "auto-update", + prefs: &ipn.Prefs{AutoUpdate: ipn.AutoUpdatePrefs{Apply: opt.NewBool(false)}}, + want: false, + }, + { + name: "sync/unset-is-false", + flag: "sync", + prefs: &ipn.Prefs{}, + want: false, + }, + { + name: "sync/explicit-true", + flag: "sync", + prefs: &ipn.Prefs{Sync: opt.NewBool(true)}, + want: true, + }, + + // snat-subnet-routes is inverted. + { + name: "snat-subnet-routes/default-true", + flag: "snat-subnet-routes", + prefs: &ipn.Prefs{}, + want: true, + }, + { + name: "snat-subnet-routes/false-when-no-snat", + flag: "snat-subnet-routes", + prefs: &ipn.Prefs{NoSNAT: true}, + want: false, + }, + + // stateful-filtering: the inversion of NoStatefulFiltering, defaulting on. + { + name: "stateful-filtering/unset-is-true", + flag: "stateful-filtering", + prefs: &ipn.Prefs{}, + want: true, + }, + { + name: "stateful-filtering/explicit-disabled-no-stateful", + flag: "stateful-filtering", + prefs: &ipn.Prefs{NoStatefulFiltering: opt.NewBool(true)}, + want: false, + }, + { + name: "stateful-filtering/explicit-enabled-no-stateful", + flag: "stateful-filtering", + prefs: &ipn.Prefs{NoStatefulFiltering: opt.NewBool(false)}, + want: true, + }, + + // netfilter-mode renders via String(). + { + name: "netfilter-mode/off", + flag: "netfilter-mode", + prefs: &ipn.Prefs{NetfilterMode: preftype.NetfilterOff}, + want: "off", + }, + { + name: "netfilter-mode/on", + flag: "netfilter-mode", + prefs: &ipn.Prefs{NetfilterMode: preftype.NetfilterOn}, + want: "on", + }, + + // relay-server-port: nil pointer vs explicit. + { + name: "relay-server-port/unset", + flag: "relay-server-port", + prefs: &ipn.Prefs{}, + want: "", + }, + { + name: "relay-server-port/set", + flag: "relay-server-port", + prefs: &ipn.Prefs{RelayServerPort: &port}, + want: "41641", + }, + + // relay-server-static-endpoints: empty vs joined. + { + name: "relay-server-static-endpoints/empty", + flag: "relay-server-static-endpoints", + prefs: &ipn.Prefs{}, + want: "", + }, + { + name: "relay-server-static-endpoints/multiple", + flag: "relay-server-static-endpoints", + prefs: &ipn.Prefs{RelayServerStaticEndpoints: []netip.AddrPort{ + netip.MustParseAddrPort("192.0.2.1:40000"), + netip.MustParseAddrPort("[2001:db8::1]:40000"), + }}, + want: "192.0.2.1:40000,[2001:db8::1]:40000", + }, + + // Unknown flag returns nil. This guards against the default branch + // silently producing nil for a flag that should have been wired up. + { + name: "unknown-flag", + flag: "no-such-flag", + prefs: &ipn.Prefs{}, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := tt.st + if st == nil { + st = &ipnstate.Status{} + } + got := prefValue(tt.flag, tt.prefs, st) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("prefValue(%q) = %v (%T), want %v (%T)", + tt.flag, got, got, tt.want, tt.want) + } + }) + } +} + +// TestPrefValueCoversAllSetFlags is the load-bearing guard: every flag +// that "tailscale set" exposes must have a corresponding prefValue case, +// or "tailscale get" silently returns nil for it. It iterates the set +// command's flag set across the platforms whose flag sets differ, so +// OS-conditional flags (snat-subnet-routes, netfilter-mode, unattended, +// operator, ...) are all covered. +func TestPrefValueCoversAllSetFlags(t *testing.T) { + for _, goos := range []string{"linux", "darwin", "windows"} { + t.Run(goos, func(t *testing.T) { + var dummy setArgsT + fs := newSetFlagSet(goos, &dummy) + fs.VisitAll(func(f *flag.Flag) { + if preflessFlag(f.Name) { + return + } + if got := prefValue(f.Name, &ipn.Prefs{}, &ipnstate.Status{}); got == nil { + t.Errorf("prefValue(%q) returned nil; add a case for it in prefValue", f.Name) + } + }) + }) + } +} + +func TestGetSettingsFromPrefsHiddenFlag(t *testing.T) { + prefs := &ipn.Prefs{} + st := &ipnstate.Status{} + + visible := getSettingsFromPrefs(prefs, st, "linux", false) + if containsSetting(visible, "sync") { + t.Error("expected hidden flag --sync to be excluded when includeHidden=false") + } + if !containsSetting(visible, "accept-dns") { + t.Error("expected visible flag --accept-dns to be included") + } + + withHidden := getSettingsFromPrefs(prefs, st, "linux", true) + if !containsSetting(withHidden, "sync") { + t.Error("expected hidden flag --sync to be included when includeHidden=true") + } + + // Ordering must match the set flag set's VisitAll order. + var wantOrder []string + var dummy setArgsT + newSetFlagSet("linux", &dummy).VisitAll(func(f *flag.Flag) { + if preflessFlag(f.Name) { + return + } + wantOrder = append(wantOrder, f.Name) + }) + var gotOrder []string + for _, s := range withHidden { + gotOrder = append(gotOrder, s.name) + } + if !reflect.DeepEqual(gotOrder, wantOrder) { + t.Errorf("setting order = %v, want %v", gotOrder, wantOrder) + } +} + +func TestSelectSettings(t *testing.T) { + prefs := &ipn.Prefs{Hostname: "h", CorpDNS: true} + st := &ipnstate.Status{} + const goos = "linux" + + t.Run("empty-args-returns-all-visible", func(t *testing.T) { + got, wantAll, err := selectSettings(prefs, st, goos, nil) + if err != nil { + t.Fatal(err) + } + if !wantAll { + t.Error("wantAll = false; want true") + } + if containsSetting(got, "sync") { + t.Error("hidden flag --sync leaked into all-settings result") + } + if !containsSetting(got, "hostname") { + t.Error("missing --hostname in all-settings result") + } + }) + + t.Run("all-arg-same-as-empty", func(t *testing.T) { + empty, _, err := selectSettings(prefs, st, goos, nil) + if err != nil { + t.Fatal(err) + } + allArg, wantAll, err := selectSettings(prefs, st, goos, []string{"all"}) + if err != nil { + t.Fatal(err) + } + if !wantAll { + t.Error("wantAll = false; want true for explicit \"all\"") + } + if !reflect.DeepEqual(empty, allArg) { + t.Errorf("\"all\" produced %v, empty produced %v", allArg, empty) + } + }) + + t.Run("specific-visible-flag", func(t *testing.T) { + got, wantAll, err := selectSettings(prefs, st, goos, []string{"hostname"}) + if err != nil { + t.Fatal(err) + } + if wantAll { + t.Error("wantAll = true; want false for specific name") + } + if len(got) != 1 || got[0].name != "hostname" || got[0].value != "h" { + t.Errorf("got %+v, want [{hostname h}]", got) + } + }) + + t.Run("specific-hidden-flag", func(t *testing.T) { + // Hidden flags must be reachable by exact name. + got, _, err := selectSettings(prefs, st, goos, []string{"sync"}) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].name != "sync" { + t.Errorf("got %+v, want [{sync ...}]", got) + } + }) + + t.Run("unknown-flag-errors", func(t *testing.T) { + _, _, err := selectSettings(prefs, st, goos, []string{"no-such-flag"}) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "unknown setting") || !strings.Contains(err.Error(), "no-such-flag") { + t.Errorf("error %q missing expected substrings", err) + } + }) + + t.Run("too-many-args-errors", func(t *testing.T) { + _, _, err := selectSettings(prefs, st, goos, []string{"hostname", "ssh"}) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "too many arguments") { + t.Errorf("error %q missing \"too many arguments\"", err) + } + }) + + t.Run("os-conditional-flag-on-wrong-goos", func(t *testing.T) { + // "netfilter-mode" is registered only on linux. Asking for it + // on darwin should produce an "unknown setting" error. + _, _, err := selectSettings(prefs, st, "darwin", []string{"netfilter-mode"}) + if err == nil || !strings.Contains(err.Error(), "unknown setting") { + t.Errorf("got err=%v, want \"unknown setting\"", err) + } + // And operator is peer-creds-only. + if safesocket.GOOSUsesPeerCreds("windows") { + t.Skip("operator is exposed on windows") + } + _, _, err = selectSettings(prefs, st, "windows", []string{"operator"}) + if err == nil || !strings.Contains(err.Error(), "unknown setting") { + t.Errorf("got err=%v, want \"unknown setting\"", err) + } + }) +} + +func TestGetOutputJSON(t *testing.T) { + var buf bytes.Buffer + tstest.Replace[io.Writer](t, &Stdout, &buf) + + settings := []getSetting{ + {name: "accept-dns", value: true}, + {name: "hostname", value: "myhost"}, + {name: "advertise-routes", value: "10.0.0.0/24"}, + {name: "shields-up", value: false}, + } + if err := getOutputJSON(settings); err != nil { + t.Fatal(err) + } + + var got map[string]any + if err := json.Unmarshal(buf.Bytes(), &got); err != nil { + t.Fatalf("output is not valid JSON: %v\noutput: %s", err, buf.String()) + } + want := map[string]any{ + "accept-dns": true, + "hostname": "myhost", + "advertise-routes": "10.0.0.0/24", + "shields-up": false, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestGetOutputTable(t *testing.T) { + var buf bytes.Buffer + tstest.Replace[io.Writer](t, &Stdout, &buf) + + settings := []getSetting{ + {name: "accept-dns", value: true}, + {name: "hostname", value: "myhost"}, + } + if err := getOutputTable(settings); err != nil { + t.Fatal(err) + } + + out := buf.String() + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if len(lines) != 3 { + t.Fatalf("got %d lines, want 3:\n%s", len(lines), out) + } + if !strings.HasPrefix(lines[0], "NAME") || !strings.Contains(lines[0], "VALUE") { + t.Errorf("header line = %q, want NAME ... VALUE", lines[0]) + } + if !strings.HasPrefix(lines[1], "accept-dns") || !strings.HasSuffix(lines[1], "true") { + t.Errorf("row 1 = %q", lines[1]) + } + if !strings.HasPrefix(lines[2], "hostname") || !strings.HasSuffix(lines[2], "myhost") { + t.Errorf("row 2 = %q", lines[2]) + } +} + +func TestGetOutputSetFlags(t *testing.T) { + var buf bytes.Buffer + tstest.Replace[io.Writer](t, &Stdout, &buf) + + settings := []getSetting{ + {name: "ssh", value: true}, + {name: "shields-up", value: false}, + {name: "hostname", value: "myhost"}, + {name: "advertise-routes", value: ""}, + } + if err := getOutputSetFlags(settings); err != nil { + t.Fatal(err) + } + + got := strings.TrimSpace(buf.String()) + // true → bare flag; false → --flag=false; empty string → --flag=; other → --flag=value + want := "--ssh --shields-up=false --hostname=myhost --advertise-routes=" + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// containsSetting reports whether settings contains a setting with the given name. +func containsSetting(settings []getSetting, name string) bool { + for _, s := range settings { + if s.name == name { + return true + } + } + return false +} diff --git a/cmd/tailscale/cli/ip.go b/cmd/tailscale/cli/ip.go index b76ef0a70..6ff1cf867 100644 --- a/cmd/tailscale/cli/ip.go +++ b/cmd/tailscale/cli/ip.go @@ -13,13 +13,14 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" ) var ipCmd = &ffcli.Command{ Name: "ip", - ShortUsage: "tailscale ip [-1] [-4] [-6] [peer hostname or ip address]", + ShortUsage: "tailscale ip [-1] [-4] [-6] [peer or service hostname or ip address]", ShortHelp: "Show Tailscale IP addresses", - LongHelp: "Show Tailscale IP addresses for peer. Peer defaults to the current machine.", + LongHelp: "Show Tailscale IP addresses for peer or service. Peer defaults to the current machine.", Exec: runIP, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("ip") @@ -79,10 +80,20 @@ func runIP(ctx context.Context, args []string) error { return err } peer, ok := peerMatchingIP(st, ip) - if !ok { - return fmt.Errorf("no peer found with IP %v", ip) + if ok { + ips = peer.TailscaleIPs + } else { + // No peer matched; check if the IP belongs to a service. + serviceIPs, err := serviceAddrsMatchingIP(ctx, ip) + if err != nil { + return err + } + if serviceIPs != nil { + ips = serviceIPs + } else { + return fmt.Errorf("no peer or service found with IP %v", ip) + } } - ips = peer.TailscaleIPs } if len(ips) == 0 { return fmt.Errorf("no current Tailscale IPs; state: %v", st.BackendState) @@ -109,6 +120,31 @@ func runIP(ctx context.Context, args []string) error { return nil } +// serviceAddrsMatchingIP checks whether ipStr matches a service's VIP address +// and returns the service's addresses if so. +func serviceAddrsMatchingIP(ctx context.Context, ipStr string) ([]netip.Addr, error) { + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return nil, nil + } + services, err := localClient.GetServices(ctx) + if err != nil { + return nil, err + } + return allIPsForServiceWithIP(services, ip), nil +} + +// allIPsForServiceWithIP returns the Addrs of the service whose VIP addresses +// contain ip, or nil if no service matches. +func allIPsForServiceWithIP(services map[tailcfg.ServiceName]tailcfg.ServiceDetails, ip netip.Addr) []netip.Addr { + for _, svc := range services { + if slices.Contains(svc.Addrs, ip) { + return svc.Addrs + } + } + return nil +} + func peerMatchingIP(st *ipnstate.Status, ipStr string) (ps *ipnstate.PeerStatus, ok bool) { ip, err := netip.ParseAddr(ipStr) if err != nil { diff --git a/cmd/tailscale/cli/ip_test.go b/cmd/tailscale/cli/ip_test.go new file mode 100644 index 000000000..9e5d31166 --- /dev/null +++ b/cmd/tailscale/cli/ip_test.go @@ -0,0 +1,199 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "net/netip" + "testing" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/key" +) + +func TestPeerMatchingIP(t *testing.T) { + st := &ipnstate.Status{ + Self: &ipnstate.PeerStatus{ + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1"), netip.MustParseAddr("fd7a:115c:a1e0::1")}, + }, + Peer: map[key.NodePublic]*ipnstate.PeerStatus{ + key.NewNode().Public(): { + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.2"), netip.MustParseAddr("fd7a:115c:a1e0::2")}, + }, + key.NewNode().Public(): { + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.3")}, + }, + }, + } + + tests := []struct { + name string + ipStr string + wantOK bool + wantIPs []netip.Addr + }{ + { + name: "match_self_v4", + ipStr: "100.64.0.1", + wantOK: true, + wantIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1"), netip.MustParseAddr("fd7a:115c:a1e0::1")}, + }, + { + name: "match_self_v6", + ipStr: "fd7a:115c:a1e0::1", + wantOK: true, + wantIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1"), netip.MustParseAddr("fd7a:115c:a1e0::1")}, + }, + { + name: "match_peer_v4", + ipStr: "100.64.0.2", + wantOK: true, + wantIPs: []netip.Addr{netip.MustParseAddr("100.64.0.2"), netip.MustParseAddr("fd7a:115c:a1e0::2")}, + }, + { + name: "match_peer_single_ip", + ipStr: "100.64.0.3", + wantOK: true, + wantIPs: []netip.Addr{netip.MustParseAddr("100.64.0.3")}, + }, + { + name: "no_match", + ipStr: "100.64.0.99", + wantOK: false, + }, + { + name: "invalid_ip", + ipStr: "not-an-ip", + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ps, ok := peerMatchingIP(st, tt.ipStr) + if ok != tt.wantOK { + t.Fatalf("peerMatchingIP(%q) ok = %v, want %v", tt.ipStr, ok, tt.wantOK) + } + if ok { + if len(ps.TailscaleIPs) != len(tt.wantIPs) { + t.Fatalf("got %d IPs, want %d", len(ps.TailscaleIPs), len(tt.wantIPs)) + } + for i, ip := range ps.TailscaleIPs { + if ip != tt.wantIPs[i] { + t.Errorf("IP[%d] = %v, want %v", i, ip, tt.wantIPs[i]) + } + } + } + }) + } +} + +func TestAllIPsForServiceWithIP(t *testing.T) { + services := map[tailcfg.ServiceName]tailcfg.ServiceDetails{ + "svc:web": { + Name: "svc:web", + Addrs: []netip.Addr{ + netip.MustParseAddr("100.100.0.1"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::1"), + }, + }, + "svc:api": { + Name: "svc:api", + Addrs: []netip.Addr{netip.MustParseAddr("100.100.0.2")}, + }, + } + // Services should have at most 2 addrs (one v4, one v6), but + // we handle more gracefully if the server ever returns them. + multiAddr := map[tailcfg.ServiceName]tailcfg.ServiceDetails{ + "svc:multi": { + Name: "svc:multi", + Addrs: []netip.Addr{ + netip.MustParseAddr("100.100.0.3"), + netip.MustParseAddr("100.100.0.4"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::3"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::4"), + }, + }, + } + + tests := []struct { + name string + services map[tailcfg.ServiceName]tailcfg.ServiceDetails + ip netip.Addr + wantIPs []netip.Addr + }{ + { + name: "match_service_v4", + services: services, + ip: netip.MustParseAddr("100.100.0.1"), + wantIPs: []netip.Addr{ + netip.MustParseAddr("100.100.0.1"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::1"), + }, + }, + { + name: "match_service_v6", + services: services, + ip: netip.MustParseAddr("fd7a:115c:a1e0:ab12::1"), + wantIPs: []netip.Addr{ + netip.MustParseAddr("100.100.0.1"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::1"), + }, + }, + { + name: "match_single_addr_service", + services: services, + ip: netip.MustParseAddr("100.100.0.2"), + wantIPs: []netip.Addr{netip.MustParseAddr("100.100.0.2")}, + }, + { + name: "match_service_multiple_addrs_v4", + services: multiAddr, + ip: netip.MustParseAddr("100.100.0.3"), + wantIPs: []netip.Addr{ + netip.MustParseAddr("100.100.0.3"), + netip.MustParseAddr("100.100.0.4"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::3"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::4"), + }, + }, + { + name: "match_service_multiple_addrs_v6", + services: multiAddr, + ip: netip.MustParseAddr("fd7a:115c:a1e0:ab12::3"), + wantIPs: []netip.Addr{ + netip.MustParseAddr("100.100.0.3"), + netip.MustParseAddr("100.100.0.4"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::3"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::4"), + }, + }, + { + name: "no_match", + services: services, + ip: netip.MustParseAddr("100.100.0.99"), + wantIPs: nil, + }, + { + name: "empty_services", + services: nil, + ip: netip.MustParseAddr("100.100.0.1"), + wantIPs: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := allIPsForServiceWithIP(tt.services, tt.ip) + if len(got) != len(tt.wantIPs) { + t.Fatalf("allIPsForServiceWithIP(%v) returned %d IPs, want %d", tt.ip, len(got), len(tt.wantIPs)) + } + for i, ip := range got { + if ip != tt.wantIPs[i] { + t.Errorf("IP[%d] = %v, want %v", i, ip, tt.wantIPs[i]) + } + } + }) + } +} diff --git a/cmd/tailscale/cli/jsonoutput/jsonoutput.go b/cmd/tailscale/cli/jsonoutput/jsonoutput.go deleted file mode 100644 index 69e7374d9..000000000 --- a/cmd/tailscale/cli/jsonoutput/jsonoutput.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -// Package jsonoutput provides stable and versioned JSON serialisation for CLI output. -// This allows us to provide stable output to scripts/clients, but also make -// breaking changes to the output when it's useful. -// -// Historically we only used `--json` as a boolean flag, so changing the output -// could break scripts that rely on the existing format. -// -// This package allows callers to pass a version number to `--json` and get -// a consistent output. We'll bump the version when we make a breaking change -// that's likely to break scripts that rely on the existing output, e.g. if -// we remove a field or change the type/format. -// -// Passing just the boolean flag `--json` will always return v1, to preserve -// compatibility with scripts written before we versioned our output. -package jsonoutput - -import ( - "errors" - "fmt" - "strconv" -) - -// JSONSchemaVersion implements flag.Value, and tracks whether the CLI has -// been called with `--json`, and if so, with what value. -type JSONSchemaVersion struct { - // IsSet tracks if the flag was provided at all. - IsSet bool - - // Value tracks the desired schema version, which defaults to 1 if - // the user passes `--json` without an argument. - Value int -} - -// String returns the default value which is printed in the CLI help text. -func (v *JSONSchemaVersion) String() string { - if v.IsSet { - return strconv.Itoa(v.Value) - } else { - return "(not set)" - } -} - -// Set is called when the user passes the flag as a command-line argument. -func (v *JSONSchemaVersion) Set(s string) error { - if v.IsSet { - return errors.New("received multiple instances of --json; only pass it once") - } - - v.IsSet = true - - // If the user doesn't supply a schema version, default to 1. - // This ensures that any existing scripts will continue to get their - // current output. - if s == "true" { - v.Value = 1 - return nil - } - - version, err := strconv.Atoi(s) - if err != nil { - return fmt.Errorf("invalid integer value passed to --json: %q", s) - } - v.Value = version - return nil -} - -// IsBoolFlag tells the flag package that JSONSchemaVersion can be set -// without an argument. -func (v *JSONSchemaVersion) IsBoolFlag() bool { - return true -} - -// ResponseEnvelope is a set of fields common to all versioned JSON output. -type ResponseEnvelope struct { - // SchemaVersion is the version of the JSON output, e.g. "1", "2", "3" - SchemaVersion string - - // ResponseWarning tells a user if a newer version of the JSON output - // is available. - ResponseWarning string `json:"_WARNING,omitzero"` -} diff --git a/cmd/tailscale/cli/maybe_captiveportal.go b/cmd/tailscale/cli/maybe_captiveportal.go new file mode 100644 index 000000000..1994b5628 --- /dev/null +++ b/cmd/tailscale/cli/maybe_captiveportal.go @@ -0,0 +1,10 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_captiveportal + +package cli + +// Import the netcheck captive portal hook package so that the netcheck +// command also probes for captive portals during its report. +import _ "tailscale.com/feature/captiveportal/netcheckhook" diff --git a/cmd/tailscale/cli/netcheck.go b/cmd/tailscale/cli/netcheck.go index 5e45445c7..52d66d516 100644 --- a/cmd/tailscale/cli/netcheck.go +++ b/cmd/tailscale/cli/netcheck.go @@ -26,6 +26,7 @@ import ( "tailscale.com/net/portmapper/portmappertype" "tailscale.com/net/tlsdial" "tailscale.com/tailcfg" + "tailscale.com/tstime" "tailscale.com/types/logger" "tailscale.com/util/eventbus" "tailscale.com/util/set" @@ -143,7 +144,7 @@ func runNetcheck(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("netcheck: %w", err) } - if err := printReport(dm, report); err != nil { + if err := printNetCheckReport(dm, report); err != nil { return err } if netcheckArgs.every == 0 { @@ -153,7 +154,7 @@ func runNetcheck(ctx context.Context, args []string) error { } } -func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error { +func printNetCheckReport(dm *tailcfg.DERPMap, report *netcheck.Report) error { var j []byte var err error switch netcheckArgs.format { @@ -175,7 +176,7 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error { } printf("\nReport:\n") - printf("\t* Time: %v\n", report.Now.Format(time.RFC3339Nano)) + printf("\t* Time: %v\n", report.Now.Local().Format(tstime.DateSpTimeNanoZ)) printf("\t* UDP: %v\n", report.UDP) if report.GlobalV4.IsValid() { printf("\t* IPv4: yes, %s\n", report.GlobalV4) diff --git a/cmd/tailscale/cli/routecheck.go b/cmd/tailscale/cli/routecheck.go new file mode 100644 index 000000000..c5162f98f --- /dev/null +++ b/cmd/tailscale/cli/routecheck.go @@ -0,0 +1,122 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_routecheck + +package cli + +import ( + "cmp" + "context" + "flag" + "fmt" + "slices" + "strings" + "text/tabwriter" + + jsonv2 "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" + "github.com/peterbourgon/ff/v3/ffcli" + + "tailscale.com/cmd/tailscale/jsonoutput" + "tailscale.com/cmd/tailscale/tsroutecheckjsonv0" + "tailscale.com/net/routecheck" + "tailscale.com/tstime" +) + +func init() { + maybeRoutecheckCmd = routecheckCmd +} + +var routecheckCmd = func() *ffcli.Command { + return &ffcli.Command{ + Name: "routecheck", + ShortUsage: "tailscale routecheck", + ShortHelp: "Print a reachability report for routes with multiple paths", + LongHelp: hidden + `"tailscale routecheck" is an experimental feature; it is not a stable interface`, + Exec: runRoutecheck, + FlagSet: routecheckFlagSet, + } +} + +var routecheckFlagSet = func() *flag.FlagSet { + fs := newFlagSet("routecheck") + fs.BoolVar(&routecheckArgs.probe, "probe", false, "probe now to generate a new reachability report") + fs.Var(&routecheckArgs.format, "format", `output format: empty (for human-readable), "json" or "json-line"`) + fs.Var(routecheckArgs.format.JSONBool(), "json", "output in JSON format") + return fs +}() + +var routecheckArgs struct { + probe bool + format jsonoutput.Format +} + +func runRoutecheck(ctx context.Context, args []string) error { + routeCheck := localClient.RouteCheck + if routecheckArgs.probe { + routeCheck = localClient.RouteCheckProbe + } + rp, err := routeCheck(ctx) + if err != nil { + return fmt.Errorf("routecheck: %w", err) + } + if err := printRouteCheckReport(rp); err != nil { + return err + } + return nil +} + +func printRouteCheckReport(rp *routecheck.Report) error { + var enc *jsontext.Encoder + switch routecheckArgs.format.String() { + case "": + case "json": + enc = jsontext.NewEncoder(Stdout, jsontext.WithIndent("\t")) + case "json-line": + enc = jsontext.NewEncoder(Stdout, jsontext.Multiline(false)) + default: + return fmt.Errorf("unknown output format %q", routecheckArgs.format) + } + + if rp == nil { + return fmt.Errorf("routecheck: report unavailable") + } + routes := rp.RoutablePrefixes() + + // Don’t render prefixes that only have one router: + for pfx, nodes := range routes { + if len(nodes) <= 1 { + delete(routes, pfx) + } + } + + if enc != nil { + out := tsroutecheckjsonv0.ReportResponse{ + Done: rp.Done, + Routes: routes, + } + if err := jsonv2.MarshalEncode(enc, out); err != nil { + return err + } + if _, err := Stdout.Write([]byte("\n")); err != nil { + return err + } + return nil + } + + w := tabwriter.NewWriter(Stdout, 10, 5, 5, ' ', 0) + defer w.Flush() + fmt.Fprintf(w, "\nReachable routers at %s:\n", rp.Done.Local().Format(tstime.DateSpTimeZ)) + fmt.Fprintf(w, "\n %s\t%s\t%s", "PREFIX", "IP", "HOSTNAME") + for prefix, nodes := range routes.Sorted() { + slices.SortFunc(nodes, func(a, b routecheck.Node) int { + return cmp.Compare(a.Name, b.Name) // order by hostname + }) + for _, n := range nodes { + fmt.Fprintf(w, "\n %s\t%s\t%s", prefix, n.Addr, strings.TrimSuffix(n.Name, ".")) + } + } + fmt.Fprintln(w) + return nil +} diff --git a/cmd/tailscale/cli/serve_legacy.go b/cmd/tailscale/cli/serve_legacy.go index 837d88513..b8d9e2766 100644 --- a/cmd/tailscale/cli/serve_legacy.go +++ b/cmd/tailscale/cli/serve_legacy.go @@ -628,7 +628,7 @@ func (e *serveEnv) runServeStatus(ctx context.Context, args []string) error { return nil } printFunnelStatus(ctx) - if sc == nil || (len(sc.TCP) == 0 && len(sc.Web) == 0 && len(sc.AllowFunnel) == 0) { + if isServeConfigEmpty(sc) { printf("No serve config\n") return nil } @@ -636,18 +636,8 @@ func (e *serveEnv) runServeStatus(ctx context.Context, args []string) error { if err != nil { return err } - if sc.IsTCPForwardingAny() { - if err := printTCPStatusTree(ctx, sc, st); err != nil { - return err - } - printf("\n") - } - for hp := range sc.Web { - err := e.printWebStatusTree(sc, hp) - if err != nil { - return err - } - printf("\n") + if err := printServeStatusTrees(sc, st); err != nil { + return err } printFunnelWarning(sc) return nil @@ -660,82 +650,97 @@ func printTCPStatusTree(ctx context.Context, sc *ipn.ServeConfig, st *ipnstate.S continue } hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(p)))) - tlsStatus := "TLS over TCP" - if h.TerminateTLS != "" { - tlsStatus = "TLS terminated" - } fStatus := "tailnet only" if sc.AllowFunnel[hp] { fStatus = "Funnel on" } - printf("|-- tcp://%s (%s, %s)\n", hp, tlsStatus, fStatus) + if h.TerminateTLS != "" { + printf("|-- tcp://%s (TLS-terminated TCP, %s)\n", hp, fStatus) + } else { + printf("|-- tcp://%s (%s)\n", hp, fStatus) + } for _, a := range st.TailscaleIPs { ipp := net.JoinHostPort(a.String(), strconv.Itoa(int(p))) printf("|-- tcp://%s\n", ipp) } - printf("|--> tcp://%s\n", h.TCPForward) + if strings.HasPrefix(h.TCPForward, "unix:") { + printf("|--> %s\n", h.TCPForward) + } else { + printf("|--> tcp://%s\n", h.TCPForward) + } } return nil } -func (e *serveEnv) printWebStatusTree(sc *ipn.ServeConfig, hp ipn.HostPort) error { - // No-op if no serve config - if sc == nil { +// printWebStatusTree renders one Web entry (the URL line plus its handler +// tree) for either a node-level serve or a service-level serve. When +// svcName is the empty string, the entry is treated as node-level. Service +// entries include the service name in the URL annotation. +// +// funnel and https are computed by the caller from the parent ServeConfig +// so this function does not need a reference to it. +func printWebStatusTree(wsc *ipn.WebServerConfig, hp ipn.HostPort, funnel, https bool, svcName tailcfg.ServiceName) error { + if wsc == nil { return nil } - fStatus := "tailnet only" - if sc.AllowFunnel[hp] { - fStatus = "Funnel on" - } host, portStr, _ := net.SplitHostPort(string(hp)) - - port, err := parseServePort(portStr) - if err != nil { - return fmt.Errorf("invalid port %q: %w", portStr, err) - } - scheme := "https" - if sc.IsServingHTTP(port, noService) { + if !https { scheme = "http" } - portPart := ":" + portStr if scheme == "http" && portStr == "80" || scheme == "https" && portStr == "443" { portPart = "" } - if scheme == "http" { - hostname, _, _ := strings.Cut(host, ".") - printf("%s://%s%s (%s)\n", scheme, hostname, portPart, fStatus) + + fStatus := "tailnet only" + if funnel { + fStatus = "Funnel on" } - printf("%s://%s%s (%s)\n", scheme, host, portPart, fStatus) - srvTypeAndDesc := func(h *ipn.HTTPHandler) (string, string) { - switch { - case h.Path != "": - return "path", h.Path - case h.Proxy != "": - return "proxy", h.Proxy - case h.Text != "": - return "text", "\"" + elipticallyTruncate(h.Text, 20) + "\"" + if svcName != "" { + printf("%s://%s%s (%s) (%s)\n", scheme, host, portPart, fStatus, svcName) + } else { + if scheme == "http" { + hostname, _, _ := strings.Cut(host, ".") + printf("%s://%s%s (%s)\n", scheme, hostname, portPart, fStatus) } - return "", "" + printf("%s://%s%s (%s)\n", scheme, host, portPart, fStatus) } - mounts := slicesx.MapKeys(sc.Web[hp].Handlers) + mounts := slicesx.MapKeys(wsc.Handlers) + if len(mounts) == 0 { + return nil + } sort.Slice(mounts, func(i, j int) bool { return len(mounts[i]) < len(mounts[j]) }) maxLen := len(mounts[len(mounts)-1]) for _, m := range mounts { - h := sc.Web[hp].Handlers[m] - t, d := srvTypeAndDesc(h) + h := wsc.Handlers[m] + t, d := serveHandlerDesc(h) printf("%s %s%s %-5s %s\n", "|--", m, strings.Repeat(" ", maxLen-len(m)), t, d) } return nil } +// serveHandlerDesc returns the type label and description for an +// HTTPHandler, matching the format used by the Web tree printer for +// node-level and service-level serves. +func serveHandlerDesc(h *ipn.HTTPHandler) (string, string) { + switch { + case h.Path != "": + return "path", h.Path + case h.Proxy != "": + return "proxy", h.Proxy + case h.Text != "": + return "text", "\"" + elipticallyTruncate(h.Text, 20) + "\"" + } + return "", "" +} + func elipticallyTruncate(s string, max int) string { if len(s) <= max { return s @@ -848,10 +853,10 @@ func (e *serveEnv) enableFeatureInteractive(ctx context.Context, feature string, e.lc.IncrementCounter(ctx, fmt.Sprintf("%s_enablement_lost_connection", feature), 1) return err } - if nm := n.NetMap; nm != nil && nm.SelfNode.Valid() { + if self := n.SelfChange; self != nil { gotAll := true for _, c := range caps { - if !nm.SelfNode.HasCap(c) { + if _, has := self.CapMap[c]; !has { // The feature is not yet enabled. // Continue blocking until it is. gotAll = false diff --git a/cmd/tailscale/cli/serve_status.go b/cmd/tailscale/cli/serve_status.go new file mode 100644 index 000000000..6ba85ca59 --- /dev/null +++ b/cmd/tailscale/cli/serve_status.go @@ -0,0 +1,120 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_serve + +package cli + +import ( + "context" + "maps" + "net" + "slices" + "strconv" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" +) + +// isServeConfigEmpty reports whether sc has no user-visible configuration +// to render in the non-JSON status output. +func isServeConfigEmpty(sc *ipn.ServeConfig) bool { + return sc == nil || (len(sc.TCP) == 0 && len(sc.Web) == 0 && len(sc.Services) == 0 && len(sc.AllowFunnel) == 0) +} + +// printServeStatusTrees prints the tree-style human-readable status of sc, +// including any node-level TCP and Web serve entries and any configured +// services, to [Stdout]. It does not print the funnel-status header, the +// no-config message, or the trailing funnel warning — callers are expected +// to handle those. +// +// Ordering is deterministic: node TCP forwards (existing behavior), then +// node Web entries by HostPort, then services by name. +func printServeStatusTrees(sc *ipn.ServeConfig, st *ipnstate.Status) error { + if sc == nil { + return nil + } + if sc.IsTCPForwardingAny() { + if err := printTCPStatusTree(context.Background(), sc, st); err != nil { + return err + } + printf("\n") + } + for _, hp := range slices.Sorted(maps.Keys(sc.Web)) { + _, portStr, _ := net.SplitHostPort(string(hp)) + port, err := parseServePort(portStr) + if err != nil { + return err + } + funnel := sc.AllowFunnel[hp] + https := !sc.IsServingHTTP(port, noService) + if err := printWebStatusTree(sc.Web[hp], hp, funnel, https, noService); err != nil { + return err + } + printf("\n") + } + for _, name := range slices.Sorted(maps.Keys(sc.Services)) { + if err := printServiceStatusTree(sc, st, name); err != nil { + return err + } + } + return nil +} + +// printServiceStatusTree prints the tree-style status for a single +// configured service. Each rendered URL/forward line is prefixed with the +// service name in the URL annotation (e.g. +// "https://db.example.ts.net (tailnet only) (svc:db)") so service entries +// are visually distinct from node-level serves. +func printServiceStatusTree(sc *ipn.ServeConfig, st *ipnstate.Status, name tailcfg.ServiceName) error { + svc, ok := sc.Services[name] + if !ok || svc == nil { + return nil + } + + if svc.Tun { + printf("tun (L3 forwarding) (%s)\n\n", name) + return nil + } + + suffix := "" + if st != nil && st.CurrentTailnet != nil { + suffix = st.CurrentTailnet.MagicDNSSuffix + } + host := name.WithoutPrefix() + if suffix != "" { + host = host + "." + suffix + } + + // TCP forwards configured directly on the service. + for _, p := range slices.Sorted(maps.Keys(svc.TCP)) { + h := svc.TCP[p] + if h == nil || h.TCPForward == "" { + continue + } + hp := ipn.HostPort(net.JoinHostPort(host, strconv.Itoa(int(p)))) + if h.TerminateTLS != "" { + printf("tcp://%s (TLS-terminated TCP, tailnet only) (%s)\n", hp, name) + } else { + printf("tcp://%s (tailnet only) (%s)\n", hp, name) + } + printf("|--> tcp://%s\n\n", h.TCPForward) + } + + // Web entries (HTTP/HTTPS). Services have no Funnel concept. + for _, hp := range slices.Sorted(maps.Keys(svc.Web)) { + _, portStr, _ := net.SplitHostPort(string(hp)) + port, err := parseServePort(portStr) + if err != nil { + return err + } + https := !sc.IsServingHTTP(port, name) + if err := printWebStatusTree(svc.Web[hp], hp, false, https, name); err != nil { + return err + } + printf("\n") + } + + return nil +} diff --git a/cmd/tailscale/cli/serve_status_test.go b/cmd/tailscale/cli/serve_status_test.go new file mode 100644 index 000000000..821a1d667 --- /dev/null +++ b/cmd/tailscale/cli/serve_status_test.go @@ -0,0 +1,308 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_serve + +package cli + +import ( + "bytes" + "encoding/json" + "io" + "net" + "strings" + "testing" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/tstest" +) + +// statusTestStatus is a minimal ipnstate.Status used by serve-status tests. +var statusTestStatus = &ipnstate.Status{ + BackendState: ipn.Running.String(), + Self: &ipnstate.PeerStatus{ + DNSName: "foo.test.ts.net.", + }, + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, +} + +func TestPrintServeStatusTrees(t *testing.T) { + tests := []struct { + name string + sc *ipn.ServeConfig + want string + }{ + { + name: "node_web_tailnet_only", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:3000"}, + }}, + }, + }, + want: strings.Join([]string{ + "https://foo.test.ts.net (tailnet only)", + "|-- / proxy http://127.0.0.1:3000", + "", + "", + }, "\n"), + }, + { + name: "node_tcp_funnel_on", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{2222: {TCPForward: "127.0.0.1:22"}}, + AllowFunnel: map[ipn.HostPort]bool{ + "foo.test.ts.net:2222": true, + }, + }, + want: strings.Join([]string{ + "|-- tcp://foo.test.ts.net:2222 (Funnel on)", + "|--> tcp://127.0.0.1:22", + "", + "", + }, "\n"), + }, + { + name: "node_tls_terminated_tcp_tailnet", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {TCPForward: "127.0.0.1:8080", TerminateTLS: "foo.test.ts.net"}, + }, + }, + want: strings.Join([]string{ + "|-- tcp://foo.test.ts.net:443 (TLS-terminated TCP, tailnet only)", + "|--> tcp://127.0.0.1:8080", + "", + "", + }, "\n"), + }, + { + name: "service_web_only", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:db": { + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "db.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:5432"}, + }}, + }, + }, + }, + }, + want: strings.Join([]string{ + "https://db.test.ts.net (tailnet only) (svc:db)", + "|-- / proxy http://127.0.0.1:5432", + "", + "", + }, "\n"), + }, + { + name: "service_tcp_forward", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:ssh": { + TCP: map[uint16]*ipn.TCPPortHandler{2222: {TCPForward: "127.0.0.1:22"}}, + }, + }, + }, + want: strings.Join([]string{ + "tcp://ssh.test.ts.net:2222 (tailnet only) (svc:ssh)", + "|--> tcp://127.0.0.1:22", + "", + "", + }, "\n"), + }, + { + name: "service_tls_terminated_tcp", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {TCPForward: "127.0.0.1:8080", TerminateTLS: "foo.test.ts.net"}, + }, + }, + }, + }, + want: strings.Join([]string{ + "tcp://foo.test.ts.net:443 (TLS-terminated TCP, tailnet only) (svc:foo)", + "|--> tcp://127.0.0.1:8080", + "", + "", + }, "\n"), + }, + { + name: "service_tun", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:vpn": {Tun: true}, + }, + }, + want: strings.Join([]string{ + "tun (L3 forwarding) (svc:vpn)", + "", + "", + }, "\n"), + }, + { + name: "node_and_services_mixed", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:3000"}, + }}, + }, + AllowFunnel: map[ipn.HostPort]bool{ + "foo.test.ts.net:443": true, + }, + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:db": { + TCP: map[uint16]*ipn.TCPPortHandler{5432: {TCPForward: "127.0.0.1:5432"}}, + }, + }, + }, + want: strings.Join([]string{ + "https://foo.test.ts.net (Funnel on)", + "|-- / proxy http://127.0.0.1:3000", + "", + "tcp://db.test.ts.net:5432 (tailnet only) (svc:db)", + "|--> tcp://127.0.0.1:5432", + "", + "", + }, "\n"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + tstest.Replace(t, &Stdout, io.Writer(&stdout)) + tstest.Replace(t, &Stderr, io.Writer(&stderr)) + + if err := printServeStatusTrees(tt.sc, statusTestStatus); err != nil { + t.Fatalf("printServeStatusTrees: %v", err) + } + if got := stdout.String(); got != tt.want { + t.Errorf("\nGot:\n%q\nExpected:\n%q", got, tt.want) + } + if got := stderr.String(); got != "" { + t.Errorf("unexpected Stderr output: %q", got) + } + }) + } +} + +// TestPrintServeStatusTreesParity asserts that the host-identifying keys +// visible in the JSON serialization of a ServeConfig also appear in the +// human-readable output, so the two views stay in lockstep. This is the +// parity contract from issue #34163. +// +// It checks: +// - every Services key (service name) +// - every node-level Web HostPort host +// - every service-level Web HostPort host +// - every node-level TCP forward as a host:port string +// - every tun-mode service rendering the "tun" marker after its name +func TestPrintServeStatusTreesParity(t *testing.T) { + sc := &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + 2222: {TCPForward: "127.0.0.1:22"}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:3000"}, + }}, + }, + AllowFunnel: map[ipn.HostPort]bool{ + "foo.test.ts.net:2222": true, + }, + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:db": { + TCP: map[uint16]*ipn.TCPPortHandler{5432: {TCPForward: "127.0.0.1:5432"}}, + }, + "svc:web": { + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "web.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/api": {Proxy: "http://127.0.0.1:9000"}, + }}, + }, + }, + "svc:vpn": {Tun: true}, + }, + } + + // Marshal to JSON and reparse as a generic map so the parity check walks + // the same wire shape clients see, not the typed Go values. + jsonBytes, err := json.Marshal(sc) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + var raw map[string]any + if err := json.Unmarshal(jsonBytes, &raw); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + + var stdout, stderr bytes.Buffer + tstest.Replace(t, &Stdout, io.Writer(&stdout)) + tstest.Replace(t, &Stderr, io.Writer(&stderr)) + + if err := printServeStatusTrees(sc, statusTestStatus); err != nil { + t.Fatalf("printServeStatusTrees: %v", err) + } + if got := stderr.String(); got != "" { + t.Errorf("unexpected Stderr output: %q", got) + } + human := stdout.String() + + services, _ := raw["Services"].(map[string]any) + for name, sval := range services { + if !strings.Contains(human, name) { + t.Errorf("human output missing service name %q\n--- human ---\n%s", name, human) + } + svc, _ := sval.(map[string]any) + if tun, _ := svc["Tun"].(bool); tun { + tunLine := "tun (L3 forwarding) (" + name + ")" + if !strings.Contains(human, tunLine) { + t.Errorf("human output missing tun marker for %q\n--- human ---\n%s", tunLine, human) + } + } + web, _ := svc["Web"].(map[string]any) + for hp := range web { + host := strings.SplitN(hp, ":", 2)[0] + if !strings.Contains(human, host) { + t.Errorf("human output missing service %s Web host %q\n--- human ---\n%s", name, host, human) + } + } + } + + if web, ok := raw["Web"].(map[string]any); ok { + for hp := range web { + host := strings.SplitN(hp, ":", 2)[0] + if !strings.Contains(human, host) { + t.Errorf("human output missing node Web host %q\n--- human ---\n%s", host, human) + } + } + } + + nodeHost := strings.TrimSuffix(statusTestStatus.Self.DNSName, ".") + if tcp, ok := raw["TCP"].(map[string]any); ok { + for portStr, hVal := range tcp { + h, _ := hVal.(map[string]any) + fwd, _ := h["TCPForward"].(string) + if fwd == "" { + continue + } + hostport := net.JoinHostPort(nodeHost, portStr) + if !strings.Contains(human, hostport) { + t.Errorf("human output missing node TCP forward %q\n--- human ---\n%s", hostport, human) + } + } + } +} diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 13f5c09b8..6a41f7156 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -702,19 +702,26 @@ func (e *serveEnv) runServeGetConfig(ctx context.Context, args []string) (err er } else { proto = conffile.ProtoTCP } - destHost, destPortStr, err := net.SplitHostPort(config.TCPForward) - if err != nil { - return nil, fmt.Errorf("parse TCPForward=%q: %w", config.TCPForward, err) + if strings.HasPrefix(config.TCPForward, "unix:") { + mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ + Protocol: proto, + Destination: config.TCPForward, + }) + } else { + destHost, destPortStr, err := net.SplitHostPort(config.TCPForward) + if err != nil { + return nil, fmt.Errorf("parse TCPForward=%q: %w", config.TCPForward, err) + } + destPort, err := strconv.ParseUint(destPortStr, 10, 16) + if err != nil { + return nil, fmt.Errorf("parse port %q: %w", destPortStr, err) + } + mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ + Protocol: proto, + Destination: destHost, + DestinationPorts: tailcfg.PortRange{First: uint16(destPort), Last: uint16(destPort)}, + }) } - destPort, err := strconv.ParseUint(destPortStr, 10, 16) - if err != nil { - return nil, fmt.Errorf("parse port %q: %w", destPortStr, err) - } - mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ - Protocol: proto, - Destination: destHost, - DestinationPorts: tailcfg.PortRange{First: uint16(destPort), Last: uint16(destPort)}, - }) } else if config.HTTP || config.HTTPS { webKey := ipn.HostPort(net.JoinHostPort(sniName, strconv.FormatUint(uint64(port), 10))) handlers, ok := serviceConfig.Web[webKey] @@ -732,25 +739,38 @@ func (e *serveEnv) runServeGetConfig(ctx context.Context, args []string) (err er DestinationPorts: tailcfg.PortRange{}, }) } else if defaultHandler.Proxy != "" { - proto, rest, ok := strings.Cut(defaultHandler.Proxy, "://") - if !ok { - return nil, fmt.Errorf("service %q: invalid proxy handler %q", svcName, defaultHandler.Proxy) - } - host, portStr, err := net.SplitHostPort(rest) - if err != nil { - return nil, fmt.Errorf("service %q: invalid proxy handler %q: %w", svcName, defaultHandler.Proxy, err) - } + if strings.HasPrefix(defaultHandler.Proxy, "unix:") { + // HTTP over unix socket: h.Proxy is "unix:/path" without "://". + // The inbound protocol is HTTP(S); infer from useTLS. + httpProto := conffile.ProtoHTTP + if config.HTTPS { + httpProto = conffile.ProtoHTTPS + } + mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ + Protocol: httpProto, + Destination: defaultHandler.Proxy, + }) + } else { + proto, rest, ok := strings.Cut(defaultHandler.Proxy, "://") + if !ok { + return nil, fmt.Errorf("service %q: invalid proxy handler %q", svcName, defaultHandler.Proxy) + } + host, portStr, err := net.SplitHostPort(rest) + if err != nil { + return nil, fmt.Errorf("service %q: invalid proxy handler %q: %w", svcName, defaultHandler.Proxy, err) + } - port, err := strconv.ParseUint(portStr, 10, 16) - if err != nil { - return nil, fmt.Errorf("service %q: parse port %q: %w", svcName, portStr, err) - } + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil { + return nil, fmt.Errorf("service %q: parse port %q: %w", svcName, portStr, err) + } - mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ - Protocol: conffile.ServiceProtocol(proto), - Destination: host, - DestinationPorts: tailcfg.PortRange{First: uint16(port), Last: uint16(port)}, - }) + mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ + Protocol: conffile.ServiceProtocol(proto), + Destination: host, + DestinationPorts: tailcfg.PortRange{First: uint16(port), Last: uint16(port)}, + }) + } } } } @@ -800,24 +820,57 @@ func (e *serveEnv) runServeGetConfig(ctx context.Context, args []string) (err er return err } +// serveConfigDocsURL documents the Services configuration file format that set-config prefers +const serveConfigDocsURL = "https://tailscale.com/kb/1589/tailscale-services-configuration-file" + +const serveLegacyFormatWarning = "Warning: %q is in the legacy raw serve config format " + + "(as emitted by `tailscale serve status --json`), which is deprecated for set-config. " + + "Applying its services only. To migrate, run `tailscale serve get-config` to save your " + + "configuration in the supported format; see %s\n" + +const serveLegacyDroppedWarning = "Warning: ignoring node-level fields not supported by set-config: %s\n" + +// legacyNodeLevelFields returns the names of the populated top-level fields in +// sc, other than Services, that set-config does not apply (it is services-only). +func legacyNodeLevelFields(sc *ipn.ServeConfig) []string { + rest := sc.Clone() + rest.Services = nil + b, err := json.Marshal(rest) + if err != nil { + return nil + } + var m map[string]json.RawMessage + if err := json.Unmarshal(b, &m); err != nil { + return nil + } + fields := make([]string, 0, len(m)) + for k := range m { + fields = append(fields, k) + } + sort.Strings(fields) + return fields +} + func (e *serveEnv) runServeSetConfig(ctx context.Context, args []string) (err error) { if len(args) != 1 { return errors.New("must specify filename") } + filename := args[0] forSingleService := e.service.Validate() == nil - - var scf *conffile.ServicesConfigFile if e.allServices && forSingleService { return errors.New("cannot specify both --all and --service") - } else if e.allServices { - scf, err = conffile.LoadServicesConfig(args[0], "") - } else if forSingleService { - scf, err = conffile.LoadServicesConfig(args[0], e.service.String()) - } else { + } + if !e.allServices && !forSingleService { return errors.New("must specify either --service=svc: or --all") } + + forService := "" + if forSingleService { + forService = e.service.String() + } + scf, err := conffile.LoadServicesConfig(filename, forService) if err != nil { - return fmt.Errorf("could not read config from file %q: %w", args[0], err) + return fmt.Errorf("could not read config from file %q: %w", filename, err) } st, err := e.getLocalClientStatusWithoutPeers(ctx) @@ -842,6 +895,29 @@ func (e *serveEnv) runServeSetConfig(ctx context.Context, args []string) (err er } advertisedServices := set.Set[string]{} + if scf.Version == conffile.LegacyVersion { + // Legacy raw ipn.ServeConfig (e.g. "tailscale serve status --json" + // output). Deprecated for set-config; apply only its services-oriented + // content, with a migration warning to stderr (never stdout, which + // callers may pipe). + legacy := scf.Legacy + fmt.Fprintf(e.stderr(), serveLegacyFormatWarning, filename, serveConfigDocsURL) + if dropped := legacyNodeLevelFields(legacy); len(dropped) > 0 { + fmt.Fprintf(e.stderr(), serveLegacyDroppedWarning, strings.Join(dropped, ", ")) + } + for name, svcCfg := range legacy.Services { + if forSingleService && name != e.service { + continue + } + mak.Set(&sc.Services, name, svcCfg.Clone()) + advertisedServices.Add(name.String()) + } + if forSingleService && sc.Services[e.service] == nil { + return fmt.Errorf("service %q not found in %q", e.service, filename) + } + } + + // scf.Services is nil for the legacy format, making this loop a no-op then. for name, details := range scf.Services { for ppr, ep := range details.Endpoints { if ep.Protocol == conffile.ProtoTUN { @@ -861,6 +937,10 @@ func (e *serveEnv) runServeSetConfig(ctx context.Context, args []string) (err er var target string if ep.Protocol == conffile.ProtoFile { target = ep.Destination + } else if strings.HasPrefix(ep.Destination, "unix:") { + // Unix socket target: pass "unix:/path" through to setServe. + // Supported for HTTP(S), TCP, and TLS-terminated-TCP inbound. + target = ep.Destination } else { // map source port range 1-1 to destination port range destPort := ep.DestinationPorts.First + (port - ppr.Ports.First) @@ -1045,21 +1125,28 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN output.WriteString(fmt.Sprintf("%s %-5s %s\n\n", "|--", t, d)) } } else if tcpHandler != nil { - - tlsStatus := "TLS over TCP" + var annotations []string if tcpHandler.TerminateTLS != "" { - tlsStatus = "TLS terminated" + annotations = append(annotations, "TLS terminated") } if ver := tcpHandler.ProxyProtocol; ver != 0 { - tlsStatus = fmt.Sprintf("%s, PROXY protocol v%d", tlsStatus, ver) + annotations = append(annotations, fmt.Sprintf("PROXY protocol v%d", ver)) } - output.WriteString(fmt.Sprintf("|-- tcp://%s:%d (%s)\n", host, srvPort, tlsStatus)) + output.WriteString(fmt.Sprintf("|-- tcp://%s:%d", host, srvPort)) + if len(annotations) != 0 { + output.WriteString(fmt.Sprintf(" (%s)", strings.Join(annotations, ", "))) + } + output.WriteString("\n") for _, a := range ips { ipp := net.JoinHostPort(a.String(), strconv.Itoa(int(srvPort))) output.WriteString(fmt.Sprintf("|-- tcp://%s\n", ipp)) } - output.WriteString(fmt.Sprintf("|--> tcp://%s\n\n", tcpHandler.TCPForward)) + if strings.HasPrefix(tcpHandler.TCPForward, "unix:") { + output.WriteString(fmt.Sprintf("|--> %s\n\n", tcpHandler.TCPForward)) + } else { + output.WriteString(fmt.Sprintf("|--> tcp://%s\n\n", tcpHandler.TCPForward)) + } } if !forService && !e.bg.Value { @@ -1122,8 +1209,8 @@ func (e *serveEnv) shouldWarnRemoteDestCompatibility(ctx context.Context, target return nil } - if filepath.IsAbs(target) || strings.HasPrefix(target, "text:") { - // local path or text target, nothing to check + if filepath.IsAbs(target) || strings.HasPrefix(target, "text:") || strings.HasPrefix(target, "unix:") { + // local path, text target, or unix socket, nothing to check return nil } @@ -1216,14 +1303,28 @@ func (e *serveEnv) applyTCPServe(sc *ipn.ServeConfig, dnsName string, srcType se svcName := tailcfg.AsServiceName(dnsName) - targetURL, err := ipn.ExpandProxyTargetValue(target, []string{"tcp"}, "tcp") + targetURL, err := ipn.ExpandProxyTargetValue(target, []string{"tcp", "unix"}, "tcp") if err != nil { return fmt.Errorf("unable to expand target: %v", err) } - dstURL, err := url.Parse(targetURL) - if err != nil { - return fmt.Errorf("invalid TCP target %q: %v", target, err) + // For unix: targets, store the full "unix:/path" string as the forward address. + // For tcp: targets, extract the host:port from the parsed URL. + var fwdAddr string + if strings.HasPrefix(targetURL, "unix:") { + if proxyProtocol != 0 { + return fmt.Errorf("PROXY protocol is not supported with unix socket targets") + } + fwdAddr = targetURL + } else { + dstURL, err := url.Parse(targetURL) + if err != nil { + return fmt.Errorf("invalid TCP target %q: %v", target, err) + } + if dstURL.Port() == "" { + return fmt.Errorf("TCP target %q must include a port", target) + } + fwdAddr = dstURL.Host } if sc.IsServingWeb(srcPort, svcName) { @@ -1232,17 +1333,17 @@ func (e *serveEnv) applyTCPServe(sc *ipn.ServeConfig, dnsName string, srcType se // TODO: needs to account for multiple configs from foreground mode if svcName := tailcfg.AsServiceName(dnsName); svcName != "" { - sc.SetTCPForwardingForService(srcPort, dstURL.Host, terminateTLS, svcName, proxyProtocol, mds) + sc.SetTCPForwardingForService(srcPort, fwdAddr, terminateTLS, svcName, proxyProtocol, mds) return nil } // TODO: needs to account for multiple configs from foreground mode if svcName != "" { - sc.SetTCPForwardingForService(srcPort, dstURL.Host, terminateTLS, svcName, proxyProtocol, mds) + sc.SetTCPForwardingForService(srcPort, fwdAddr, terminateTLS, svcName, proxyProtocol, mds) return nil } - sc.SetTCPForwarding(srcPort, dstURL.Host, terminateTLS, proxyProtocol, dnsName) + sc.SetTCPForwarding(srcPort, fwdAddr, terminateTLS, proxyProtocol, dnsName) return nil } diff --git a/cmd/tailscale/cli/serve_v2_test.go b/cmd/tailscale/cli/serve_v2_test.go index 1d2a8ef86..b86c40139 100644 --- a/cmd/tailscale/cli/serve_v2_test.go +++ b/cmd/tailscale/cli/serve_v2_test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "reflect" "regexp" + "runtime" "slices" "strconv" "strings" @@ -40,6 +41,7 @@ func TestServeDevConfigMutations(t *testing.T) { name string steps []step initialState fakeLocalServeClient // use the zero value for empty config + skipOn []string // platforms on which to skip; GOOS values } // creaet a temporary directory for path-based destinations @@ -483,6 +485,63 @@ func TestServeDevConfigMutations(t *testing.T) { }, }, }, + { + name: "tcp_unix_socket", + steps: []step{{ + command: cmd("serve --tcp=3128 --bg unix:/var/run/app.sock"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 3128: { + TCPForward: "unix:/var/run/app.sock", + }, + }, + }, + }}, + skipOn: []string{"windows"}, + }, + { + name: "tls_terminated_tcp_unix_socket", + steps: []step{{ + command: cmd("serve --tls-terminated-tcp=443 --bg unix:/var/run/app.sock"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + TCPForward: "unix:/var/run/app.sock", + TerminateTLS: "foo.test.ts.net", + }, + }, + }, + }}, + skipOn: []string{"windows"}, + }, + { + name: "tcp_unix_socket_off", + steps: []step{ + { + command: cmd("serve --tcp=3128 --bg unix:/var/run/app.sock"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 3128: { + TCPForward: "unix:/var/run/app.sock", + }, + }, + }, + }, + { + command: cmd("serve --tcp=3128 off"), + want: &ipn.ServeConfig{}, + }, + }, + skipOn: []string{"windows"}, + }, + { + name: "tcp_unix_socket_proxy_protocol_rejected", + steps: []step{{ + command: cmd("serve --tcp=3128 --proxy-protocol=1 --bg unix:/var/run/app.sock"), + wantErr: anyErr(), + }}, + skipOn: []string{"windows"}, + }, { name: "tcp_off", steps: []step{ @@ -993,6 +1052,9 @@ func TestServeDevConfigMutations(t *testing.T) { for _, group := range groups { t.Run(group.name, func(t *testing.T) { + if slices.Contains(group.skipOn, runtime.GOOS) { + t.Skip("skipping on", runtime.GOOS) + } lc := group.initialState for i, st := range group.steps { var stderr bytes.Buffer @@ -1603,7 +1665,7 @@ func TestMessageForPort(t *testing.T) { expected: strings.Join([]string{ msgServeAvailable, "", - "|-- tcp://foo.test.ts.net:2200 (TLS over TCP)", + "|-- tcp://foo.test.ts.net:2200", "|-- tcp://100.101.101.101:2200", "|-- tcp://[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:2200", "|--> tcp://localhost:3000", @@ -2361,3 +2423,278 @@ func ptrToReadOnlySlice[T any](s []T) *views.Slice[T] { vs := views.SliceOf(s) return &vs } + +func writeTmpServeConfig(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "serve.json") + if err := os.WriteFile(p, []byte(body), 0600); err != nil { + t.Fatal(err) + } + return p +} + +// TestRunServeSetConfig covers set-config accepting both the declarative +// Services configuration file (with a "version" field) and the legacy raw +// ipn.ServeConfig format (no "version"), the latter applying services only and +// warning on stderr. +func TestRunServeSetConfig(t *testing.T) { + const fooSvc = tailcfg.ServiceName("svc:foo") + + t.Run("legacy_all_services_only", func(t *testing.T) { + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + var stdout, stderr bytes.Buffer + e := &serveEnv{lc: lc, allServices: true, testStdout: &stdout, testStderr: &stderr} + path := writeTmpServeConfig(t, `{"Services":{"svc:foo":{"TCP":{"443":{"HTTPS":true}}}}}`) + + if err := e.runServeSetConfig(context.Background(), []string{path}); err != nil { + t.Fatal(err) + } + if lc.setCount != 1 { + t.Fatalf("setCount = %d, want 1", lc.setCount) + } + svc := lc.config.Services[fooSvc] + if svc == nil || svc.TCP[443] == nil || !svc.TCP[443].HTTPS { + t.Errorf("svc:foo TCP/443 HTTPS not applied; got %+v", lc.config.Services) + } + if !slices.Contains(lc.prefs.AdvertiseServices, fooSvc.String()) { + t.Errorf("svc:foo not advertised; AdvertiseServices=%v", lc.prefs.AdvertiseServices) + } + if !strings.Contains(stderr.String(), "legacy raw serve config format") || + !strings.Contains(stderr.String(), serveConfigDocsURL) { + t.Errorf("missing legacy migration warning; stderr:\n%s", stderr.String()) + } + if strings.Contains(stderr.String(), "ignoring node-level fields") { + t.Errorf("unexpected dropped-fields warning; stderr:\n%s", stderr.String()) + } + if stdout.Len() != 0 { + t.Errorf("stdout must stay clean, got:\n%s", stdout.String()) + } + }) + + t.Run("legacy_drops_node_level_fields", func(t *testing.T) { + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + var stdout, stderr bytes.Buffer + e := &serveEnv{lc: lc, allServices: true, testStdout: &stdout, testStderr: &stderr} + path := writeTmpServeConfig(t, `{ + "TCP":{"443":{"HTTPS":true}}, + "Web":{"foo.test.ts.net:443":{"Handlers":{"/":{"Proxy":"http://127.0.0.1:3000"}}}}, + "AllowFunnel":{"foo.test.ts.net:443":true}, + "Services":{"svc:foo":{"Tun":true}} + }`) + + if err := e.runServeSetConfig(context.Background(), []string{path}); err != nil { + t.Fatal(err) + } + if svc := lc.config.Services[fooSvc]; svc == nil || !svc.Tun { + t.Errorf("svc:foo Tun not applied; got %+v", lc.config.Services) + } + // Fields are derived from the JSON dynamically and sorted. + if !strings.Contains(stderr.String(), "ignoring node-level fields not supported by set-config: AllowFunnel, TCP, Web") { + t.Errorf("missing/incorrect dropped-fields warning; stderr:\n%s", stderr.String()) + } + // Node-level content must not have leaked into the applied config. + if len(lc.config.TCP) != 0 || len(lc.config.Web) != 0 { + t.Errorf("node-level TCP/Web should not be applied; got TCP=%v Web=%v", lc.config.TCP, lc.config.Web) + } + if stdout.Len() != 0 { + t.Errorf("stdout must stay clean, got:\n%s", stdout.String()) + } + }) + + t.Run("versionless_new_format_errors", func(t *testing.T) { + // A Services config file (lowercase "services"/"endpoints") that omits + // the required "version" field must error, not be misread as a legacy + // raw ServeConfig and silently wipe the existing config. + existing := &ipn.ServeConfig{Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + fooSvc: {Tun: true}, + }} + lc := &fakeLocalServeClient{config: existing} + e := &serveEnv{lc: lc, allServices: true, testStdout: &bytes.Buffer{}, testStderr: &bytes.Buffer{}} + path := writeTmpServeConfig(t, `{"services":{"svc:foo":{"endpoints":{"tcp:443":"https://localhost:8000"}}}}`) + + err := e.runServeSetConfig(context.Background(), []string{path}) + if err == nil || !strings.Contains(err.Error(), "version") { + t.Fatalf("err = %v, want an error mentioning the missing version field", err) + } + if lc.setCount != 0 { + t.Errorf("setCount = %d, want 0 (existing config must not be wiped)", lc.setCount) + } + if lc.config.Services[fooSvc] == nil { + t.Errorf("existing svc:foo was wiped; got %+v", lc.config.Services) + } + }) + + t.Run("version_0_0_0_rejected", func(t *testing.T) { + // A file can never forge the internal LegacyVersion ("0.0.0") sentinel + // that LoadServicesConfig uses to wrap a legacy raw config: it is + // rejected as an unsupported version, and must not wipe existing config. + existing := &ipn.ServeConfig{Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + fooSvc: {Tun: true}, + }} + lc := &fakeLocalServeClient{config: existing} + e := &serveEnv{lc: lc, allServices: true, testStdout: &bytes.Buffer{}, testStderr: &bytes.Buffer{}} + path := writeTmpServeConfig(t, `{"version":"0.0.0","services":{}}`) + + err := e.runServeSetConfig(context.Background(), []string{path}) + if err == nil || !strings.Contains(err.Error(), `unsupported config file version "0.0.0"`) { + t.Fatalf("err = %v, want an 'unsupported config file version \"0.0.0\"' error", err) + } + if lc.setCount != 0 { + t.Errorf("setCount = %d, want 0 (existing config must not be wiped)", lc.setCount) + } + if lc.config.Services[fooSvc] == nil { + t.Errorf("existing svc:foo was wiped; got %+v", lc.config.Services) + } + }) + + t.Run("legacy_service_selects_one", func(t *testing.T) { + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + var stdout, stderr bytes.Buffer + e := &serveEnv{lc: lc, service: fooSvc, testStdout: &stdout, testStderr: &stderr} + path := writeTmpServeConfig(t, `{"Services":{"svc:foo":{"Tun":true},"svc:bar":{"Tun":true}}}`) + + if err := e.runServeSetConfig(context.Background(), []string{path}); err != nil { + t.Fatal(err) + } + if lc.config.Services[fooSvc] == nil { + t.Errorf("svc:foo not applied; got %+v", lc.config.Services) + } + if lc.config.Services[tailcfg.ServiceName("svc:bar")] != nil { + t.Errorf("svc:bar should not be applied with --service=svc:foo; got %+v", lc.config.Services) + } + }) + + t.Run("legacy_service_missing_errors", func(t *testing.T) { + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + e := &serveEnv{lc: lc, service: tailcfg.ServiceName("svc:missing"), testStdout: &bytes.Buffer{}, testStderr: &bytes.Buffer{}} + path := writeTmpServeConfig(t, `{"Services":{"svc:foo":{"Tun":true}}}`) + + err := e.runServeSetConfig(context.Background(), []string{path}) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("err = %v, want a 'not found' error", err) + } + if lc.setCount != 0 { + t.Errorf("setCount = %d, want 0", lc.setCount) + } + }) + + t.Run("both_flags_error", func(t *testing.T) { + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + e := &serveEnv{lc: lc, allServices: true, service: fooSvc, testStdout: &bytes.Buffer{}, testStderr: &bytes.Buffer{}} + err := e.runServeSetConfig(context.Background(), []string{"unused.json"}) + if err == nil || !strings.Contains(err.Error(), "cannot specify both") { + t.Fatalf("err = %v, want 'cannot specify both'", err) + } + }) + + t.Run("neither_flag_error", func(t *testing.T) { + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + e := &serveEnv{lc: lc, testStdout: &bytes.Buffer{}, testStderr: &bytes.Buffer{}} + err := e.runServeSetConfig(context.Background(), []string{"unused.json"}) + if err == nil || !strings.Contains(err.Error(), "must specify either") { + t.Fatalf("err = %v, want 'must specify either'", err) + } + }) + + t.Run("new_format_all_no_warning", func(t *testing.T) { + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + var stdout, stderr bytes.Buffer + e := &serveEnv{lc: lc, allServices: true, testStdout: &stdout, testStderr: &stderr} + path := writeTmpServeConfig(t, `{"version":"0.0.1","services":{"svc:foo":{"endpoints":{"tcp:443":"https://localhost:8000"}}}}`) + + if err := e.runServeSetConfig(context.Background(), []string{path}); err != nil { + t.Fatal(err) + } + if lc.setCount != 1 { + t.Fatalf("setCount = %d, want 1", lc.setCount) + } + if lc.config.Services[fooSvc] == nil { + t.Errorf("svc:foo not applied; got %+v", lc.config.Services) + } + if stderr.Len() != 0 { + t.Errorf("new format must not warn; stderr:\n%s", stderr.String()) + } + if stdout.Len() != 0 { + t.Errorf("stdout must stay clean, got:\n%s", stdout.String()) + } + }) + + t.Run("new_format_single_service_no_warning", func(t *testing.T) { + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + var stdout, stderr bytes.Buffer + e := &serveEnv{lc: lc, service: fooSvc, testStdout: &stdout, testStderr: &stderr} + path := writeTmpServeConfig(t, `{"version":"0.0.1","endpoints":{"tcp:443":"https://localhost:8000"}}`) + + if err := e.runServeSetConfig(context.Background(), []string{path}); err != nil { + t.Fatal(err) + } + if lc.config.Services[fooSvc] == nil { + t.Errorf("svc:foo not applied; got %+v", lc.config.Services) + } + if stderr.Len() != 0 { + t.Errorf("new format must not warn; stderr:\n%s", stderr.String()) + } + }) + + t.Run("http_over_unix_roundtrip", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping on windows") + } + + // set-config: apply HTTP-over-unix declarative config; then get-config + // should reproduce a target of "http://unix:/var/run/app.sock" without + // mangling it through host:port parsing. + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + var stdout, stderr bytes.Buffer + e := &serveEnv{lc: lc, service: fooSvc, testStdout: &stdout, testStderr: &stderr} + path := writeTmpServeConfig(t, `{"version":"0.0.1","endpoints":{"tcp:443":"http://unix:/var/run/app.sock"}}`) + + if err := e.runServeSetConfig(context.Background(), []string{path}); err != nil { + t.Fatalf("set-config: %v", err) + } + svc := lc.config.Services[fooSvc] + if svc == nil { + t.Fatalf("svc:foo not applied; got %+v", lc.config.Services) + } + if got := svc.Web["foo.test.ts.net:443"].Handlers["/"].Proxy; got != "unix:/var/run/app.sock" { + t.Errorf("Handler Proxy = %q, want %q", got, "unix:/var/run/app.sock") + } + if stderr.Len() != 0 { + t.Errorf("stderr must be empty; got:\n%s", stderr.String()) + } + + // Round-trip through get-config. + var gotStdout, gotStderr bytes.Buffer + g := &serveEnv{lc: lc, service: fooSvc, testStdout: &gotStdout, testStderr: &gotStderr} + if err := g.runServeGetConfig(context.Background(), nil); err != nil { + t.Fatalf("get-config: %v", err) + } + if !strings.Contains(gotStdout.String(), `"tcp:443": "http://unix:/var/run/app.sock"`) { + t.Errorf("get-config output missing http-over-unix target:\n%s", gotStdout.String()) + } + }) + + t.Run("https_over_unix_roundtrip", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping on windows") + } + + lc := &fakeLocalServeClient{config: &ipn.ServeConfig{}} + var stdout, stderr bytes.Buffer + e := &serveEnv{lc: lc, service: fooSvc, testStdout: &stdout, testStderr: &stderr} + path := writeTmpServeConfig(t, `{"version":"0.0.1","endpoints":{"tcp:443":"https://unix:/var/run/app.sock"}}`) + + if err := e.runServeSetConfig(context.Background(), []string{path}); err != nil { + t.Fatalf("set-config: %v", err) + } + + var gotStdout, gotStderr bytes.Buffer + g := &serveEnv{lc: lc, service: fooSvc, testStdout: &gotStdout, testStderr: &gotStderr} + if err := g.runServeGetConfig(context.Background(), nil); err != nil { + t.Fatalf("get-config: %v", err) + } + if !strings.Contains(gotStdout.String(), `"tcp:443": "https://unix:/var/run/app.sock"`) { + t.Errorf("get-config output missing https-over-unix target:\n%s", gotStdout.String()) + } + }) +} diff --git a/cmd/tailscale/cli/service.go b/cmd/tailscale/cli/service.go new file mode 100644 index 000000000..35df8b6a0 --- /dev/null +++ b/cmd/tailscale/cli/service.go @@ -0,0 +1,266 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "cmp" + "context" + "encoding/json" + "flag" + "fmt" + "slices" + "strings" + "text/tabwriter" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/ipproto" +) + +// serviceLister is the subset of [local.Client] that the service commands +// need. It is injected via the command's context (see [withServiceLister] / +// [serviceListerFromContext]) so tests can supply a mock. Outside of tests +// it is suplied by the package-level localClient var. +// +// NOTE(adrianosela): Could be replaced by a more generic way to mock +// local.Client e.g. https://github.com/tailscale/tailscale/issues/20164. +type serviceLister interface { + GetServices(ctx context.Context) (map[tailcfg.ServiceName]tailcfg.ServiceDetails, error) + Status(ctx context.Context) (*ipnstate.Status, error) +} + +type serviceListerCtxKey struct{} + +// withServiceLister returns a copy of ctx carrying sl, retrievable with +// [serviceListerFromContext]. +func withServiceLister(ctx context.Context, sl serviceLister) context.Context { + return context.WithValue(ctx, serviceListerCtxKey{}, sl) +} + +// serviceListerFromContext returns the [serviceLister] stored in ctx by +// [withServiceLister], or the package-level localClient if none was injected. +func serviceListerFromContext(ctx context.Context) serviceLister { + if sl, ok := ctx.Value(serviceListerCtxKey{}).(serviceLister); ok { + return sl + } + return &localClient +} + +const serviceListUsage = "tailscale service list" + +var serviceCmd = &ffcli.Command{ + Name: "service", + ShortHelp: "Interact with Tailscale Services", + ShortUsage: "tailscale service", + LongHelp: strings.TrimSpace(` +The 'tailscale service' command groups subcommands for Tailscale Services. + +A Tailscale Service is a virtual service with its own IP addresses. Which +Services this node can reach is determined by the tailnet's ACLs. Use the 'list' +subcommand to see the Services currently available to this node. +`), + UsageFunc: usageFuncNoDefaultValues, + Exec: func(context.Context, []string) error { return flag.ErrHelp }, + Subcommands: []*ffcli.Command{ + { + Name: "list", + ShortUsage: serviceListUsage, + ShortHelp: "List the Tailscale Services your node can access", + Exec: runServiceList, + FlagSet: func() *flag.FlagSet { + fs := newFlagSet("list") + fs.BoolVar(&serviceListArgs.json, "json", false, "output in JSON format") + return fs + }(), + }, + }, +} + +var serviceListArgs struct { + json bool +} + +// serviceListEntry decorates a [tailcfg.ServiceDetails] with the Service's +// MagicDNS hostname, both for the table's HOSTNAME column and so it is included +// in the JSON output. +type serviceListEntry struct { + tailcfg.ServiceDetails + Hostname string +} + +// runServiceList is the entry point for the "tailscale service list" command. +func runServiceList(ctx context.Context, args []string) error { + if len(args) != 0 { + return fmt.Errorf("usage: %s", serviceListUsage) + } + + lc := serviceListerFromContext(ctx) + + services, err := lc.GetServices(ctx) + if err != nil { + return err + } + + // We need the tailnet's MagicDNS suffix to build each Service's hostname. + st, err := lc.Status(ctx) + if err != nil { + return fixTailscaledConnectError(err) + } + var magicDNSSuffix string + if st.CurrentTailnet != nil { + magicDNSSuffix = st.CurrentTailnet.MagicDNSSuffix + } + + // Sort the services by name for stable output. + names := make([]tailcfg.ServiceName, 0, len(services)) + for name := range services { + names = append(names, name) + } + slices.Sort(names) + + entries := make([]serviceListEntry, 0, len(names)) + for _, name := range names { + svc := services[name] + entries = append(entries, serviceListEntry{ + ServiceDetails: svc, + Hostname: serviceHostname(name, magicDNSSuffix), + }) + } + + if serviceListArgs.json { + enc := json.NewEncoder(Stdout) + enc.SetIndent("", " ") + return enc.Encode(entries) + } + + if len(entries) == 0 { + fmt.Fprintln(Stdout, "No Tailscale Services are available to this node.") + return nil + } + + w := tabwriter.NewWriter(Stdout, 10, 5, 5, ' ', 0) + defer w.Flush() + fmt.Fprintf(w, "\n %s\t%s\t%s\t%s\t%s\t", "IP", "HOSTNAME", "DISPLAY NAME", "ENDPOINTS", "TYPE") + for _, e := range entries { + // Show a single IP, always the first in Addrs. If a tailnet has IPv4 + // disabled, the netmap only includes the v6 address, so the 0th index + // is the v6 address and that's what we show. + var ip string + if len(e.Addrs) > 0 { + ip = e.Addrs[0].String() + } + fmt.Fprintf(w, "\n %s\t%s\t%s\t%s\t%s\t", + cmp.Or(ip, "-"), + cmp.Or(e.Hostname, "-"), + cmp.Or(e.DisplayName, "-"), + joinStringers(e.Ports, "-"), + serviceActionTypes(e.ServiceDetails), + ) + } + fmt.Fprintln(w) + return nil +} + +// wellKnownPortActions maps well-known TCP ports to the service action type +// they conventionally correspond to. It is used to infer actions for Services +// that don't carry explicit ones. +// +// TODO(adrianosela): move this to tailcfg as +// InferredServiceActionForPort(p uint16) ServiceActionType. +var wellKnownPortActions = map[uint16]tailcfg.ServiceActionType{ + 22: tailcfg.ServiceActionTypeSSH, + 80: tailcfg.ServiceActionTypeHTTP, + 443: tailcfg.ServiceActionTypeHTTP, + 1433: tailcfg.ServiceActionTypeMSSQL, + 3306: tailcfg.ServiceActionTypeMySQL, + 3389: tailcfg.ServiceActionTypeRDP, + 5432: tailcfg.ServiceActionTypePostgreSQL, + 5900: tailcfg.ServiceActionTypeVNC, + 6443: tailcfg.ServiceActionTypeKubernetes, + 9200: tailcfg.ServiceActionTypeElasticSearch, + 26257: tailcfg.ServiceActionTypeCockroachDB, + 27017: tailcfg.ServiceActionTypeMongoDB, +} + +// maxNamedTypes is the most action types serviceActionTypes names before +// summarizing the remainder as "N other(s)". +const maxNamedTypes = 2 + +// serviceActionTypes renders a Service's action identifiers (types) for the +// TYPE column. Types are deduplicated; it names at most [maxNamedTypes] of them +// and summarizes any remainder, e.g. "-" for none, "http" for one, "http, ssh" +// for two, and "http, ssh, 2 others" for more. +// +// Explicit actions are shown by type. When a Service carries no explicit +// actions, types are inferred from well-known TCP ports (see +// [wellKnownPortActions]). +func serviceActionTypes(svc tailcfg.ServiceDetails) string { + var raw []tailcfg.ServiceActionType + if len(svc.Actions) > 0 { + for _, a := range svc.Actions { + raw = append(raw, a.Type) + } + } else { + for _, ppr := range svc.Ports { + // Only single TCP ports map to a well-known action. + if ppr.Proto != 0 && ppr.Proto != int(ipproto.TCP) { + continue + } + if ppr.Ports.First != ppr.Ports.Last { + continue + } + if t, ok := wellKnownPortActions[ppr.Ports.First]; ok { + raw = append(raw, t) + } + } + } + + // Deduplicate, preserving first-seen order. + var types []string + seen := make(map[tailcfg.ServiceActionType]bool) + for _, t := range raw { + if seen[t] { + continue + } + seen[t] = true + types = append(types, string(t)) + } + + if len(types) == 0 { + return "-" + } + if len(types) <= maxNamedTypes { + return strings.Join(types, ", ") + } + extra := len(types) - maxNamedTypes + noun := "others" + if extra == 1 { + noun = "other" + } + return fmt.Sprintf("%s, %d %s", strings.Join(types[:maxNamedTypes], ", "), extra, noun) +} + +// serviceHostname returns the MagicDNS hostname for a Service, of the form +// ".". It returns "" if the name or suffix is missing. +func serviceHostname(name tailcfg.ServiceName, magicDNSSuffix string) string { + bare := name.WithoutPrefix() + if bare == "" || magicDNSSuffix == "" { + return "" + } + return bare + "." + strings.Trim(magicDNSSuffix, ".") +} + +// joinStringers renders a slice of fmt.Stringer-like values as a +// comma-separated string, returning empty if the slice is empty. +func joinStringers[T fmt.Stringer](vals []T, empty string) string { + if len(vals) == 0 { + return empty + } + strs := make([]string, len(vals)) + for i, v := range vals { + strs[i] = v.String() + } + return strings.Join(strs, ", ") +} diff --git a/cmd/tailscale/cli/service_test.go b/cmd/tailscale/cli/service_test.go new file mode 100644 index 000000000..f289ce97c --- /dev/null +++ b/cmd/tailscale/cli/service_test.go @@ -0,0 +1,357 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/netip" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/tstest" +) + +// fakeServiceLister is a test-only [serviceLister] injected via +// [withServiceLister] in place of the real localClient. +type fakeServiceLister struct { + services map[tailcfg.ServiceName]tailcfg.ServiceDetails + servicesErr error + status *ipnstate.Status + statusErr error +} + +func (f fakeServiceLister) GetServices(context.Context) (map[tailcfg.ServiceName]tailcfg.ServiceDetails, error) { + return f.services, f.servicesErr +} + +func (f fakeServiceLister) Status(context.Context) (*ipnstate.Status, error) { + return f.status, f.statusErr +} + +// mustParsePorts parses ProtoPortRanges or fails the test. +func mustParsePorts(t *testing.T, ports ...string) []tailcfg.ProtoPortRange { + t.Helper() + ppr, err := tailcfg.ParseProtoPortRanges(ports) + if err != nil { + t.Fatalf("ParseProtoPortRanges(%q): %v", ports, err) + } + return ppr +} + +// statusWithSuffix returns a Status carrying the given MagicDNS suffix. +func statusWithSuffix(suffix string) *ipnstate.Status { + return &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: suffix}, + } +} + +// runList runs the list command with the given lister, capturing stdout. +func runList(t *testing.T, lc serviceLister, jsonOut bool) string { + t.Helper() + var buf strings.Builder + tstest.Replace[io.Writer](t, &Stdout, &buf) + tstest.Replace(t, &serviceListArgs.json, jsonOut) + + ctx := withServiceLister(context.Background(), lc) + if err := runServiceList(ctx, nil); err != nil { + t.Fatalf("runServiceList: %v", err) + } + return buf.String() +} + +func TestServiceListTable(t *testing.T) { + lc := fakeServiceLister{ + status: statusWithSuffix("ts-tailnet.ts.net."), + services: map[tailcfg.ServiceName]tailcfg.ServiceDetails{ + // Intentionally out of order to exercise sorting. + "svc:web": { + Name: "svc:web", + DisplayName: "Web", + Addrs: []netip.Addr{netip.MustParseAddr("100.80.0.2"), netip.MustParseAddr("fd7a::2")}, + Ports: mustParsePorts(t, "tcp:443"), + // Two actions render comma-separated. + Actions: []tailcfg.ServiceAction{ + {Type: tailcfg.ServiceActionTypeHTTP, Port: 443}, + {Type: tailcfg.ServiceActionTypeSSH, Port: 22}, + }, + }, + "svc:db": { + Name: "svc:db", + Addrs: []netip.Addr{netip.MustParseAddr("100.80.0.1")}, + Ports: mustParsePorts(t, "tcp:5432"), + // No explicit actions, but tcp:5432 infers "postgresql". + }, + }, + } + + out := runList(t, lc, false) + + for _, h := range []string{"IP", "HOSTNAME", "DISPLAY NAME", "ENDPOINTS", "TYPE"} { + if !strings.Contains(out, h) { + t.Errorf("output missing header %q\n%s", h, out) + } + } + + // Services are sorted by name, so svc:db's row precedes svc:web's. + if i, j := strings.Index(out, "db.ts-tailnet.ts.net"), strings.Index(out, "web.ts-tailnet.ts.net"); i < 0 || j < 0 || i > j { + t.Errorf("services not sorted by name (db=%d, web=%d)\n%s", i, j, out) + } + + // IP is the first address only; the v6 address is not shown. + if !strings.Contains(out, "100.80.0.2") { + t.Errorf("output missing svc:web first IP\n%s", out) + } + if strings.Contains(out, "fd7a::2") { + t.Errorf("output should show only the first IP, not the v6 address\n%s", out) + } + + // Endpoints come from Ports. + if !strings.Contains(out, "tcp:443") || !strings.Contains(out, "tcp:5432") { + t.Errorf("output missing endpoints\n%s", out) + } + + // svc:web has two explicit actions, rendered comma-separated. + if !strings.Contains(out, "http, ssh") { + t.Errorf("output missing svc:web actions %q\n%s", "http, ssh", out) + } + + // svc:db has no explicit actions but a well-known port (5432), so its + // type is inferred. + if !strings.Contains(out, "postgresql") { + t.Errorf("output missing svc:db inferred type %q\n%s", "postgresql", out) + } +} + +func TestServiceActionTypes(t *testing.T) { + tests := []struct { + name string + svc tailcfg.ServiceDetails + want string + }{ + { + "explicit one", + tailcfg.ServiceDetails{Actions: []tailcfg.ServiceAction{{Type: tailcfg.ServiceActionTypeSSH}}}, + "ssh", + }, + { + "explicit two", + tailcfg.ServiceDetails{Actions: []tailcfg.ServiceAction{ + {Type: tailcfg.ServiceActionTypeHTTP}, + {Type: tailcfg.ServiceActionTypeSSH}, + }}, + "http, ssh", + }, + { + // More than maxNamedTypes: name two, summarize the rest. + "explicit three summarizes one other", + tailcfg.ServiceDetails{Actions: []tailcfg.ServiceAction{ + {Type: tailcfg.ServiceActionTypeHTTP}, + {Type: tailcfg.ServiceActionTypeSSH}, + {Type: tailcfg.ServiceActionTypeMySQL}, + }}, + "http, ssh, 1 other", + }, + { + "explicit four summarizes multiple others", + tailcfg.ServiceDetails{Actions: []tailcfg.ServiceAction{ + {Type: tailcfg.ServiceActionTypeHTTP}, + {Type: tailcfg.ServiceActionTypeSSH}, + {Type: tailcfg.ServiceActionTypeMySQL}, + {Type: tailcfg.ServiceActionTypeRDP}, + }}, + "http, ssh, 2 others", + }, + { + // Duplicate types collapse before counting. + "explicit dedupes by type", + tailcfg.ServiceDetails{Actions: []tailcfg.ServiceAction{ + {Type: tailcfg.ServiceActionTypeHTTP, Port: 80}, + {Type: tailcfg.ServiceActionTypeHTTP, Port: 443}, + {Type: tailcfg.ServiceActionTypeSSH, Port: 22}, + }}, + "http, ssh", + }, + { + // Explicit actions win; ports are not consulted for inference. + "explicit beats inference", + tailcfg.ServiceDetails{ + Actions: []tailcfg.ServiceAction{{Type: tailcfg.ServiceActionTypeTCP}}, + Ports: mustParsePorts(t, "tcp:22"), + }, + "tcp", + }, + { + "inferred from well-known port", + tailcfg.ServiceDetails{Ports: mustParsePorts(t, "tcp:22")}, + "ssh", + }, + { + "inferred from multiple well-known ports", + tailcfg.ServiceDetails{Ports: mustParsePorts(t, "tcp:80", "tcp:5432")}, + "http, postgresql", + }, + { + // 443 and 80 both map to http; it is only listed once. + "inferred dedupes by action type", + tailcfg.ServiceDetails{Ports: mustParsePorts(t, "tcp:80", "tcp:443")}, + "http", + }, + { + "no actions and unknown port", + tailcfg.ServiceDetails{Ports: mustParsePorts(t, "tcp:12345")}, + "-", + }, + { + // A port range is not a single well-known port, so nothing is inferred. + "port range is not inferred", + tailcfg.ServiceDetails{Ports: mustParsePorts(t, "tcp:20-30")}, + "-", + }, + {"nothing at all", tailcfg.ServiceDetails{}, "-"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := serviceActionTypes(tt.svc); got != tt.want { + t.Errorf("serviceActionTypes() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestServiceListIPv6Only verifies that when a Service only has a v6 address +// (e.g. a tailnet with IPv4 disabled), the v6 address is shown as the IP. +func TestServiceListIPv6Only(t *testing.T) { + lc := fakeServiceLister{ + status: statusWithSuffix("ts-tailnet.ts.net"), + services: map[tailcfg.ServiceName]tailcfg.ServiceDetails{ + "svc:v6": { + Name: "svc:v6", + Addrs: []netip.Addr{netip.MustParseAddr("fd7a::9")}, + Ports: mustParsePorts(t, "tcp:80"), + }, + }, + } + + out := runList(t, lc, false) + if !strings.Contains(out, "fd7a::9") { + t.Errorf("expected v6 address as IP\n%s", out) + } +} + +func TestServiceListJSON(t *testing.T) { + lc := fakeServiceLister{ + status: statusWithSuffix("ts-tailnet.ts.net"), + services: map[tailcfg.ServiceName]tailcfg.ServiceDetails{ + // Intentionally out of order to exercise sorting, and svc:web has + // both v4 and v6 addresses to confirm the JSON carries the full + // address list (unlike the table, which shows only the first). + "svc:web": { + Name: "svc:web", + DisplayName: "Web", + Addrs: []netip.Addr{netip.MustParseAddr("100.80.0.2"), netip.MustParseAddr("fd7a::2")}, + Ports: mustParsePorts(t, "tcp:443", "tcp:8443"), + }, + "svc:db": { + Name: "svc:db", + Addrs: []netip.Addr{netip.MustParseAddr("100.80.0.1")}, + Ports: mustParsePorts(t, "tcp:5432"), + }, + }, + } + + out := runList(t, lc, true) + + var got []serviceListEntry + if err := json.Unmarshal([]byte(out), &got); err != nil { + t.Fatalf("unmarshal JSON output: %v\n%s", err, out) + } + + // Entries are sorted by name and each embedded ServiceDetails (including + // the full Addrs and Ports) round-trips, decorated with the hostname. + want := []serviceListEntry{ + { + ServiceDetails: tailcfg.ServiceDetails{ + Name: "svc:db", + Addrs: []netip.Addr{netip.MustParseAddr("100.80.0.1")}, + Ports: mustParsePorts(t, "tcp:5432"), + }, + Hostname: "db.ts-tailnet.ts.net", + }, + { + ServiceDetails: tailcfg.ServiceDetails{ + Name: "svc:web", + DisplayName: "Web", + Addrs: []netip.Addr{netip.MustParseAddr("100.80.0.2"), netip.MustParseAddr("fd7a::2")}, + Ports: mustParsePorts(t, "tcp:443", "tcp:8443"), + }, + Hostname: "web.ts-tailnet.ts.net", + }, + } + // netip.Addr has unexported fields but is comparable, so compare it by ==. + if diff := cmp.Diff(want, got, cmpopts.EquateComparable(netip.Addr{})); diff != "" { + t.Errorf("JSON output mismatch (-want +got):\n%s\nraw:\n%s", diff, out) + } +} + +func TestServiceListEmpty(t *testing.T) { + lc := fakeServiceLister{ + status: statusWithSuffix("ts-tailnet.ts.net"), + services: map[tailcfg.ServiceName]tailcfg.ServiceDetails{}, + } + out := runList(t, lc, false) + if !strings.Contains(out, "No Tailscale Services are available") { + t.Errorf("expected empty message, got\n%s", out) + } +} + +func TestServiceListGetServicesError(t *testing.T) { + wantErr := errors.New("boom") + lc := fakeServiceLister{servicesErr: wantErr} + + ctx := withServiceLister(context.Background(), lc) + err := runServiceList(ctx, nil) + if !errors.Is(err, wantErr) { + t.Errorf("err = %v, want %v", err, wantErr) + } +} + +func TestServiceListRejectsArgs(t *testing.T) { + ctx := withServiceLister(context.Background(), fakeServiceLister{}) + if err := runServiceList(ctx, []string{"unexpected"}); err == nil { + t.Error("expected error for extra args, got nil") + } +} + +func TestServiceHostname(t *testing.T) { + tests := []struct { + name tailcfg.ServiceName + suffix string + want string + }{ + {"svc:web", "ts-tailnet.ts.net", "web.ts-tailnet.ts.net"}, + {"svc:web", "ts-tailnet.ts.net.", "web.ts-tailnet.ts.net"}, // trailing dot trimmed + {"svc:web", "", ""}, // no suffix + {"not-a-service", "ts-tailnet.ts.net", ""}, // invalid name (no svc: prefix) + } + for _, tt := range tests { + if got := serviceHostname(tt.name, tt.suffix); got != tt.want { + t.Errorf("serviceHostname(%q, %q) = %q, want %q", tt.name, tt.suffix, got, tt.want) + } + } +} + +// TestServiceListerFromContextDefault verifies that without injection the +// real localClient is returned. +func TestServiceListerFromContextDefault(t *testing.T) { + if got := serviceListerFromContext(context.Background()); got != &localClient { + t.Errorf("serviceListerFromContext default = %v, want &localClient", got) + } +} diff --git a/cmd/tailscale/cli/set.go b/cmd/tailscale/cli/set.go index 6fd4b09ad..95ee35bb1 100644 --- a/cmd/tailscale/cli/set.go +++ b/cmd/tailscale/cli/set.go @@ -62,6 +62,7 @@ type setArgsT struct { updateCheck bool updateApply bool reportPosture bool + remoteConfig bool snat bool statefulFiltering bool sync bool @@ -77,7 +78,7 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet { setf.BoolVar(&setArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes") setf.BoolVar(&setArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel") setf.StringVar(&setArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP, base name, or auto:any) for internet traffic, or empty string to not use an exit node") - setf.BoolVar(&setArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node") + setf.BoolVar(&setArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "allow direct access to the local network when routing traffic via an exit node") setf.BoolVar(&setArgs.shieldsUp, "shields-up", false, "don't allow incoming connections") setf.BoolVar(&setArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy") setf.StringVar(&setArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS") @@ -88,6 +89,7 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet { setf.BoolVar(&setArgs.updateApply, "auto-update", false, "automatically update to the latest available version") setf.BoolVar(&setArgs.reportPosture, "report-posture", false, "allow management plane to gather device posture information") setf.BoolVar(&setArgs.runWebClient, "webclient", false, "expose the web interface for managing this node over Tailscale at port 5252") + setf.BoolVar(&setArgs.remoteConfig, "remote-config", false, hidden+"delegate FULL remote control of this node's prefs and LocalAPI to the tailnet admin, bypassing Tailscale's per-feature double opt-in; only use when the tailnet admin owns or is fully trusted with this machine") setf.BoolVar(&setArgs.sync, "sync", false, hidden+"actively sync configuration from the control plane (set to false only for network failure testing)") setf.StringVar(&setArgs.relayServerPort, "relay-server-port", "", "UDP port number (0 will pick a random unused port) for the relay server to bind to, on all interfaces, or empty string to disable relay server functionality") setf.StringVar(&setArgs.relayServerStaticEndpoints, "relay-server-static-endpoints", "", "static IP:port endpoints to advertise as candidates for relay connections (comma-separated, e.g. \"[2001:db8::1]:40000,192.0.2.1:40000\") or empty string to not advertise any static endpoints") @@ -163,6 +165,7 @@ func runSet(ctx context.Context, args []string) (retErr error) { Advertise: setArgs.advertiseConnector, }, PostureChecking: setArgs.reportPosture, + RemoteConfig: setArgs.remoteConfig, NoStatefulFiltering: opt.NewBool(!setArgs.statefulFiltering), }, } diff --git a/cmd/tailscale/cli/ssh.go b/cmd/tailscale/cli/ssh.go index 9efab8cf7..eb1ce37f8 100644 --- a/cmd/tailscale/cli/ssh.go +++ b/cmd/tailscale/cli/ssh.go @@ -11,7 +11,6 @@ import ( "log" "net/netip" "os" - "os/user" "path/filepath" "runtime" "slices" @@ -59,11 +58,7 @@ func runSSH(ctx context.Context, args []string) error { username, host, ok := strings.Cut(arg, "@") if !ok { host = arg - lu, err := user.Current() - if err != nil { - return nil - } - username = lu.Username + username = "" } st, err := localClient.Status(ctx) @@ -146,7 +141,11 @@ func runSSH(ctx context.Context, args []string) error { // to use a different one, we'll later be making stock ssh // work well by default too. (doing things like automatically // setting known_hosts, etc) - argv = append(argv, username+"@"+hostForSSH) + if username == "" { + argv = append(argv, hostForSSH) + } else { + argv = append(argv, username+"@"+hostForSSH) + } argv = append(argv, argRest...) diff --git a/cmd/tailscale/cli/status.go b/cmd/tailscale/cli/status.go index 9ce4debda..a058a2d3c 100644 --- a/cmd/tailscale/cli/status.go +++ b/cmd/tailscale/cli/status.go @@ -54,7 +54,7 @@ https://github.com/tailscale/tailscale/blob/main/ipn/ipnstate/ipnstate.go fs.BoolVar(&statusArgs.self, "self", true, "show status of local machine") fs.BoolVar(&statusArgs.peers, "peers", true, "show status of peers") fs.StringVar(&statusArgs.listen, "listen", "127.0.0.1:8384", "listen address for web mode; use port 0 for automatic") - fs.BoolVar(&statusArgs.browser, "browser", true, "Open a browser in web mode") + fs.BoolVar(&statusArgs.browser, "browser", true, "open a browser in web mode") fs.BoolVar(&statusArgs.header, "header", false, "show column headers in table format") return fs })(), diff --git a/cmd/tailscale/cli/systray.go b/cmd/tailscale/cli/systray.go index ca0840fe9..07de5c786 100644 --- a/cmd/tailscale/cli/systray.go +++ b/cmd/tailscale/cli/systray.go @@ -7,6 +7,7 @@ package cli import ( "context" + "flag" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/client/systray" @@ -17,10 +18,20 @@ var systrayCmd = &ffcli.Command{ ShortUsage: "tailscale systray", ShortHelp: "Run a systray application to manage Tailscale", LongHelp: "Run a systray application to manage Tailscale.", - Exec: runSystray, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("systray") + fs.StringVar(&systrayArgs.theme, "theme", "dark", "color theme for Tailscale icon: dark, dark:nobg, light, light:nobg") + return fs + })(), + Exec: runSystray, +} + +var systrayArgs struct { + theme string } func runSystray(ctx context.Context, _ []string) error { + systray.SetTheme(systrayArgs.theme) new(systray.Menu).Run(&localClient) return nil } diff --git a/cmd/tailscale/cli/network-lock.go b/cmd/tailscale/cli/tailnet-lock.go similarity index 83% rename from cmd/tailscale/cli/network-lock.go rename to cmd/tailscale/cli/tailnet-lock.go index 4febd56a9..2a9331954 100644 --- a/cmd/tailscale/cli/network-lock.go +++ b/cmd/tailscale/cli/tailnet-lock.go @@ -22,7 +22,8 @@ import ( "github.com/mattn/go-isatty" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/cmd/tailscale/cli/jsonoutput" + + "tailscale.com/cmd/tailscale/jsonoutput" "tailscale.com/ipn/ipnstate" "tailscale.com/tka" "tailscale.com/tsconst" @@ -32,30 +33,30 @@ import ( ) func init() { - maybeNetlockCmd = func() *ffcli.Command { return netlockCmd } + maybeTailnetLockCmd = func() *ffcli.Command { return tailnetLockCmd } } -var netlockCmd = &ffcli.Command{ +var tailnetLockCmd = &ffcli.Command{ Name: "lock", ShortUsage: "tailscale lock [arguments...]", ShortHelp: "Manage tailnet lock", LongHelp: "Manage tailnet lock", Subcommands: []*ffcli.Command{ - nlInitCmd, - nlStatusCmd, - nlAddCmd, - nlRemoveCmd, - nlSignCmd, - nlDisableCmd, - nlDisablementKDFCmd, - nlLogCmd, - nlLocalDisableCmd, - nlRevokeKeysCmd, + tlInitCmd, + tlStatusCmd, + tlAddCmd, + tlRemoveCmd, + tlSignCmd, + tlDisableCmd, + tlDisablementKDFCmd, + tlLogCmd, + tlLocalDisableCmd, + tlRevokeKeysCmd, }, - Exec: runNetworkLockNoSubcommand, + Exec: runTailnetLockNoSubcommand, } -func runNetworkLockNoSubcommand(ctx context.Context, args []string) error { +func runTailnetLockNoSubcommand(ctx context.Context, args []string) error { // Detect & handle the deprecated command 'lock tskey-wrap'. if len(args) >= 2 && args[0] == "tskey-wrap" { return runTskeyWrapCmd(ctx, args[1:]) @@ -64,7 +65,7 @@ func runNetworkLockNoSubcommand(ctx context.Context, args []string) error { return fmt.Errorf("tailscale lock: unknown subcommand: %s", args[0]) } - return runNetworkLockStatus(ctx, args) + return runTailnetLockStatus(ctx, args) } var nlInitArgs struct { @@ -73,7 +74,7 @@ var nlInitArgs struct { confirm bool } -var nlInitCmd = &ffcli.Command{ +var tlInitCmd = &ffcli.Command{ Name: "init", ShortUsage: "tailscale lock init [--gen-disablement-for-support] --gen-disablements N ...", ShortHelp: "Initialize tailnet lock", @@ -98,7 +99,7 @@ will be generated and transmitted to Tailscale, which support can use to disable tailnet lock. We recommend setting this flag. `), - Exec: runNetworkLockInit, + Exec: runTailnetLockInit, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock init") fs.IntVar(&nlInitArgs.numDisablements, "gen-disablements", 1, "number of disablement secrets to generate") @@ -108,8 +109,8 @@ tailnet lock. We recommend setting this flag. })(), } -func runNetworkLockInit(ctx context.Context, args []string) error { - st, err := localClient.NetworkLockStatus(ctx) +func runTailnetLockInit(ctx context.Context, args []string) error { + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } @@ -118,7 +119,7 @@ func runNetworkLockInit(ctx context.Context, args []string) error { } // Parse initially-trusted keys & disablement values. - keys, disablementValues, err := parseNLArgs(args, true, true) + keys, disablementValues, err := parseTLArgs(args, true, true) if err != nil { return err } @@ -183,9 +184,9 @@ func runNetworkLockInit(ctx context.Context, args []string) error { fmt.Fprintln(&successMsg, "A disablement secret for Tailscale support has been generated and transmitted to Tailscale.") } - // The state returned by NetworkLockInit likely doesn't contain the initialized state, + // The state returned by TailnetLockInit likely doesn't contain the initialized state, // because that has to tick through from netmaps. - if _, err := localClient.NetworkLockInit(ctx, keys, disablementValues, supportDisablement); err != nil { + if _, err := localClient.TailnetLockInit(ctx, keys, disablementValues, supportDisablement); err != nil { return err } @@ -195,14 +196,14 @@ func runNetworkLockInit(ctx context.Context, args []string) error { } var nlStatusArgs struct { - json jsonoutput.JSONSchemaVersion + json jsonoutput.SchemaVersion } -var nlStatusCmd = &ffcli.Command{ +var tlStatusCmd = &ffcli.Command{ Name: "status", ShortUsage: "tailscale lock status", ShortHelp: "Output the state of tailnet lock", - Exec: runNetworkLockStatus, + Exec: runTailnetLockStatus, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock status") fs.Var(&nlStatusArgs.json, "json", "output in JSON format") @@ -210,21 +211,21 @@ var nlStatusCmd = &ffcli.Command{ })(), } -func runNetworkLockStatus(ctx context.Context, args []string) error { +func runTailnetLockStatus(ctx context.Context, args []string) error { if len(args) > 0 { return fmt.Errorf("tailscale lock status: unexpected argument") } - st, err := localClient.NetworkLockStatus(ctx) + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } if nlStatusArgs.json.IsSet { - if nlStatusArgs.json.Value == 1 { - return jsonoutput.PrintNetworkLockStatusJSONV1(os.Stdout, st) + if nlStatusArgs.json.Version == 1 { + return jsonoutput.PrintTailnetLockStatusJSONV1(os.Stdout, st) } else { - return fmt.Errorf("unrecognised version: %d", nlStatusArgs.json.Value) + return fmt.Errorf("unrecognised version: %d", nlStatusArgs.json.Version) } } @@ -301,22 +302,22 @@ func runNetworkLockStatus(ctx context.Context, args []string) error { return nil } -var nlAddCmd = &ffcli.Command{ +var tlAddCmd = &ffcli.Command{ Name: "add", ShortUsage: "tailscale lock add ...", ShortHelp: "Add one or more trusted signing keys to tailnet lock", - Exec: runNetworkLockAdd, + Exec: runTailnetLockAdd, } var nlRemoveArgs struct { resign bool } -var nlRemoveCmd = &ffcli.Command{ +var tlRemoveCmd = &ffcli.Command{ Name: "remove", ShortUsage: "tailscale lock remove [--re-sign=false] ...", ShortHelp: "Remove one or more trusted signing keys from tailnet lock", - Exec: runNetworkLockRemove, + Exec: runTailnetLockRemove, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock remove") fs.BoolVar(&nlRemoveArgs.resign, "re-sign", true, "resign signatures which would be invalidated by removal of trusted signing keys") @@ -324,15 +325,15 @@ var nlRemoveCmd = &ffcli.Command{ })(), } -func runNetworkLockRemove(ctx context.Context, args []string) error { - removeKeys, _, err := parseNLArgs(args, true, false) +func runTailnetLockRemove(ctx context.Context, args []string) error { + removeKeys, _, err := parseTLArgs(args, true, false) if err != nil { return err } if len(removeKeys) == 0 { return fmt.Errorf("missing argument, expected one or more tailnet lock keys") } - st, err := localClient.NetworkLockStatus(ctx) + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } @@ -359,7 +360,7 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { // Resign affected signatures for each of the keys we are removing. for _, k := range removeKeys { kID, _ := k.ID() // err already checked above - sigs, err := localClient.NetworkLockAffectedSigs(ctx, kID) + sigs, err := localClient.TailnetLockAffectedSigs(ctx, kID) if err != nil { return fmt.Errorf("affected sigs for key %X: %w", kID, err) } @@ -374,10 +375,10 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { return fmt.Errorf("failed decoding pubkey for signature: %w", err) } - // Safety: NetworkLockAffectedSigs() verifies all signatures before + // Safety: TailnetLockAffectedSigs() verifies all signatures before // successfully returning. rotationKey, _ := sig.UnverifiedWrappingPublic() - if err := localClient.NetworkLockSign(ctx, nodeKey, []byte(rotationKey)); err != nil { + if err := localClient.TailnetLockSign(ctx, nodeKey, []byte(rotationKey)); err != nil { return fmt.Errorf("failed to sign %v: %w", nodeKey, err) } } @@ -386,7 +387,7 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { if isatty.IsTerminal(os.Stdout.Fd()) { fmt.Printf(`Warning Removal of a signing key(s) without resigning nodes (--re-sign=false) -will cause any nodes signed by the the given key(s) to be locked out +will cause any nodes signed by the given key(s) to be locked out of the Tailscale network. Proceed with caution. `) if !prompt.YesNo("Are you sure you want to remove the signing key(s)?", true) { @@ -396,10 +397,10 @@ of the Tailscale network. Proceed with caution. } } - return localClient.NetworkLockModify(ctx, nil, removeKeys) + return localClient.TailnetLockModify(ctx, nil, removeKeys) } -// parseNLArgs parses a slice of strings into slices of tka.Key & disablement +// parseTLArgs parses a slice of strings into slices of tka.Key & disablement // values/secrets. // The keys encoded in args should be specified using their key.NLPublic.MarshalText // representation with an optional '?' suffix. @@ -408,7 +409,7 @@ of the Tailscale network. Proceed with caution. // // If any element could not be parsed, // a nil slice is returned along with an appropriate error. -func parseNLArgs(args []string, parseKeys, parseDisablements bool) (keys []tka.Key, disablements [][]byte, err error) { +func parseTLArgs(args []string, parseKeys, parseDisablements bool) (keys []tka.Key, disablements [][]byte, err error) { for i, a := range args { if parseDisablements && (strings.HasPrefix(a, "disablement:") || strings.HasPrefix(a, "disablement-secret:")) { b, err := hex.DecodeString(a[strings.Index(a, ":")+1:]) @@ -446,8 +447,8 @@ func parseNLArgs(args []string, parseKeys, parseDisablements bool) (keys []tka.K return keys, disablements, nil } -func runNetworkLockAdd(ctx context.Context, addArgs []string) error { - addKeys, _, err := parseNLArgs(addArgs, true, false) +func runTailnetLockAdd(ctx context.Context, addArgs []string) error { + addKeys, _, err := parseTLArgs(addArgs, true, false) if err != nil { return err } @@ -455,7 +456,7 @@ func runNetworkLockAdd(ctx context.Context, addArgs []string) error { return fmt.Errorf("missing argument, expected one or more tailnet lock keys") } - st, err := localClient.NetworkLockStatus(ctx) + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } @@ -463,13 +464,13 @@ func runNetworkLockAdd(ctx context.Context, addArgs []string) error { return errors.New("tailnet lock is not enabled") } - if err := localClient.NetworkLockModify(ctx, addKeys, nil); err != nil { + if err := localClient.TailnetLockModify(ctx, addKeys, nil); err != nil { return err } return nil } -var nlSignCmd = &ffcli.Command{ +var tlSignCmd = &ffcli.Command{ Name: "sign", ShortUsage: "tailscale lock sign []\ntailscale lock sign ", ShortHelp: "Sign a node or pre-approved auth key", @@ -481,10 +482,10 @@ var nlSignCmd = &ffcli.Command{ If any of the key arguments begin with "file:", the key is retrieved from the file at the path specified in the argument suffix.`, - Exec: runNetworkLockSign, + Exec: runTailnetLockSign, } -func runNetworkLockSign(ctx context.Context, args []string) error { +func runTailnetLockSign(ctx context.Context, args []string) error { // If any of the arguments start with "file:", replace that argument // with the contents of the file. We do this early, before the check // to see if the first argument is an auth key. @@ -519,7 +520,7 @@ func runNetworkLockSign(ctx context.Context, args []string) error { } } - err := localClient.NetworkLockSign(ctx, nodeKey, []byte(rotationKey.Verifier())) + err := localClient.TailnetLockSign(ctx, nodeKey, []byte(rotationKey.Verifier())) // Provide a better help message for when someone clicks through the signing flow // on the wrong device. if err != nil && strings.Contains(err.Error(), tsconst.TailnetLockNotTrustedMsg) { @@ -531,7 +532,7 @@ func runNetworkLockSign(ctx context.Context, args []string) error { return err } -var nlDisableCmd = &ffcli.Command{ +var tlDisableCmd = &ffcli.Command{ Name: "disable", ShortUsage: "tailscale lock disable ", ShortHelp: "Consume a disablement secret to shut down tailnet lock for the tailnet", @@ -546,21 +547,21 @@ Once this secret is used, it has been distributed to all nodes in the tailnet and should be considered public. `), - Exec: runNetworkLockDisable, + Exec: runTailnetLockDisable, } -func runNetworkLockDisable(ctx context.Context, args []string) error { - _, secrets, err := parseNLArgs(args, false, true) +func runTailnetLockDisable(ctx context.Context, args []string) error { + _, secrets, err := parseTLArgs(args, false, true) if err != nil { return err } if len(secrets) != 1 { return errors.New("usage: tailscale lock disable ") } - return localClient.NetworkLockDisable(ctx, secrets[0]) + return localClient.TailnetLockDisable(ctx, secrets[0]) } -var nlLocalDisableCmd = &ffcli.Command{ +var tlLocalDisableCmd = &ffcli.Command{ Name: "local-disable", ShortUsage: "tailscale lock local-disable", ShortHelp: "Disable tailnet lock for this node only", @@ -575,22 +576,22 @@ that the current node will accept traffic from other nodes in the tailnet that are locked out. `), - Exec: runNetworkLockLocalDisable, + Exec: runTailnetLockLocalDisable, } -func runNetworkLockLocalDisable(ctx context.Context, args []string) error { - return localClient.NetworkLockForceLocalDisable(ctx) +func runTailnetLockLocalDisable(ctx context.Context, args []string) error { + return localClient.TailnetLockForceLocalDisable(ctx) } -var nlDisablementKDFCmd = &ffcli.Command{ +var tlDisablementKDFCmd = &ffcli.Command{ Name: "disablement-kdf", ShortUsage: "tailscale lock disablement-kdf ", ShortHelp: "Compute a disablement value from a disablement secret (advanced users only)", LongHelp: "Compute a disablement value from a disablement secret (advanced users only)", - Exec: runNetworkLockDisablementKDF, + Exec: runTailnetLockDisablementKDF, } -func runNetworkLockDisablementKDF(ctx context.Context, args []string) error { +func runTailnetLockDisablementKDF(ctx context.Context, args []string) error { if len(args) != 1 { return errors.New("usage: tailscale lock disablement-kdf ") } @@ -604,15 +605,15 @@ func runNetworkLockDisablementKDF(ctx context.Context, args []string) error { var nlLogArgs struct { limit int - json jsonoutput.JSONSchemaVersion + json jsonoutput.SchemaVersion } -var nlLogCmd = &ffcli.Command{ +var tlLogCmd = &ffcli.Command{ Name: "log", ShortUsage: "tailscale lock log [--limit N]", ShortHelp: "List changes applied to tailnet lock", LongHelp: "List changes applied to tailnet lock", - Exec: runNetworkLockLog, + Exec: runTailnetLockLog, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock log") fs.IntVar(&nlLogArgs.limit, "limit", 50, "max number of updates to list") @@ -621,7 +622,7 @@ var nlLogCmd = &ffcli.Command{ })(), } -func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, error) { +func nlDescribeUpdate(update ipnstate.TailnetLockUpdate, color bool) (string, error) { terminalYellow := "" terminalClear := "" if color { @@ -693,8 +694,8 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er return stanza.String(), nil } -func runNetworkLockLog(ctx context.Context, args []string) error { - st, err := localClient.NetworkLockStatus(ctx) +func runTailnetLockLog(ctx context.Context, args []string) error { + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } @@ -702,22 +703,22 @@ func runNetworkLockLog(ctx context.Context, args []string) error { return errors.New("Tailnet Lock is not enabled") } - updates, err := localClient.NetworkLockLog(ctx, nlLogArgs.limit) + updates, err := localClient.TailnetLockLog(ctx, nlLogArgs.limit) if err != nil { return fixTailscaledConnectError(err) } out, useColor := colorableOutput() - return printNetworkLockLog(updates, out, nlLogArgs.json, useColor) + return printTailnetLockLog(updates, out, nlLogArgs.json, useColor) } -func printNetworkLockLog(updates []ipnstate.NetworkLockUpdate, out io.Writer, jsonSchema jsonoutput.JSONSchemaVersion, useColor bool) error { +func printTailnetLockLog(updates []ipnstate.NetworkLockUpdate, out io.Writer, jsonSchema jsonoutput.SchemaVersion, useColor bool) error { if jsonSchema.IsSet { - if jsonSchema.Value == 1 { - return jsonoutput.PrintNetworkLockLogJSONV1(out, updates) + if jsonSchema.Version == 1 { + return jsonoutput.PrintTailnetLockLogJSONV1(out, updates) } else { - return fmt.Errorf("unrecognised version: %d", jsonSchema.Value) + return fmt.Errorf("unrecognised version: %d", jsonSchema.Version) } } @@ -772,11 +773,11 @@ func wrapAuthKey(ctx context.Context, keyStr string, status *ipnstate.Status) er Meta: m, } - wrapped, err := localClient.NetworkLockWrapPreauthKey(ctx, keyStr, priv) + wrapped, err := localClient.TailnetLockWrapPreauthKey(ctx, keyStr, priv) if err != nil { return fmt.Errorf("wrapping failed: %w", err) } - if err := localClient.NetworkLockModify(ctx, []tka.Key{k}, nil); err != nil { + if err := localClient.TailnetLockModify(ctx, []tka.Key{k}, nil); err != nil { return fmt.Errorf("add key failed: %w", err) } @@ -784,13 +785,13 @@ func wrapAuthKey(ctx context.Context, keyStr string, status *ipnstate.Status) er return nil } -var nlRevokeKeysArgs struct { +var tlRevokeKeysArgs struct { cosign bool finish bool forkFrom string } -var nlRevokeKeysCmd = &ffcli.Command{ +var tlRevokeKeysCmd = &ffcli.Command{ Name: "revoke-keys", ShortUsage: "tailscale lock revoke-keys ...\n revoke-keys [--cosign] [--finish] ", ShortHelp: "Revoke compromised tailnet-lock keys", @@ -806,20 +807,20 @@ Revocation is a multi-step process that requires several signing nodes to ` + "` most recent command output on the next signing node in sequence. 3. Once the number of ` + "`--cosign`" + `s is greater than the number of keys being revoked, run the command one final time with ` + "`--finish`" + ` instead of ` + "`--cosign`" + `.`, - Exec: runNetworkLockRevokeKeys, + Exec: runTailnetLockRevokeKeys, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock revoke-keys") - fs.BoolVar(&nlRevokeKeysArgs.cosign, "cosign", false, "continue generating the recovery using the tailnet lock key on this device and the provided recovery blob") - fs.BoolVar(&nlRevokeKeysArgs.finish, "finish", false, "finish the recovery process by transmitting the revocation") - fs.StringVar(&nlRevokeKeysArgs.forkFrom, "fork-from", "", "parent AUM hash to rewrite from (advanced users only)") + fs.BoolVar(&tlRevokeKeysArgs.cosign, "cosign", false, "continue generating the recovery using the tailnet lock key on this device and the provided recovery blob") + fs.BoolVar(&tlRevokeKeysArgs.finish, "finish", false, "finish the recovery process by transmitting the revocation") + fs.StringVar(&tlRevokeKeysArgs.forkFrom, "fork-from", "", "parent AUM hash to rewrite from (advanced users only)") return fs })(), } -func runNetworkLockRevokeKeys(ctx context.Context, args []string) error { +func runTailnetLockRevokeKeys(ctx context.Context, args []string) error { // First step in the process - if !nlRevokeKeysArgs.cosign && !nlRevokeKeysArgs.finish { - revokeKeys, _, err := parseNLArgs(args, true, false) + if !tlRevokeKeysArgs.cosign && !tlRevokeKeysArgs.finish { + revokeKeys, _, err := parseTLArgs(args, true, false) if err != nil { return err } @@ -837,22 +838,22 @@ func runNetworkLockRevokeKeys(ctx context.Context, args []string) error { } var forkFrom tka.AUMHash - if nlRevokeKeysArgs.forkFrom != "" { - if len(nlRevokeKeysArgs.forkFrom) == (len(forkFrom) * 2) { + if tlRevokeKeysArgs.forkFrom != "" { + if len(tlRevokeKeysArgs.forkFrom) == (len(forkFrom) * 2) { // Hex-encoded: like the output of the lock log command. - b, err := hex.DecodeString(nlRevokeKeysArgs.forkFrom) + b, err := hex.DecodeString(tlRevokeKeysArgs.forkFrom) if err != nil { return fmt.Errorf("invalid fork-from hash: %v", err) } copy(forkFrom[:], b) } else { - if err := forkFrom.UnmarshalText([]byte(nlRevokeKeysArgs.forkFrom)); err != nil { + if err := forkFrom.UnmarshalText([]byte(tlRevokeKeysArgs.forkFrom)); err != nil { return fmt.Errorf("invalid fork-from hash: %v", err) } } } - aumBytes, err := localClient.NetworkLockGenRecoveryAUM(ctx, keyIDs, forkFrom) + aumBytes, err := localClient.TailnetLockGenRecoveryAUM(ctx, keyIDs, forkFrom) if err != nil { return fmt.Errorf("generation of recovery AUM failed: %w", err) } @@ -873,8 +874,8 @@ func runNetworkLockRevokeKeys(ctx context.Context, args []string) error { return fmt.Errorf("decoding recovery AUM: %v", err) } - if nlRevokeKeysArgs.cosign { - aumBytes, err := localClient.NetworkLockCosignRecoveryAUM(ctx, recoveryAUM) + if tlRevokeKeysArgs.cosign { + aumBytes, err := localClient.TailnetLockCosignRecoveryAUM(ctx, recoveryAUM) if err != nil { return fmt.Errorf("co-signing recovery AUM failed: %w", err) } @@ -889,8 +890,8 @@ Alternatively if you are done with co-signing, complete recovery by running the `, os.Args[0], aumBytes, os.Args[0], aumBytes) } - if nlRevokeKeysArgs.finish { - if err := localClient.NetworkLockSubmitRecoveryAUM(ctx, recoveryAUM); err != nil { + if tlRevokeKeysArgs.finish { + if err := localClient.TailnetLockSubmitRecoveryAUM(ctx, recoveryAUM); err != nil { return fmt.Errorf("submitting recovery AUM failed: %w", err) } fmt.Println("Recovery completed.") diff --git a/cmd/tailscale/cli/network-lock_test.go b/cmd/tailscale/cli/tailnet-lock_test.go similarity index 92% rename from cmd/tailscale/cli/network-lock_test.go rename to cmd/tailscale/cli/tailnet-lock_test.go index 8e49265bf..76fba6517 100644 --- a/cmd/tailscale/cli/network-lock_test.go +++ b/cmd/tailscale/cli/tailnet-lock_test.go @@ -10,7 +10,8 @@ import ( "github.com/google/go-cmp/cmp" "go4.org/mem" - "tailscale.com/cmd/tailscale/cli/jsonoutput" + + "tailscale.com/cmd/tailscale/jsonoutput" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" "tailscale.com/tka" @@ -18,7 +19,7 @@ import ( "tailscale.com/types/tkatype" ) -func TestNetworkLockLogOutput(t *testing.T) { +func TestTailnetLockLogOutput(t *testing.T) { votes := uint(1) aum1 := tka.AUM{ MessageKind: tka.AUMAddKey, @@ -63,7 +64,7 @@ func TestNetworkLockLogOutput(t *testing.T) { Votes: &votes, } - updates := []ipnstate.NetworkLockUpdate{ + updates := []ipnstate.TailnetLockUpdate{ { Hash: aum3.Hash(), Change: aum3.MessageKind.String(), @@ -85,10 +86,10 @@ func TestNetworkLockLogOutput(t *testing.T) { t.Parallel() var outBuf bytes.Buffer - json := jsonoutput.JSONSchemaVersion{} + json := jsonoutput.SchemaVersion{} useColor := false - printNetworkLockLog(updates, &outBuf, json, useColor) + printTailnetLockLog(updates, &outBuf, json, useColor) t.Logf("%s", outBuf.String()) @@ -189,13 +190,13 @@ KeyID: tlpub:0202 t.Parallel() var outBuf bytes.Buffer - json := jsonoutput.JSONSchemaVersion{ - IsSet: true, - Value: 1, + json := jsonoutput.SchemaVersion{ + IsSet: true, + Version: 1, } useColor := false - printNetworkLockLog(updates, &outBuf, json, useColor) + printTailnetLockLog(updates, &outBuf, json, useColor) want := jsonV1 @@ -205,7 +206,7 @@ KeyID: tlpub:0202 }) } -func TestNetworkLockStatusOutput(t *testing.T) { +func TestTailnetLockStatusOutput(t *testing.T) { aum := tka.AUM{ MessageKind: tka.AUMNoOp, } @@ -226,12 +227,12 @@ func TestNetworkLockStatusOutput(t *testing.T) { t.Run("json-1", func(t *testing.T) { for _, tt := range []struct { Name string - Status ipnstate.NetworkLockStatus + Status ipnstate.TailnetLockStatus Want string }{ { Name: "tailnet-lock-disabled", - Status: ipnstate.NetworkLockStatus{Enabled: false}, + Status: ipnstate.TailnetLockStatus{Enabled: false}, Want: `{ "SchemaVersion": "1", "Enabled": false @@ -240,7 +241,7 @@ func TestNetworkLockStatusOutput(t *testing.T) { }, { Name: "tailnet-lock-disabled-with-keys", - Status: ipnstate.NetworkLockStatus{ + Status: ipnstate.TailnetLockStatus{ Enabled: false, NodeKey: &nodeKey1, PublicKey: trustedNlPub, @@ -255,7 +256,7 @@ func TestNetworkLockStatusOutput(t *testing.T) { }, { Name: "tailnet-lock-enabled", - Status: ipnstate.NetworkLockStatus{ + Status: ipnstate.TailnetLockStatus{ Enabled: true, Head: &head, PublicKey: nlPub, @@ -355,9 +356,9 @@ func TestNetworkLockStatusOutput(t *testing.T) { t.Parallel() var outBuf bytes.Buffer - err := jsonoutput.PrintNetworkLockStatusJSONV1(&outBuf, &tt.Status) + err := jsonoutput.PrintTailnetLockStatusJSONV1(&outBuf, &tt.Status) if err != nil { - t.Fatalf("PrintNetworkLockStatusJSONV1: %v", err) + t.Fatalf("PrintTailnetLockStatusJSONV1: %v", err) } if diff := cmp.Diff(outBuf.String(), tt.Want); diff != "" { diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index 586df07bb..982a69ca9 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -110,15 +110,15 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet { upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel") upf.Var(notFalseVar{}, "host-routes", hidden+"install host routes to other Tailscale nodes (must be true as of Tailscale 1.67+)") upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP, base name, or auto:any) for internet traffic, or empty string to not use an exit node") - upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node") + upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "allow direct access to the local network when routing traffic via an exit node") upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections") upf.BoolVar(&upArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy") - upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")") + upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request (e.g. \"tag:eng,tag:montreal,tag:ssh\"); the \"tag:\" prefix is optional and added automatically when omitted (e.g. \"eng,montreal,ssh\")") upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS") upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes") upf.BoolVar(&upArgs.advertiseConnector, "advertise-connector", false, "advertise this node as an app connector") upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet") - upf.BoolVar(&upArgs.postureChecking, "report-posture", false, hidden+"allow management plane to gather device posture information") + upf.BoolVar(&upArgs.postureChecking, "report-posture", false, "allow management plane to gather device posture information") if safesocket.GOOSUsesPeerCreds(goos) { upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo") @@ -309,9 +309,15 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo var tags []string if upArgs.advertiseTags != "" { tags = strings.Split(upArgs.advertiseTags, ",") - for _, tag := range tags { - err := tailcfg.CheckTag(tag) - if err != nil { + for i, tag := range tags { + // Allow users to omit the "tag:" prefix; if the tag has no + // colon at all, add it for them. Tags with a colon must be + // fully qualified ("tag:foo") and are validated as-is. + if !strings.Contains(tag, ":") { + tag = "tag:" + tag + tags[i] = tag + } + if err := tailcfg.CheckTag(tag); err != nil { return nil, fmt.Errorf("tag: %q: %s", tag, err) } } @@ -647,7 +653,10 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE } } - authKey, err = f(ctx, clientSecret, prefs.AdvertiseTags) + authKey, err = f(ctx, tailscale.ResolveAuthKeyArgs{ + AuthKey: clientSecret, + Tags: prefs.AdvertiseTags, + }) if err != nil { return err } @@ -660,7 +669,13 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE return err } - authKey, err = f(ctx, prefs.ControlURL, upArgs.clientID, idToken, upArgs.audience, prefs.AdvertiseTags) + authKey, err = f(ctx, tailscale.ResolveAuthKeyWIFArgs{ + BaseURL: prefs.ControlURL, + ClientID: upArgs.clientID, + IDToken: idToken, + Audience: upArgs.audience, + Tags: prefs.AdvertiseTags, + }) if err != nil { return err } @@ -726,7 +741,7 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE if s := n.State; s != nil { ipnIsRunning = *s == ipn.Running } - if n.NetMap != nil && n.NetMap.NodeKey != origNodeKey { + if n.SelfChange != nil && n.SelfChange.Key != origNodeKey { waitingForKeyChange = false } if ipnIsRunning && !waitingForKeyChange { @@ -911,6 +926,7 @@ func init() { addPrefFlagMapping("auto-update", "AutoUpdate.Apply") addPrefFlagMapping("advertise-connector", "AppConnector") addPrefFlagMapping("report-posture", "PostureChecking") + addPrefFlagMapping("remote-config", "RemoteConfig") addPrefFlagMapping("relay-server-port", "RelayServerPort") addPrefFlagMapping("sync", "Sync") addPrefFlagMapping("relay-server-static-endpoints", "RelayServerStaticEndpoints") diff --git a/cmd/tailscale/cli/update.go b/cmd/tailscale/cli/update.go index 47177347d..39cdfe6c3 100644 --- a/cmd/tailscale/cli/update.go +++ b/cmd/tailscale/cli/update.go @@ -10,6 +10,7 @@ import ( "errors" "flag" "fmt" + "io" "runtime" "github.com/peterbourgon/ff/v3/ffcli" @@ -67,8 +68,19 @@ var updateArgs struct { version string // explicit version; empty means auto } +const gokrazyUpdateFromURLMagicArg = "--gokrazy-update-from-url" + func runUpdate(ctx context.Context, args []string) error { if len(args) > 0 { + if runtime.GOOS == "linux" && distro.Get() == distro.Gokrazy { + gokArgs, err := gokrazyUpdateArgsFromMagicArg(args) + if err != nil { + return err + } + if gokArgs != nil { + return clientupdate.GokrazyUpdateFromURL.Get()(ctx, *gokArgs) + } + } return flag.ErrHelp } if updateArgs.version != "" && updateArgs.track != "" { @@ -102,3 +114,36 @@ func confirmUpdate(ver string) bool { msg := fmt.Sprintf("This will update Tailscale from %v to %v. Continue?", version.Short(), ver) return prompt.YesNo(msg, true) } + +// gokrazyUpdateArgsFromMagicArg parses the Gokrazy update-from-URL command-line +// flow. It returns nil if args do not select that flow. A non-nil result means +// the caller may safely invoke clientupdate.GokrazyUpdateFromURL. +func gokrazyUpdateArgsFromMagicArg(args []string) (*clientupdate.GokrazyUpdateArgs, error) { + var updateURL string + var unsigned bool + + fs := flag.NewFlagSet("gokrazy-update", flag.ContinueOnError) + fs.SetOutput(io.Discard) + // This flag path is exercised end-to-end by TestGokrazyUpdatesItselfToSameImage. + fs.StringVar(&updateURL, gokrazyUpdateFromURLMagicArg[2:], "", "URL of the Gokrazy archive format file to install") + fs.BoolVar(&unsigned, "unsigned", false, "skip GAF signature verification; for tests only") + if err := fs.Parse(args); err != nil { + return nil, err + } + if fs.NArg() != 0 { + return nil, nil + } + if updateURL == "" { + return nil, nil + } + if !clientupdate.GokrazyUpdateFromURL.IsSet() { + return nil, errors.New("gokrazy update support is not linked into this binary") + } + return &clientupdate.GokrazyUpdateArgs{ + URL: updateURL, + AllowUnsigned: unsigned, + Logf: func(format string, args ...any) { + printf(format+"\n", args...) + }, + }, nil +} diff --git a/cmd/tailscale/cli/whoami.go b/cmd/tailscale/cli/whoami.go new file mode 100644 index 000000000..5c7966216 --- /dev/null +++ b/cmd/tailscale/cli/whoami.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" +) + +var whoamiCmd = &ffcli.Command{ + Name: "whoami", + ShortUsage: "tailscale whoami [--json]", + ShortHelp: "Show the machine and user identity of the current machine", + LongHelp: strings.TrimSpace(` + 'tailscale whoami' shows the machine and user identity of the current machine. + It is equivalent to running 'tailscale whois' against one of the current machine's own Tailscale IP addresses. + `), + Exec: runWhoami, + FlagSet: func() *flag.FlagSet { + fs := newFlagSet("whoami") + fs.BoolVar(&whoamiArgs.json, "json", false, "output in JSON format") + return fs + }(), +} + +var whoamiArgs struct { + json bool // output in JSON format +} + +func runWhoami(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("too many arguments, expected none") + } + st, err := localClient.StatusWithoutPeers(ctx) + if err != nil { + return err + } + if len(st.TailscaleIPs) == 0 { + return fmt.Errorf("no current Tailscale IP address; state: %v", st.BackendState) + } + who, err := localClient.WhoIsProto(ctx, "", st.TailscaleIPs[0].String()) + if err != nil { + return err + } + return printWhoIs(who, st.CurrentTailnet, whoamiArgs.json) +} diff --git a/cmd/tailscale/cli/whois.go b/cmd/tailscale/cli/whois.go index 7cc8f2889..c6cae118c 100644 --- a/cmd/tailscale/cli/whois.go +++ b/cmd/tailscale/cli/whois.go @@ -4,6 +4,7 @@ package cli import ( + "cmp" "context" "encoding/json" "errors" @@ -13,6 +14,9 @@ import ( "text/tabwriter" "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/client/tailscale/apitype" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" ) var whoisCmd = &ffcli.Command{ @@ -46,10 +50,27 @@ func runWhoIs(ctx context.Context, args []string) error { if err != nil { return err } - if whoIsArgs.json { + return printWhoIs(who, nil, whoIsArgs.json) +} + +// whoisAndTailnet combines a WhoIsResponse and TailnetStatus for display. +type whoisAndTailnet struct { + *apitype.WhoIsResponse + CurrentTailnet *ipnstate.TailnetStatus `json:",omitzero"` +} + +// printWhoIs prints the WhoIsResponse to Stdout, either as JSON (if asJSON is +// true) or in a human-readable form. +// If tailnet is non-nil, tailnet information is included in the output. +func printWhoIs(who *apitype.WhoIsResponse, tailnet *ipnstate.TailnetStatus, asJSON bool) error { + wat := whoisAndTailnet{ + WhoIsResponse: who, + CurrentTailnet: tailnet, + } + if asJSON { ec := json.NewEncoder(Stdout) ec.SetIndent("", " ") - ec.Encode(who) + ec.Encode(wat) return nil } @@ -68,6 +89,20 @@ func runWhoIs(ctx context.Context, args []string) error { fmt.Fprintf(w, " Name:\t%s\n", who.UserProfile.LoginName) fmt.Fprintf(w, " ID:\t%d\n", who.UserProfile.ID) } + if tailnet != nil { + // use the tailnet display name, if present + var displayName string + if who.Node.HasCap(tailcfg.NodeAttrTailnetDisplayName) { + v, err := tailcfg.UnmarshalNodeCapJSON[string](who.Node.CapMap, tailcfg.NodeAttrTailnetDisplayName) + if err == nil && len(v) > 0 { + displayName = v[0] + } + } + fmt.Fprintln(w, "Tailnet:") + // TODO(will@): plumb through StableTailnetID so we can show that here as well + fmt.Fprintf(w, " Name:\t%s\n", cmp.Or(displayName, tailnet.Name)) + fmt.Fprintf(w, " MagicDNS Suffix:\t%s\n", tailnet.MagicDNSSuffix) + } w.Flush() w = nil // avoid accidental use diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 01d3f418f..c240581e7 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -24,10 +24,12 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep github.com/aws/aws-sdk-go-v2/aws/signer/v4 from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+ github.com/aws/aws-sdk-go-v2/aws/transport/http from github.com/aws/aws-sdk-go-v2/config+ github.com/aws/aws-sdk-go-v2/config from tailscale.com/wif+ + github.com/aws/aws-sdk-go-v2/config/internal/ini from github.com/aws/aws-sdk-go-v2/config github.com/aws/aws-sdk-go-v2/credentials from github.com/aws/aws-sdk-go-v2/config github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds from github.com/aws/aws-sdk-go-v2/config github.com/aws/aws-sdk-go-v2/credentials/endpointcreds from github.com/aws/aws-sdk-go-v2/config github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds + github.com/aws/aws-sdk-go-v2/credentials/logincreds from github.com/aws/aws-sdk-go-v2/config github.com/aws/aws-sdk-go-v2/credentials/processcreds from github.com/aws/aws-sdk-go-v2/config github.com/aws/aws-sdk-go-v2/credentials/ssocreds from github.com/aws/aws-sdk-go-v2/config github.com/aws/aws-sdk-go-v2/credentials/stscreds from github.com/aws/aws-sdk-go-v2/config @@ -40,17 +42,21 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep github.com/aws/aws-sdk-go-v2/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sso+ github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn from github.com/aws/aws-sdk-go-v2/service/sso+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 from github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints+ - github.com/aws/aws-sdk-go-v2/internal/ini from github.com/aws/aws-sdk-go-v2/config - github.com/aws/aws-sdk-go-v2/internal/middleware from github.com/aws/aws-sdk-go-v2/service/sso+ github.com/aws/aws-sdk-go-v2/internal/rand from github.com/aws/aws-sdk-go-v2/aws+ github.com/aws/aws-sdk-go-v2/internal/sdk from github.com/aws/aws-sdk-go-v2/aws+ github.com/aws/aws-sdk-go-v2/internal/sdkio from github.com/aws/aws-sdk-go-v2/credentials/processcreds github.com/aws/aws-sdk-go-v2/internal/shareddefaults from github.com/aws/aws-sdk-go-v2/config+ - github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 + github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4+ github.com/aws/aws-sdk-go-v2/internal/sync/singleflight from github.com/aws/aws-sdk-go-v2/aws github.com/aws/aws-sdk-go-v2/internal/timeconv from github.com/aws/aws-sdk-go-v2/aws/retry + github.com/aws/aws-sdk-go-v2/internal/v4a from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/aws-sdk-go-v2/internal/v4a/internal/crypto from github.com/aws/aws-sdk-go-v2/internal/v4a + github.com/aws/aws-sdk-go-v2/internal/v4a/internal/v4 from github.com/aws/aws-sdk-go-v2/internal/v4a github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts github.com/aws/aws-sdk-go-v2/service/internal/presigned-url from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/aws-sdk-go-v2/service/signin from github.com/aws/aws-sdk-go-v2/config+ + github.com/aws/aws-sdk-go-v2/service/signin/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/signin + github.com/aws/aws-sdk-go-v2/service/signin/types from github.com/aws/aws-sdk-go-v2/credentials/logincreds+ L github.com/aws/aws-sdk-go-v2/service/ssm from tailscale.com/feature/awsparamstore L github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm L github.com/aws/aws-sdk-go-v2/service/ssm/types from github.com/aws/aws-sdk-go-v2/service/ssm @@ -63,7 +69,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep github.com/aws/aws-sdk-go-v2/service/sts from github.com/aws/aws-sdk-go-v2/config+ github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sts github.com/aws/aws-sdk-go-v2/service/sts/types from github.com/aws/aws-sdk-go-v2/credentials/stscreds+ - github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ + 💣 github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ github.com/aws/smithy-go/auth from github.com/aws/aws-sdk-go-v2/internal/auth+ github.com/aws/smithy-go/auth/bearer from github.com/aws/aws-sdk-go-v2/aws+ github.com/aws/smithy-go/context from github.com/aws/smithy-go/auth/bearer @@ -73,7 +79,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep github.com/aws/smithy-go/encoding/json from github.com/aws/aws-sdk-go-v2/service/ssooidc+ github.com/aws/smithy-go/encoding/xml from github.com/aws/aws-sdk-go-v2/service/sts github.com/aws/smithy-go/endpoints from github.com/aws/aws-sdk-go-v2/service/sso+ - github.com/aws/smithy-go/endpoints/private/rulesfn from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/smithy-go/endpoints/private/rulesfn from github.com/aws/aws-sdk-go-v2/service/sts+ + github.com/aws/smithy-go/eventstream from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+ github.com/aws/smithy-go/internal/sync/singleflight from github.com/aws/smithy-go/auth/bearer github.com/aws/smithy-go/io from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+ github.com/aws/smithy-go/logging from github.com/aws/aws-sdk-go-v2/aws+ @@ -82,17 +89,26 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep github.com/aws/smithy-go/private/requestcompression from github.com/aws/aws-sdk-go-v2/config github.com/aws/smithy-go/ptr from github.com/aws/aws-sdk-go-v2/aws+ github.com/aws/smithy-go/rand from github.com/aws/aws-sdk-go-v2/aws/middleware+ - github.com/aws/smithy-go/time from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/smithy-go/sync from github.com/aws/smithy-go/transport/http + github.com/aws/smithy-go/time from github.com/aws/aws-sdk-go-v2/service/ssm+ github.com/aws/smithy-go/tracing from github.com/aws/aws-sdk-go-v2/aws/middleware+ + github.com/aws/smithy-go/traits from github.com/aws/smithy-go/eventstream github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws+ github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http L github.com/aws/smithy-go/waiter from github.com/aws/aws-sdk-go-v2/service/ssm + github.com/bradfitz/monogok/disklayout from tailscale.com/cmd/tailscale/cli+ github.com/coder/websocket from tailscale.com/util/eventbus github.com/coder/websocket/internal/errd from github.com/coder/websocket github.com/coder/websocket/internal/util from github.com/coder/websocket - github.com/coder/websocket/internal/xsync from github.com/coder/websocket W 💣 github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/pe+ W 💣 github.com/dblohm7/wingoes/pe from tailscale.com/util/winutil/authenticode + github.com/diskfs/go-diskfs/backend from github.com/diskfs/go-diskfs/filesystem/ext4+ + github.com/diskfs/go-diskfs/filesystem from github.com/diskfs/go-diskfs/filesystem/ext4 + github.com/diskfs/go-diskfs/filesystem/ext4 from tailscale.com/gokrazy/mkfs + github.com/diskfs/go-diskfs/filesystem/ext4/crc from github.com/diskfs/go-diskfs/filesystem/ext4 + github.com/diskfs/go-diskfs/filesystem/ext4/md4 from github.com/diskfs/go-diskfs/filesystem/ext4 + github.com/diskfs/go-diskfs/util/bitmap from github.com/diskfs/go-diskfs/filesystem/ext4 + github.com/diskfs/go-diskfs/util/slices from github.com/diskfs/go-diskfs/filesystem/ext4 L github.com/fogleman/gg from tailscale.com/client/systray github.com/fxamacker/cbor/v2 from tailscale.com/tka github.com/gaissmai/bart from tailscale.com/net/tsdial @@ -109,13 +125,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/v1 from tailscale.com/net/routecheck+ L 💣 github.com/godbus/dbus/v5 from fyne.io/systray+ L github.com/godbus/dbus/v5/introspect from fyne.io/systray+ L github.com/godbus/dbus/v5/prop from fyne.io/systray L github.com/golang/freetype/raster from github.com/fogleman/gg+ L github.com/golang/freetype/truetype from github.com/fogleman/gg github.com/golang/groupcache/lru from tailscale.com/net/dnscache - DW github.com/google/uuid from tailscale.com/clientupdate+ + github.com/google/uuid from tailscale.com/clientupdate+ github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ github.com/huin/goupnp from github.com/huin/goupnp/dcps/internetgateway2+ github.com/huin/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper @@ -144,7 +161,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep W 💣 github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - github.com/tailscale/hujson from tailscale.com/ipn/conffile + github.com/tailscale/hujson from tailscale.com/ipn/conffile+ github.com/tailscale/web-client-prebuilt from tailscale.com/client/web github.com/toqueteos/webbrowser from tailscale.com/cmd/tailscale/cli+ github.com/x448/float16 from github.com/fxamacker/cbor/v2 @@ -165,11 +182,13 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+ tailscale.com/client/web from tailscale.com/cmd/tailscale/cli tailscale.com/clientupdate from tailscale.com/cmd/tailscale/cli - LW tailscale.com/clientupdate/distsign from tailscale.com/clientupdate + tailscale.com/clientupdate/distsign from tailscale.com/clientupdate+ tailscale.com/cmd/tailscale/cli from tailscale.com/cmd/tailscale tailscale.com/cmd/tailscale/cli/ffcomplete from tailscale.com/cmd/tailscale/cli tailscale.com/cmd/tailscale/cli/ffcomplete/internal from tailscale.com/cmd/tailscale/cli/ffcomplete - tailscale.com/cmd/tailscale/cli/jsonoutput from tailscale.com/cmd/tailscale/cli + tailscale.com/cmd/tailscale/jsonoutput from tailscale.com/cmd/tailscale/cli + tailscale.com/cmd/tailscale/tsdnsjsonv0 from tailscale.com/cmd/tailscale/cli + tailscale.com/cmd/tailscale/tsroutecheckjsonv0 from tailscale.com/cmd/tailscale/cli tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ tailscale.com/control/controlhttp from tailscale.com/control/ts2021 tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp @@ -183,6 +202,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/feature from tailscale.com/tsweb+ L tailscale.com/feature/awsparamstore from tailscale.com/feature/condregister/awsparamstore tailscale.com/feature/buildfeatures from tailscale.com/cmd/tailscale/cli+ + tailscale.com/feature/captiveportal/netcheckhook from tailscale.com/cmd/tailscale/cli tailscale.com/feature/capture/dissector from tailscale.com/cmd/tailscale/cli tailscale.com/feature/condregister/awsparamstore from tailscale.com/cmd/tailscale/cli tailscale.com/feature/condregister/identityfederation from tailscale.com/cmd/tailscale/cli @@ -192,8 +212,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/feature/identityfederation from tailscale.com/feature/condregister/identityfederation tailscale.com/feature/oauthkey from tailscale.com/feature/condregister/oauthkey tailscale.com/feature/portmapper from tailscale.com/feature/condregister/portmapper + tailscale.com/feature/serviceclientprefs/serviceclient from tailscale.com/client/local tailscale.com/feature/syspolicy from tailscale.com/cmd/tailscale/cli tailscale.com/feature/useproxy from tailscale.com/feature/condregister/useproxy + tailscale.com/gokrazy/mkfs from tailscale.com/cmd/tailscale/cli tailscale.com/health from tailscale.com/net/tlsdial+ tailscale.com/health/healthmsg from tailscale.com/cmd/tailscale/cli tailscale.com/hostinfo from tailscale.com/client/web+ @@ -206,11 +228,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/metrics from tailscale.com/tsweb+ tailscale.com/net/ace from tailscale.com/cmd/tailscale/cli tailscale.com/net/bakedroots from tailscale.com/net/tlsdial - tailscale.com/net/captivedetection from tailscale.com/net/netcheck + tailscale.com/net/captivedetection from tailscale.com/feature/captiveportal/netcheckhook tailscale.com/net/dnscache from tailscale.com/control/controlhttp+ tailscale.com/net/dnsfallback from tailscale.com/control/controlhttp+ tailscale.com/net/netaddr from tailscale.com/ipn+ - tailscale.com/net/netcheck from tailscale.com/cmd/tailscale/cli + tailscale.com/net/netcheck from tailscale.com/cmd/tailscale/cli+ tailscale.com/net/neterror from tailscale.com/net/netcheck+ tailscale.com/net/netknob from tailscale.com/net/netns+ 💣 tailscale.com/net/netmon from tailscale.com/cmd/tailscale/cli+ @@ -220,10 +242,13 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/net/ping from tailscale.com/net/netcheck tailscale.com/net/portmapper from tailscale.com/feature/portmapper tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ + tailscale.com/net/routecheck from tailscale.com/client/local+ + tailscale.com/net/routecheck/peernode from tailscale.com/net/routecheck tailscale.com/net/sockstats from tailscale.com/control/controlhttp+ tailscale.com/net/stun from tailscale.com/net/netcheck tailscale.com/net/tlsdial from tailscale.com/cmd/tailscale/cli+ tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/net/routecheck tailscale.com/net/tsaddr from tailscale.com/client/web+ tailscale.com/net/tsdial from tailscale.com/cmd/tailscale/cli+ 💣 tailscale.com/net/tshttpproxy from tailscale.com/feature/useproxy @@ -239,7 +264,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/tstime from tailscale.com/control/controlhttp+ tailscale.com/tstime/mono from tailscale.com/tstime/rate tailscale.com/tstime/rate from tailscale.com/cmd/tailscale/cli - tailscale.com/tsweb from tailscale.com/util/eventbus + tailscale.com/tsweb from tailscale.com/util/eventbus+ tailscale.com/tsweb/varz from tailscale.com/util/usermetric+ tailscale.com/types/appctype from tailscale.com/client/local+ tailscale.com/types/dnstype from tailscale.com/tailcfg+ @@ -275,6 +300,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/util/mak from tailscale.com/cmd/tailscale/cli+ tailscale.com/util/must from tailscale.com/clientupdate/distsign+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto + tailscale.com/util/progresstracking from tailscale.com/clientupdate/distsign+ tailscale.com/util/prompt from tailscale.com/cmd/tailscale/cli 💣 tailscale.com/util/qrcodes from tailscale.com/cmd/tailscale/cli tailscale.com/util/quarantine from tailscale.com/cmd/tailscale/cli @@ -331,7 +357,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep golang.org/x/net/icmp from tailscale.com/net/ping golang.org/x/net/idna from golang.org/x/net/http/httpproxy+ golang.org/x/net/internal/iana from golang.org/x/net/icmp+ - golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ golang.org/x/net/internal/socks from golang.org/x/net/proxy golang.org/x/net/ipv4 from golang.org/x/net/icmp+ golang.org/x/net/ipv6 from golang.org/x/net/icmp+ @@ -369,6 +395,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna archive/tar from tailscale.com/clientupdate + archive/zip from tailscale.com/clientupdate+ bufio from compress/flate+ bytes from archive/tar+ cmp from slices+ @@ -394,23 +421,24 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ - crypto/internal/fips140 from crypto/internal/fips140/aes+ + crypto/internal/fips140 from crypto/fips140+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ crypto/internal/fips140/alias from crypto/cipher+ crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ - crypto/internal/fips140/check from crypto/internal/fips140/aes+ - crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/check from crypto/fips140+ + crypto/internal/fips140/drbg from crypto/hpke+ crypto/internal/fips140/ecdh from crypto/ecdh crypto/internal/fips140/ecdsa from crypto/ecdsa crypto/internal/fips140/ed25519 from crypto/ed25519 crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 crypto/internal/fips140/edwards25519/field from crypto/ecdh+ - crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ + crypto/internal/fips140/hkdf from crypto/hkdf+ crypto/internal/fips140/hmac from crypto/hmac+ crypto/internal/fips140/mlkem from crypto/mlkem - crypto/internal/fips140/nistec from crypto/elliptic+ + crypto/internal/fips140/nistec from crypto/ecdsa+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec + crypto/internal/fips140/pbkdf2 from crypto/pbkdf2 crypto/internal/fips140/rsa from crypto/rsa crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ crypto/internal/fips140/sha3 from crypto/internal/fips140/hmac+ @@ -431,6 +459,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep crypto/internal/sysrand from crypto/internal/fips140/drbg crypto/md5 from crypto/tls+ crypto/mlkem from crypto/hpke+ + crypto/pbkdf2 from golang.org/x/crypto/pbkdf2 crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls crypto/rsa from crypto/tls+ @@ -444,7 +473,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep crypto/x509 from crypto/tls+ D crypto/x509/internal/macos from crypto/x509 crypto/x509/pkix from crypto/x509+ - DW database/sql/driver from github.com/google/uuid + database/sql/driver from github.com/google/uuid W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/peterbourgon/ff/v3+ @@ -464,6 +493,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep hash from compress/zlib+ hash/adler32 from compress/zlib hash/crc32 from compress/gzip+ + hash/fnv from tailscale.com/net/traffic+ hash/maphash from go4.org/mem html from html/template+ html/template from tailscale.com/util/eventbus @@ -490,7 +520,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep internal/goos from crypto/x509+ internal/msan from internal/runtime/maps+ internal/nettrace from net+ - internal/oserror from io/fs+ + internal/oserror from internal/syscall/windows+ internal/poll from net+ internal/profile from net/http/pprof internal/profilerecord from runtime+ @@ -500,9 +530,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep internal/runtime/atomic from internal/runtime/exithook+ L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime - internal/runtime/gc from runtime+ + internal/runtime/gc from internal/runtime/gc/scan+ internal/runtime/gc/scan from runtime - internal/runtime/maps from reflect+ + internal/runtime/maps from hash/maphash+ internal/runtime/math from internal/runtime/maps+ internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ @@ -516,7 +546,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep internal/synctest from sync internal/syscall/execenv from os+ LD internal/syscall/unix from crypto/internal/sysrand+ - W internal/syscall/windows from crypto/internal/sysrand+ + W internal/syscall/windows from crypto/internal/fips140deps/time+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os diff --git a/cmd/tailscale/jsonoutput/example_format_test.go b/cmd/tailscale/jsonoutput/example_format_test.go new file mode 100644 index 000000000..e9f60336c --- /dev/null +++ b/cmd/tailscale/jsonoutput/example_format_test.go @@ -0,0 +1,41 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "flag" + "fmt" + + "tailscale.com/cmd/tailscale/jsonoutput" +) + +func ExampleFormat_JSONBool() { + var args struct { + format jsonoutput.Format + } + + fs := flag.NewFlagSet("ExampleFormat", flag.ExitOnError) + fs.Var(&args.format, "format", `output format; empty (for human-readable), "json" or "json-line"`) + fs.Var(args.format.JSONBool(), "json", "output in JSON format") + + fs.Parse([]string{"-json"}) + fmt.Printf(`{format: %q, set: %t, version: %d}`, args.format, args.format.IsSet, args.format.Version) + // Output: + // {format: "json", set: true, version: 1} +} + +func ExampleFormat_JSONSchemaVersion() { + var args struct { + format jsonoutput.Format + } + + fs := flag.NewFlagSet("ExampleFormat", flag.ExitOnError) + fs.Var(&args.format, "format", `output format; empty (for human-readable), "json" or "json-line"`) + fs.Var(args.format.JSONSchemaVersion(), "json", "output in JSON format") + + fs.Parse([]string{"-json=2", "-format=json-line"}) + fmt.Printf(`{format: %q, set: %t, version: %d}`, args.format, args.format.IsSet, args.format.Version) + // Output: + // {format: "json-line", set: true, version: 2} +} diff --git a/cmd/tailscale/jsonoutput/example_responseenvelope_test.go b/cmd/tailscale/jsonoutput/example_responseenvelope_test.go new file mode 100644 index 000000000..38f27af7d --- /dev/null +++ b/cmd/tailscale/jsonoutput/example_responseenvelope_test.go @@ -0,0 +1,33 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "encoding/json" + "fmt" + + "tailscale.com/cmd/tailscale/jsonoutput" +) + +type Hello struct { + jsonoutput.ResponseEnvelope + Greeting string +} + +func ExampleResponseEnvelope() { + hi := Hello{ + ResponseEnvelope: jsonoutput.ResponseEnvelope{SchemaVersion: "1"}, + Greeting: "Hello, world", + } + out, err := json.MarshalIndent(hi, "", " ") + if err != nil { + panic(err) + } + fmt.Printf("%s\n", out) + // Output: + // { + // "SchemaVersion": "1", + // "Greeting": "Hello, world" + // } +} diff --git a/cmd/tailscale/jsonoutput/example_schemaversion_test.go b/cmd/tailscale/jsonoutput/example_schemaversion_test.go new file mode 100644 index 000000000..a198d2528 --- /dev/null +++ b/cmd/tailscale/jsonoutput/example_schemaversion_test.go @@ -0,0 +1,25 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "flag" + "fmt" + + "tailscale.com/cmd/tailscale/jsonoutput" +) + +var args struct { + json jsonoutput.SchemaVersion +} + +func ExampleSchemaVersion() { + fs := flag.NewFlagSet("ExampleSchemaVersion", flag.ExitOnError) + fs.Var(&args.json, "json", "output in JSON format") + + fs.Parse([]string{"-json=2"}) + fmt.Printf(`{set: %t, version: %d}`, args.json.IsSet, args.json.Version) + // Output: + // {set: true, version: 2} +} diff --git a/cmd/tailscale/jsonoutput/format.go b/cmd/tailscale/jsonoutput/format.go new file mode 100644 index 000000000..422f76860 --- /dev/null +++ b/cmd/tailscale/jsonoutput/format.go @@ -0,0 +1,130 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput + +import ( + "errors" + "flag" + "strconv" +) + +var _ flag.Value = new(Format) + +// Format implements the [flag.Value] interface, +// supporting a combination of both -format and -json flags. +// +// For some commands, like tailscale netcheck or tailscale routecheck, +// the user can specify the output format using the -format flag: +// +// tailscale routecheck -format=json-line. +// +// Setting this flag to "json" or "json-line" implies that the -json flag is set. +type Format struct { + s string + SchemaVersion +} + +// String returns the default value which is printed in the CLI help text. +func (f Format) String() string { + return string(f.s) +} + +// Set is called when the user passes the flag as a command-line argument. +func (f *Format) Set(s string) error { + f.s = s + isJSON := f.isJSON() + f.IsSet = isJSON + if !isJSON { + f.Version = 0 + } + return nil +} + +// IsBoolFlag reports that this [flag.Value] can be set without an argument. +// This is the magic interface that makes -name equivalent to -name=true +// rather than using the next command-line argument. +func (f *Format) IsBoolFlag() bool { + return false // requires argument, overrides [SchemaVersion.IsBoolFlag] +} + +// isJSON reports whether f represents a format based on JSON. +func (f *Format) isJSON() bool { + return f.s == "json" || f.s == "json-line" +} + +// setJSON either sets or clears the format value if it doesn’t match x. +func (f *Format) setJSON(x bool) { + switch x { + case f.isJSON(): + return // no change + case true: + f.s = "json" + case false: + f.s = "" // clear + f.Version = 0 + } +} + +// JSONBool returns a [flag.Value] for a boolean -json flag +// which is aware of the underlying format. +func (f *Format) JSONBool() flag.Value { + return &formatBool{f} +} + +type formatBool struct { + *Format +} + +// String returns the default value which is printed in the CLI help text. +func (f formatBool) String() string { + return strconv.FormatBool(f.isJSON()) +} + +// Set is called when the user passes the flag as a command-line argument. +func (f *formatBool) Set(s string) error { + if _, err := strconv.ParseBool(s); err != nil { + return errors.New("parse error") + } + f.SchemaVersion.Set(s) + f.setJSON(f.IsSet) + return nil +} + +// IsBoolFlag reports that this [flag.Value] can be set without an argument. +// This is the magic interface that makes -name equivalent to -name=true +// rather than using the next command-line argument. +func (f *formatBool) IsBoolFlag() bool { + return true +} + +// JSONSchemaVersion returns a [flag.Value] for a [SchemaVersion] -json flag +// which is aware of the underlying format. +func (f *Format) JSONSchemaVersion() flag.Value { + return &formatSchemaVersion{f} +} + +type formatSchemaVersion struct { + *Format +} + +// String returns the default value which is printed in the CLI help text. +func (f formatSchemaVersion) String() string { + return f.SchemaVersion.String() +} + +// Set is called when the user passes the flag as a command-line argument. +func (f *formatSchemaVersion) Set(s string) error { + if err := f.SchemaVersion.Set(s); err != nil { + return err + } + f.setJSON(f.IsSet) + return nil +} + +// IsBoolFlag reports that this [flag.Value] can be set without an argument. +// This is the magic interface that makes -name equivalent to -name=true +// rather than using the next command-line argument. +func (f *formatSchemaVersion) IsBoolFlag() bool { + return f.SchemaVersion.IsBoolFlag() +} diff --git a/cmd/tailscale/jsonoutput/format_test.go b/cmd/tailscale/jsonoutput/format_test.go new file mode 100644 index 000000000..c072a8746 --- /dev/null +++ b/cmd/tailscale/jsonoutput/format_test.go @@ -0,0 +1,404 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "errors" + "flag" + "io" + "strings" + "testing" + + "tailscale.com/cmd/tailscale/jsonoutput" +) + +func TestFormat(t *testing.T) { + for _, tc := range []struct { + name string + flags []string + wantString string + wantIsSet bool + wantVersion int + wantErr string + }{ + { + name: "no-flags", + flags: []string{}, + wantString: "", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "format-json", + flags: []string{"-format=json"}, + wantString: "json", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "format-json-line", + flags: []string{"-format=json-line"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "format-not-json", + flags: []string{"-format=xml"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "bool-bare", + flags: []string{"-bool"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "bool-true", + flags: []string{"-bool=true"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "bool-false", + flags: []string{"-bool=false"}, + wantString: "", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "bool-invalid", + flags: []string{"-bool=2"}, + wantErr: "parse error", + }, + { + name: "version-bare", + flags: []string{"-version"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "version-true", + flags: []string{"-version=true"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "version-false", + flags: []string{"-version=false"}, + wantString: "", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "version-number", + flags: []string{"-version=2"}, + wantString: "json", + wantIsSet: true, + wantVersion: 2, + }, + { + name: "version-zero", + flags: []string{"-version=0"}, + wantString: "json", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "version-invalid", + flags: []string{"-version=1.0"}, + wantErr: "parse error", + }, + { + name: "format-xml-then-bool-true", + flags: []string{"-format=xml", "-bool"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "format-json-then-bool-true", + flags: []string{"-format=json", "-bool"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "format-json-line-then-bool-true", + flags: []string{"-format=json-line", "-bool"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "format-xml-then-bool-false", + flags: []string{"-format=xml", "-bool=false"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "format-json-then-bool-false", + flags: []string{"-format=json", "-bool=false"}, + wantString: "", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "format-json-line-then-bool-false", + flags: []string{"-format=json-line", "-bool=false"}, + wantString: "", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "bool-true-then-format-xml", + flags: []string{"-bool", "-format=xml"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "bool-true-then-format-json", + flags: []string{"-bool", "--format=json"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "bool-true-then-format-json-line", + flags: []string{"-bool", "--format=json-line"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "bool-false-then-format-xml", + flags: []string{"-bool=false", "--format=xml"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "bool-false-then-format-json", + flags: []string{"-bool=false", "--format=json"}, + wantString: "json", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "bool-false-then-format-json-line", + flags: []string{"-bool=false", "--format=json-line"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "format-xml-then-version-true", + flags: []string{"-format=xml", "-version"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "format-json-then-version-true", + flags: []string{"-format=json", "-version"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "format-json-line-then-version-true", + flags: []string{"-format=json-line", "-version"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "format-xml-then-version-number", + flags: []string{"-format=xml", "-version=2"}, + wantString: "json", + wantIsSet: true, + wantVersion: 2, + }, + { + name: "format-json-then-version-number", + flags: []string{"-format=json", "-version=2"}, + wantString: "json", + wantIsSet: true, + wantVersion: 2, + }, + { + name: "format-json-line-then-version-number", + flags: []string{"-format=json-line", "-version=2"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 2, + }, + { + name: "format-xml-then-version-zero", + flags: []string{"-format=xml", "-version=0"}, + wantString: "json", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "format-json-then-version-zero", + flags: []string{"-format=json", "-version=0"}, + wantString: "json", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "format-json-line-then-version-zero", + flags: []string{"-format=json-line", "-version=0"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "format-xml-then-version-false", + flags: []string{"-format=xml", "-version=false"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "format-json-then-version-false", + flags: []string{"-format=json", "-version=false"}, + wantString: "", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "format-json-line-then-version-false", + flags: []string{"-format=json-line", "-version=false"}, + wantString: "", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "version-true-then-format-xml", + flags: []string{"-version", "-format=xml"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "version-true-then-format-json", + flags: []string{"-version", "--format=json"}, + wantString: "json", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "version-true-then-format-json-line", + flags: []string{"-version", "--format=json-line"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 1, + }, + { + name: "version-false-then-format-xml", + flags: []string{"-version=false", "--format=xml"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "version-false-then-format-json", + flags: []string{"-version=false", "--format=json"}, + wantString: "json", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "version-false-then-format-json-line", + flags: []string{"-version=false", "--format=json-line"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "version-number-then-format-xml", + flags: []string{"-version=2", "--format=xml"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "version-number-then-format-json", + flags: []string{"-version=2", "--format=json"}, + wantString: "json", + wantIsSet: true, + wantVersion: 2, + }, + { + name: "version-number-then-format-json-line", + flags: []string{"-version=2", "--format=json-line"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 2, + }, + { + name: "version-zero-then-format-xml", + flags: []string{"-version=0", "--format=xml"}, + wantString: "xml", + wantIsSet: false, + wantVersion: 0, + }, + { + name: "version-zero-then-format-json", + flags: []string{"-version=0", "--format=json"}, + wantString: "json", + wantIsSet: true, + wantVersion: 0, + }, + { + name: "version-zero-then-format-json-line", + flags: []string{"-version=0", "--format=json-line"}, + wantString: "json-line", + wantIsSet: true, + wantVersion: 0, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var f jsonoutput.Format + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.Var(&f, "format", "") + fs.Var(f.JSONBool(), "bool", "") + fs.Var(f.JSONSchemaVersion(), "version", "") + fs.SetOutput(io.Discard) // silence + + t.Logf("flags: %q", tc.flags) + if err := fs.Parse(tc.flags); err != nil { + // Unwrap the header added by FlagSet.failf: + // `invalid boolean value "invalid" for -json: ` + bits := strings.SplitN(err.Error(), ": ", 2) + err := errors.New(bits[len(bits)-1]) + if tc.wantErr == "" && err != nil { + t.Fatalf("err %s, want ", err) + } else if tc.wantErr != "" && (err == nil || err.Error() != tc.wantErr) { + t.Fatalf("err %v, want %s", err, tc.wantErr) + } + } + + if got := f.String(); got != tc.wantString { + t.Errorf("Format.String %q, want %q", got, tc.wantString) + } + if got := f.IsSet; got != tc.wantIsSet { + t.Errorf("Format.IsSet %t, want %t", got, tc.wantIsSet) + } + if got := f.Version; got != tc.wantVersion { + t.Errorf("Format.Version %d, want %d", got, tc.wantVersion) + } + }) + } +} diff --git a/cmd/tailscale/jsonoutput/jsonoutput.go b/cmd/tailscale/jsonoutput/jsonoutput.go new file mode 100644 index 000000000..1b062b816 --- /dev/null +++ b/cmd/tailscale/jsonoutput/jsonoutput.go @@ -0,0 +1,110 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package jsonoutput provides stable and versioned JSON serialisation for CLI output. +// This allows us to provide stable output to scripts/clients, but also make +// breaking changes to the output when it's useful. +// +// Historically we only used a boolean -json flag, so changing the output +// could break scripts that rely on the existing format. +// +// This package provides a [SchemaVersion] flag type that allows callers +// to pass either a boolean or a version number and get a consistent output. +// We'll bump the version when we make a breaking change +// that's likely to break scripts that rely on the existing output, +// e.g. if we remove a field or change the type/format. +// Passing just the boolean flag will always return 1, to preserve +// compatibility with scripts written before we versioned our output. +// +// This package provides a [Format] flag type that allows callers to specify +// which output format the command should print. +// This flag provides [Format.JSONBool] and [Format.JSONSchemaVersion] methods +// to support combining both -format=json and -format=json-line options +// with either boolean or versioned -json flags. +// +// This package also provides [ResponseEnvelope] which is used to provide the +// set of fields common to all versioned JSON output. +package jsonoutput + +import ( + "errors" + "flag" + "io" + "strconv" + "strings" +) + +var _ flag.Value = &SchemaVersion{} + +// SchemaVersion implements the [flag.Value] interface, +// tracking whether the flag has been set or cleared, and its value when set. +type SchemaVersion struct { + // IsSet tracks if the flag was set or cleared. + // This flag is true when set by -name or -name=true or -name=INT, + // otherwise it is false when cleared by -name=false. + IsSet bool + + // Version tracks the desired schema version, as set by the -name=INT flag. + // The version defaults to 1 when implicitly set by -name or -name=true. + Version int +} + +// String returns the default value which is printed in the CLI help text. +func (v SchemaVersion) String() string { + if v.IsSet { + return strconv.Itoa(v.Version) + } + return strconv.FormatBool(false) +} + +// Set is called when the user passes the flag as a command-line argument. +func (v *SchemaVersion) Set(s string) error { + // Delegate to a FlagSet to parse this as both a BoolVar and an IntVar. + // This is less efficient than copying the implementation from the standard library + // but this design makes it likelier that Set will inherit any upstream fixes. + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.BoolVar(&v.IsSet, "bool", false, "") + fs.IntVar(&v.Version, "int", 0, "") + fs.SetOutput(io.Discard) // silence + + // First, try to parse as an IntVar to handle -flag=INT. + // This order is important because -bool=0 will parse as false. + if err := fs.Parse([]string{"-int=" + s}); err == nil { + v.IsSet = true + return nil + } + // If that fails, parse as a BoolVar to handle -flag and -flag=false. + // This is checked last for compatibility with the boolean -json flag. + if err := fs.Parse([]string{"-bool=" + s}); err != nil { + // Unwrap the header added by FlagSet.failf: + // `invalid boolean value "invalid" for -bool: ` + bits := strings.SplitN(err.Error(), ": ", 2) + return errors.New(bits[len(bits)-1]) + } + // If the user doesn't supply a schema version, default to 1. + // This ensures that any existing scripts will continue to get their + // current output. + if v.IsSet { + v.Version = 1 + } else { + v.Version = 0 // if unset, zero out the Version + } + return nil +} + +// IsBoolFlag reports that this [flag.Value] can be set without an argument. +// This is the magic interface that makes -name equivalent to -name=true +// rather than using the next command-line argument. +func (v *SchemaVersion) IsBoolFlag() bool { + return true +} + +// ResponseEnvelope is a set of fields common to all versioned JSON output. +type ResponseEnvelope struct { + // SchemaVersion is the version of the JSON output, e.g. "1", "2", "3" + SchemaVersion string + + // ResponseWarning tells a user if a newer version of the JSON output + // is available. + ResponseWarning string `json:"_WARNING,omitzero"` +} diff --git a/cmd/tailscale/jsonoutput/jsonoutput_test.go b/cmd/tailscale/jsonoutput/jsonoutput_test.go new file mode 100644 index 000000000..b66f5a12e --- /dev/null +++ b/cmd/tailscale/jsonoutput/jsonoutput_test.go @@ -0,0 +1,164 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "flag" + "math" + "testing" + + gcmp "github.com/google/go-cmp/cmp" + "github.com/kballard/go-shellquote" + + "tailscale.com/cmd/tailscale/jsonoutput" +) + +func TestSchemaVersion(t *testing.T) { + for _, tc := range []struct { + name string + args string + want jsonoutput.SchemaVersion + wantErr string + wantStr string + }{ + { + name: "none", + want: jsonoutput.SchemaVersion{IsSet: false, Version: 0}, + wantStr: "false", + }, + { + name: "default", + args: "-got", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 1}, + wantStr: "1", + }, + { + name: "true", + args: "-got=true", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 1}, + wantStr: "1", + }, + { + name: "false", + args: "-got=false", + want: jsonoutput.SchemaVersion{IsSet: false, Version: 0}, + wantStr: "false", + }, + { + // Test that -got=0 isn’t interpreted as -bool=0, i.e. false. + name: "zero_not_false", + args: "-got=0", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 0}, + wantStr: "0", + }, + { + name: "one", + args: "-got=1", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 1}, + wantStr: "1", + }, + { + name: "two", + args: "-got=2", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 2}, + wantStr: "2", + }, + { + name: "max", + args: "-got=2147483647", + want: jsonoutput.SchemaVersion{IsSet: true, Version: math.MaxInt32}, + wantStr: "2147483647", + }, + { + name: "min", + args: "-got=-2147483648", + want: jsonoutput.SchemaVersion{IsSet: true, Version: math.MinInt32}, + wantStr: "-2147483648", + }, + { + name: "invalid", + args: "-got=invalid", + wantErr: `invalid boolean value "invalid" for -got: parse error`, + }, + { + name: "float", + args: "-got=1.3", + wantErr: `invalid boolean value "1.3" for -got: parse error`, + }, + { + name: "space", + args: "-got=' '", + wantErr: `invalid boolean value " " for -got: parse error`, + }, + { + name: "trailing_space", + args: "-got='1 '", + wantErr: `invalid boolean value "1 " for -got: parse error`, + }, + } { + args, err := shellquote.Split(tc.args) + if err != nil { + t.Fatalf("broken args %q: %v", tc.args, err) + } + + // Test both Set and String methods. + t.Run(tc.name, func(t *testing.T) { + var got jsonoutput.SchemaVersion + fs := flag.NewFlagSet("name", flag.ContinueOnError) + fs.Var(&got, "got", "usage") + + err = fs.Parse(args) + if err != nil && tc.wantErr == "" { + t.Errorf("parse error: %v", err) + } else if err != nil && err.Error() != tc.wantErr { + t.Errorf("parse error mismatch: %q, want %q", err, tc.wantErr) + } else if err == nil && tc.wantErr != "" { + t.Errorf("parse error: %v, want %q", err, tc.wantErr) + } + + if len(fs.Args()) != 0 { + t.Errorf("unexpected positional arguments: %q", fs.Args()) + } + + if diff := gcmp.Diff(tc.want, got); diff != "" { + t.Errorf("parse mismatch: -want +got\n%s", diff) + } + + if s := got.String(); s != tc.wantStr && tc.wantStr != "" { + t.Errorf("string %q, want %q", s, tc.wantStr) + } + }) + + if tc.args == "" { + continue // nothing to clobber + } + if tc.wantErr != "" { + continue // clobbering will just trigger another error + } + + // The last -got flag will clobber all previous -got flags. + t.Run(tc.name+"/clobber", func(t *testing.T) { + var got jsonoutput.SchemaVersion + fs := flag.NewFlagSet("name", flag.ContinueOnError) + fs.Var(&got, "got", "usage") + + sentinel := []string{"-got=-1"} + if err := fs.Parse(append(sentinel, args...)); err != nil { + t.Errorf("parse error: %v", err) + } + + if got.Version == -1 { + t.Errorf("sentinel detected: flag didn’t clobber") + } + + if len(fs.Args()) != 0 { + t.Errorf("unexpected positional arguments: %q", fs.Args()) + } + + if diff := gcmp.Diff(tc.want, got); diff != "" { + t.Errorf("parse mismatch: -want +got\n%s", diff) + } + }) + } +} diff --git a/cmd/tailscale/cli/jsonoutput/network-lock-log.go b/cmd/tailscale/jsonoutput/tailnet-lock-log.go similarity index 94% rename from cmd/tailscale/cli/jsonoutput/network-lock-log.go rename to cmd/tailscale/jsonoutput/tailnet-lock-log.go index c7c16e223..97af3bed7 100644 --- a/cmd/tailscale/cli/jsonoutput/network-lock-log.go +++ b/cmd/tailscale/jsonoutput/tailnet-lock-log.go @@ -16,7 +16,7 @@ import ( "tailscale.com/tka" ) -// PrintNetworkLockLogJSONV1 prints the stored TKA state as a JSON object to the CLI, +// PrintTailnetLockLogJSONV1 prints the stored TKA state as a JSON object to the CLI, // in a stable "v1" format. // // This format includes: @@ -24,7 +24,7 @@ import ( // - the AUM hash as a base32-encoded string // - the raw AUM as base64-encoded bytes // - the expanded AUM, which prints named fields for consumption by other tools -func PrintNetworkLockLogJSONV1(out io.Writer, updates []ipnstate.NetworkLockUpdate) error { +func PrintTailnetLockLogJSONV1(out io.Writer, updates []ipnstate.TailnetLockUpdate) error { messages := make([]logMessageV1, len(updates)) for i, update := range updates { @@ -57,9 +57,9 @@ func PrintNetworkLockLogJSONV1(out io.Writer, updates []ipnstate.NetworkLockUpda return enc.Encode(result) } -// toLogMessageV1 converts a [tka.AUM] and [ipnstate.NetworkLockUpdate] to the +// toLogMessageV1 converts a [tka.AUM] and [ipnstate.TailnetLockUpdate] to the // JSON output returned by the CLI. -func toLogMessageV1(aum tka.AUM, update ipnstate.NetworkLockUpdate) logMessageV1 { +func toLogMessageV1(aum tka.AUM, update ipnstate.TailnetLockUpdate) logMessageV1 { expandedAUM := expandedAUMV1{} expandedAUM.MessageKind = aum.MessageKind.String() if len(aum.PrevAUMHash) > 0 { @@ -159,7 +159,7 @@ type expandedAUMV1 struct { } // tkaKeyV1 is the expanded version of a [tka.Key], which describes -// the public components of a key known to network-lock. +// the public components of a key known to tailnet-lock. type tkaKeyV1 struct { Kind string `json:"Kind,omitzero"` diff --git a/cmd/tailscale/cli/jsonoutput/network-lock-status.go b/cmd/tailscale/jsonoutput/tailnet-lock-status.go similarity index 94% rename from cmd/tailscale/cli/jsonoutput/network-lock-status.go rename to cmd/tailscale/jsonoutput/tailnet-lock-status.go index a1d95b871..b5121c5ef 100644 --- a/cmd/tailscale/cli/jsonoutput/network-lock-status.go +++ b/cmd/tailscale/jsonoutput/tailnet-lock-status.go @@ -15,9 +15,9 @@ import ( "tailscale.com/tka" ) -// PrintNetworkLockStatusJSONV1 prints the current Tailnet Lock status +// PrintTailnetLockStatusJSONV1 prints the current Tailnet Lock status // as a JSON object to the CLI, in a stable "v1" format. -func PrintNetworkLockStatusJSONV1(out io.Writer, status *ipnstate.NetworkLockStatus) error { +func PrintTailnetLockStatusJSONV1(out io.Writer, status *ipnstate.TailnetLockStatus) error { responseEnvelope := ResponseEnvelope{ SchemaVersion: "1", } @@ -46,7 +46,7 @@ func PrintNetworkLockStatusJSONV1(out io.Writer, status *ipnstate.NetworkLockSta return enc.Encode(result) } -func toTailnetLockDisabledStatusV1(status *ipnstate.NetworkLockStatus) tailnetLockDisabledStatusV1 { +func toTailnetLockDisabledStatusV1(status *ipnstate.TailnetLockStatus) tailnetLockDisabledStatusV1 { out := tailnetLockDisabledStatusV1{ tailnetLockStatusV1Base: tailnetLockStatusV1Base{ Enabled: status.Enabled, @@ -61,7 +61,7 @@ func toTailnetLockDisabledStatusV1(status *ipnstate.NetworkLockStatus) tailnetLo return out } -func toTailnetLockEnabledStatusV1(status *ipnstate.NetworkLockStatus) tailnetLockEnabledStatusV1 { +func toTailnetLockEnabledStatusV1(status *ipnstate.TailnetLockStatus) tailnetLockEnabledStatusV1 { out := tailnetLockEnabledStatusV1{ tailnetLockStatusV1Base: tailnetLockStatusV1Base{ Enabled: status.Enabled, @@ -116,7 +116,7 @@ type tailnetLockStatusV1Base struct { // Enabled is true if Tailnet Lock is enabled. Enabled bool - // PublicKey describes the node's network-lock public key. + // PublicKey describes the node's tailnet-lock public key. PublicKey string `json:"PublicKey,omitzero"` // NodeKey describes the node's current node-key. This field is not @@ -144,11 +144,11 @@ type tailnetLockEnabledStatusV1 struct { NodeKeySignature *tkaNodeKeySignatureV1 // TrustedKeys describes the keys currently trusted to make changes - // to network-lock. + // to tailnet-lock. TrustedKeys []tkaKeyV1 // VisiblePeers describes peers which are visible in the netmap that - // have valid Tailnet Lock signatures signatures. + // have valid Tailnet Lock signatures. VisiblePeers []tkaTrustedPeerV1 // FilteredPeers describes peers which were removed from the netmap diff --git a/cmd/tailscale/tsdnsjsonv0/doc.go b/cmd/tailscale/tsdnsjsonv0/doc.go new file mode 100644 index 000000000..6017e157c --- /dev/null +++ b/cmd/tailscale/tsdnsjsonv0/doc.go @@ -0,0 +1,14 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package tsdnsjsonv0 provides types for unmarshalling the JSON output of the +// "tailscale dns --json" command: +// +// - [QueryResponse] will unmarshal the output of "tailscale dns query --json=1" +// - [StatusResponse] will unmarshal the output of "tailscale dns status --json=1". +// +// # WARNING: unstable +// +// Format is "v0" and is subject to change. +// There is no guarantee of backwards or forwards compatibility. +package tsdnsjsonv0 diff --git a/cmd/tailscale/tsdnsjsonv0/example_dnsqueryresult_test.go b/cmd/tailscale/tsdnsjsonv0/example_dnsqueryresult_test.go new file mode 100644 index 000000000..718bd18c5 --- /dev/null +++ b/cmd/tailscale/tsdnsjsonv0/example_dnsqueryresult_test.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package tsdnsjsonv0_test + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + + "tailscale.com/cmd/tailscale/tsdnsjsonv0" +) + +func ExampleQueryResponse() { + cmd := exec.Command("tailscale", "dns", "query", "--json", "hello.ts.net") + out, err := cmd.Output() + if err != nil { + if err, ok := errors.AsType[*exec.ExitError](err); ok { + fmt.Fprintf(os.Stderr, "%s", err.Stderr) + } + panic(err) + } + + var resp tsdnsjsonv0.QueryResponse + if err := json.Unmarshal(out, &resp); err != nil { + panic(err) + } + fmt.Printf("{type: %s, name: %q}\n", resp.QueryType, resp.Name) +} diff --git a/cmd/tailscale/tsdnsjsonv0/example_dnsstatusresult_test.go b/cmd/tailscale/tsdnsjsonv0/example_dnsstatusresult_test.go new file mode 100644 index 000000000..aabe92574 --- /dev/null +++ b/cmd/tailscale/tsdnsjsonv0/example_dnsstatusresult_test.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package tsdnsjsonv0_test + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + + "tailscale.com/cmd/tailscale/tsdnsjsonv0" +) + +func ExampleStatusResponse() { + cmd := exec.Command("tailscale", "dns", "status", "--json") + out, err := cmd.Output() + if err != nil { + if err, ok := errors.AsType[*exec.ExitError](err); ok { + fmt.Fprintf(os.Stderr, "%s", err.Stderr) + } + panic(err) + } + + var resp tsdnsjsonv0.StatusResponse + if err := json.Unmarshal(out, &resp); err != nil { + panic(err) + } + fmt.Printf("{accept-dns: %t, resolvers: %q}\n", resp.TailscaleDNS, resp.Resolvers) +} diff --git a/cmd/tailscale/tsdnsjsonv0/query.go b/cmd/tailscale/tsdnsjsonv0/query.go new file mode 100644 index 000000000..311ff1d5c --- /dev/null +++ b/cmd/tailscale/tsdnsjsonv0/query.go @@ -0,0 +1,25 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package tsdnsjsonv0 + +// Answer is a single DNS resource record from a query response. +type Answer struct { + Name string + TTL uint32 + Class string // e.g. "ClassINET" + Type string // e.g. "TypeA", "TypeAAAA" + Body string // human-readable record data +} + +// QueryResponse is the result of a DNS query via the Tailscale +// internal forwarder (100.100.100.100). It is the output of: +// +// $ tailscale dns query --json NAME +type QueryResponse struct { + Name string + QueryType string // e.g. "A", "AAAA" + Resolvers []ResolverInfo `json:",omitzero"` + ResponseCode string // e.g. "RCodeSuccess", "RCodeNameError" + Answers []Answer `json:",omitzero"` +} diff --git a/cmd/tailscale/cli/jsonoutput/dns.go b/cmd/tailscale/tsdnsjsonv0/status.go similarity index 59% rename from cmd/tailscale/cli/jsonoutput/dns.go rename to cmd/tailscale/tsdnsjsonv0/status.go index d9d3cc0bb..1397a36ea 100644 --- a/cmd/tailscale/cli/jsonoutput/dns.go +++ b/cmd/tailscale/tsdnsjsonv0/status.go @@ -1,10 +1,17 @@ // Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -package jsonoutput +package tsdnsjsonv0 -// DNSResolverInfo is the JSON form of [dnstype.Resolver]. -type DNSResolverInfo struct { +// ExtraRecord is the JSON form of [tailscale.com/tailcfg.DNSRecord]. +type ExtraRecord struct { + Name string + Type string `json:",omitempty"` // empty means A or AAAA, depending on Value + Value string // typically an IP address +} + +// ResolverInfo is the JSON form of [tailscale.com/types/dnstype.Resolver]. +type ResolverInfo struct { // Addr is a plain IP, IP:port, DoH URL, or HTTP-over-WireGuard URL. Addr string @@ -13,16 +20,9 @@ type DNSResolverInfo struct { BootstrapResolution []string `json:",omitempty"` } -// DNSExtraRecord is the JSON form of [tailcfg.DNSRecord]. -type DNSExtraRecord struct { - Name string - Type string `json:",omitempty"` // empty means A or AAAA, depending on Value - Value string // typically an IP address -} - -// DNSSystemConfig is the OS DNS configuration as observed by Tailscale, -// mirroring [net/dns.OSConfig]. -type DNSSystemConfig struct { +// SystemConfig is the OS DNS configuration as observed by Tailscale, +// mirroring [tailscale.com/net/dns.OSConfig]. +type SystemConfig struct { Nameservers []string `json:",omitzero"` SearchDomains []string `json:",omitzero"` @@ -32,9 +32,10 @@ type DNSSystemConfig struct { MatchDomains []string `json:",omitzero"` } -// DNSTailnetInfo describes MagicDNS configuration for the tailnet, -// combining [ipnstate.TailnetStatus] and [ipnstate.PeerStatus]. -type DNSTailnetInfo struct { +// TailnetInfo describes MagicDNS configuration for the tailnet, +// combining [tailscale.com/ipn/ipnstate.TailnetStatus] +// and [tailscale.com/ipn/ipnstate.PeerStatus]. +type TailnetInfo struct { // MagicDNSEnabled is whether MagicDNS is enabled for the // tailnet. The device may still not use it if // --accept-dns=false. @@ -49,28 +50,30 @@ type DNSTailnetInfo struct { SelfDNSName string `json:",omitempty"` } -// DNSStatusResult is the full DNS status collected from the local -// Tailscale daemon. -type DNSStatusResult struct { +// StatusResponse is the full DNS status collected from the local +// Tailscale daemon. It is the output of: +// +// $ tailscale dns status --json +type StatusResponse struct { // TailscaleDNS is whether the Tailscale DNS configuration is // installed on this device (the --accept-dns setting). TailscaleDNS bool // CurrentTailnet describes MagicDNS configuration for the tailnet. - CurrentTailnet *DNSTailnetInfo `json:",omitzero"` // nil if not connected + CurrentTailnet *TailnetInfo `json:",omitzero"` // nil if not connected // Resolvers are the DNS resolvers, in preference order. If // empty, the system defaults are used. - Resolvers []DNSResolverInfo `json:",omitzero"` + Resolvers []ResolverInfo `json:",omitzero"` // SplitDNSRoutes maps domain suffixes to dedicated resolvers. // An empty resolver slice means the suffix is handled by // Tailscale's built-in resolver (100.100.100.100). - SplitDNSRoutes map[string][]DNSResolverInfo `json:",omitzero"` + SplitDNSRoutes map[string][]ResolverInfo `json:",omitzero"` // FallbackResolvers are like Resolvers but only used when // split DNS needs explicit default resolvers. - FallbackResolvers []DNSResolverInfo `json:",omitzero"` + FallbackResolvers []ResolverInfo `json:",omitzero"` SearchDomains []string `json:",omitzero"` @@ -84,7 +87,7 @@ type DNSStatusResult struct { CertDomains []string `json:",omitzero"` // ExtraRecords contains extra DNS records in the MagicDNS config. - ExtraRecords []DNSExtraRecord `json:",omitzero"` + ExtraRecords []ExtraRecord `json:",omitzero"` // ExitNodeFilteredSet are DNS suffixes this node won't resolve // when acting as an exit node DNS proxy. Period-prefixed @@ -92,25 +95,6 @@ type DNSStatusResult struct { // lowercase, no trailing dots. ExitNodeFilteredSet []string `json:",omitzero"` - SystemDNS *DNSSystemConfig `json:",omitzero"` // nil if unavailable - SystemDNSError string `json:",omitempty"` -} - -// DNSAnswer is a single DNS resource record from a query response. -type DNSAnswer struct { - Name string - TTL uint32 - Class string // e.g. "ClassINET" - Type string // e.g. "TypeA", "TypeAAAA" - Body string // human-readable record data -} - -// DNSQueryResult is the result of a DNS query via the Tailscale -// internal forwarder (100.100.100.100). -type DNSQueryResult struct { - Name string - QueryType string // e.g. "A", "AAAA" - Resolvers []DNSResolverInfo `json:",omitzero"` - ResponseCode string // e.g. "RCodeSuccess", "RCodeNameError" - Answers []DNSAnswer `json:",omitzero"` + SystemDNS *SystemConfig `json:",omitzero"` // nil if unavailable + SystemDNSError string `json:",omitempty"` } diff --git a/cmd/tailscale/tsroutecheckjsonv0/tsroutecheck.go b/cmd/tailscale/tsroutecheckjsonv0/tsroutecheck.go new file mode 100644 index 000000000..f9dd89348 --- /dev/null +++ b/cmd/tailscale/tsroutecheckjsonv0/tsroutecheck.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_routecheck + +// Package tsroutecheckjsonv0 provides types for unmarshalling the JSON output of the +// "tailscale routecheck --json" command: +// +// - [ReportResponse] will unmarshal the output of "tailscale routecheck --json" +// +// # WARNING: unstable +// +// Format is "v0" and is subject to change. +// There is no guarantee of backwards or forwards compatibility. +package tsroutecheckjsonv0 + +import ( + "time" + + "tailscale.com/net/routecheck" +) + +// ReportResponse is the JSON form of [routecheck.Report]. +// Experimental: This output is not yet stable: tailscale/tailscale#17366. +type ReportResponse struct { + Done time.Time `json:"done"` + Routes routecheck.RoutablePrefixes `json:"routes"` +} diff --git a/cmd/tailscaled/depaware-min.txt b/cmd/tailscaled/depaware-min.txt index 94ebf144b..e57c3bbb6 100644 --- a/cmd/tailscaled/depaware-min.txt +++ b/cmd/tailscaled/depaware-min.txt @@ -15,6 +15,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/v1 from tailscale.com/types/logger github.com/golang/groupcache/lru from tailscale.com/net/dnscache 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink @@ -62,6 +63,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/feature/buildfeatures from tailscale.com/cmd/tailscaled+ tailscale.com/feature/condlite/expvar from tailscale.com/wgengine/magicsock tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled + tailscale.com/feature/condregister/netlog from tailscale.com/feature/condregister tailscale.com/feature/condregister/portmapper from tailscale.com/feature/condregister tailscale.com/feature/condregister/useproxy from tailscale.com/feature/condregister tailscale.com/health from tailscale.com/control/controlclient+ @@ -95,7 +97,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/flowtrack from tailscale.com/wgengine/filter tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ tailscale.com/net/netaddr from tailscale.com/ipn+ - tailscale.com/net/netcheck from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/netcheck from tailscale.com/wgengine/magicsock tailscale.com/net/neterror from tailscale.com/net/batching+ tailscale.com/net/netkernelconf from tailscale.com/ipn/ipnlocal tailscale.com/net/netknob from tailscale.com/logpolicy+ @@ -107,11 +109,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/packet/checksum from tailscale.com/net/tstun tailscale.com/net/ping from tailscale.com/net/netcheck+ tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ + tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/net/netcheck+ tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/ipn/ipnlocal tailscale.com/net/tsaddr from tailscale.com/ipn+ tailscale.com/net/tsdial from tailscale.com/cmd/tailscaled+ tailscale.com/net/tstun from tailscale.com/cmd/tailscaled+ @@ -139,7 +144,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/types/lazy from tailscale.com/hostinfo+ tailscale.com/types/logger from tailscale.com/appc+ tailscale.com/types/logid from tailscale.com/cmd/tailscaled+ - tailscale.com/types/mapx from tailscale.com/ipn/ipnext + tailscale.com/types/mapx from tailscale.com/ipn/ipnext+ tailscale.com/types/netlogfunc from tailscale.com/net/tstun+ tailscale.com/types/netmap from tailscale.com/control/controlclient+ tailscale.com/types/nettype from tailscale.com/net/batching+ @@ -158,6 +163,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/cloudenv from tailscale.com/hostinfo+ tailscale.com/util/cloudinfo from tailscale.com/wgengine/magicsock tailscale.com/util/ctxkey from tailscale.com/client/tailscale/apitype+ + tailscale.com/util/def from tailscale.com/ipn/localapi tailscale.com/util/dnsname from tailscale.com/appc+ tailscale.com/util/eventbus from tailscale.com/control/controlclient+ tailscale.com/util/execqueue from tailscale.com/appc+ @@ -192,11 +198,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ 💣 tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/netlog from tailscale.com/wgengine tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ tailscale.com/wgengine/router from tailscale.com/cmd/tailscaled+ tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal 💣 tailscale.com/wgengine/wgint from tailscale.com/wgengine+ tailscale.com/wgengine/wglog from tailscale.com/wgengine golang.org/x/crypto/blake2b from golang.org/x/crypto/nacl/box @@ -219,7 +223,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/net/icmp from tailscale.com/net/ping golang.org/x/net/idna from golang.org/x/net/http/httpguts golang.org/x/net/internal/iana from golang.org/x/net/icmp+ - golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ golang.org/x/net/ipv4 from github.com/tailscale/wireguard-go/conn+ golang.org/x/net/ipv6 from github.com/tailscale/wireguard-go/conn+ golang.org/x/sync/errgroup from github.com/mdlayher/socket @@ -334,6 +338,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de fmt from compress/flate+ hash from crypto+ hash/crc32 from compress/gzip+ + hash/fnv from tailscale.com/net/traffic hash/maphash from go4.org/mem html from tailscale.com/ipn/ipnlocal+ internal/abi from hash/maphash+ diff --git a/cmd/tailscaled/depaware-minbox.txt b/cmd/tailscaled/depaware-minbox.txt index e518613f8..cf34705e6 100644 --- a/cmd/tailscaled/depaware-minbox.txt +++ b/cmd/tailscaled/depaware-minbox.txt @@ -15,6 +15,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/v1 from tailscale.com/types/logger github.com/golang/groupcache/lru from tailscale.com/net/dnscache 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink @@ -54,7 +55,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/cmd/tailscale/cli from tailscale.com/cmd/tailscaled tailscale.com/cmd/tailscale/cli/ffcomplete from tailscale.com/cmd/tailscale/cli tailscale.com/cmd/tailscale/cli/ffcomplete/internal from tailscale.com/cmd/tailscale/cli/ffcomplete - tailscale.com/cmd/tailscale/cli/jsonoutput from tailscale.com/cmd/tailscale/cli + tailscale.com/cmd/tailscale/tsdnsjsonv0 from tailscale.com/cmd/tailscale/cli tailscale.com/cmd/tailscaled/childproc from tailscale.com/cmd/tailscaled tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ tailscale.com/control/controlclient from tailscale.com/cmd/tailscaled+ @@ -75,6 +76,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled tailscale.com/feature/condregister/awsparamstore from tailscale.com/cmd/tailscale/cli tailscale.com/feature/condregister/identityfederation from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/netlog from tailscale.com/feature/condregister tailscale.com/feature/condregister/oauthkey from tailscale.com/cmd/tailscale/cli tailscale.com/feature/condregister/portmapper from tailscale.com/feature/condregister+ tailscale.com/feature/condregister/useproxy from tailscale.com/cmd/tailscale/cli+ @@ -112,7 +114,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/flowtrack from tailscale.com/wgengine/filter tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ tailscale.com/net/netaddr from tailscale.com/ipn+ - tailscale.com/net/netcheck from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/netcheck from tailscale.com/cmd/tailscale/cli+ tailscale.com/net/neterror from tailscale.com/net/batching+ tailscale.com/net/netkernelconf from tailscale.com/ipn/ipnlocal tailscale.com/net/netknob from tailscale.com/logpolicy+ @@ -124,11 +126,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/packet/checksum from tailscale.com/net/tstun tailscale.com/net/ping from tailscale.com/net/netcheck+ tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ + tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/net/netcheck+ tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/ipn/ipnlocal tailscale.com/net/tsaddr from tailscale.com/ipn+ tailscale.com/net/tsdial from tailscale.com/cmd/tailscaled+ tailscale.com/net/tstun from tailscale.com/cmd/tailscaled+ @@ -158,7 +163,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/types/lazy from tailscale.com/hostinfo+ tailscale.com/types/logger from tailscale.com/appc+ tailscale.com/types/logid from tailscale.com/cmd/tailscaled+ - tailscale.com/types/mapx from tailscale.com/ipn/ipnext + tailscale.com/types/mapx from tailscale.com/ipn/ipnext+ tailscale.com/types/netlogfunc from tailscale.com/net/tstun+ tailscale.com/types/netmap from tailscale.com/control/controlclient+ tailscale.com/types/nettype from tailscale.com/net/batching+ @@ -177,6 +182,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/cloudenv from tailscale.com/hostinfo+ tailscale.com/util/cloudinfo from tailscale.com/wgengine/magicsock tailscale.com/util/ctxkey from tailscale.com/client/tailscale/apitype+ + tailscale.com/util/def from tailscale.com/ipn/localapi tailscale.com/util/dnsname from tailscale.com/appc+ tailscale.com/util/eventbus from tailscale.com/client/local+ tailscale.com/util/execqueue from tailscale.com/appc+ @@ -213,11 +219,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ 💣 tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/netlog from tailscale.com/wgengine tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ tailscale.com/wgengine/router from tailscale.com/cmd/tailscaled+ tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal 💣 tailscale.com/wgengine/wgint from tailscale.com/wgengine+ tailscale.com/wgengine/wglog from tailscale.com/wgengine golang.org/x/crypto/blake2b from golang.org/x/crypto/nacl/box @@ -240,7 +244,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/net/icmp from tailscale.com/net/ping golang.org/x/net/idna from golang.org/x/net/http/httpguts+ golang.org/x/net/internal/iana from golang.org/x/net/icmp+ - golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ golang.org/x/net/ipv4 from github.com/tailscale/wireguard-go/conn+ golang.org/x/net/ipv6 from github.com/tailscale/wireguard-go/conn+ golang.org/x/sync/errgroup from github.com/mdlayher/socket @@ -355,6 +359,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de fmt from compress/flate+ hash from crypto+ hash/crc32 from compress/gzip+ + hash/fnv from tailscale.com/net/traffic hash/maphash from go4.org/mem html from tailscale.com/ipn/ipnlocal+ internal/abi from hash/maphash+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 678d72560..27fe06551 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -20,10 +20,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/aws-sdk-go-v2/aws/signer/v4 from github.com/aws/aws-sdk-go-v2/service/internal/presigned-url+ L github.com/aws/aws-sdk-go-v2/aws/transport/http from github.com/aws/aws-sdk-go-v2/config+ L github.com/aws/aws-sdk-go-v2/config from tailscale.com/ipn/store/awsstore + L github.com/aws/aws-sdk-go-v2/config/internal/ini from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds + L github.com/aws/aws-sdk-go-v2/credentials/logincreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/processcreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/ssocreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/stscreds from github.com/aws/aws-sdk-go-v2/config @@ -36,17 +38,21 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/aws-sdk-go-v2/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ L github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn from github.com/aws/aws-sdk-go-v2/service/ssm+ L github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+ - L github.com/aws/aws-sdk-go-v2/internal/ini from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/internal/middleware from github.com/aws/aws-sdk-go-v2/service/sso+ L github.com/aws/aws-sdk-go-v2/internal/rand from github.com/aws/aws-sdk-go-v2/aws+ L github.com/aws/aws-sdk-go-v2/internal/sdk from github.com/aws/aws-sdk-go-v2/aws+ L github.com/aws/aws-sdk-go-v2/internal/sdkio from github.com/aws/aws-sdk-go-v2/credentials/processcreds L github.com/aws/aws-sdk-go-v2/internal/shareddefaults from github.com/aws/aws-sdk-go-v2/config+ - L github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 + L github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4+ L github.com/aws/aws-sdk-go-v2/internal/sync/singleflight from github.com/aws/aws-sdk-go-v2/aws L github.com/aws/aws-sdk-go-v2/internal/timeconv from github.com/aws/aws-sdk-go-v2/aws/retry + L github.com/aws/aws-sdk-go-v2/internal/v4a from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/internal/v4a/internal/crypto from github.com/aws/aws-sdk-go-v2/internal/v4a + L github.com/aws/aws-sdk-go-v2/internal/v4a/internal/v4 from github.com/aws/aws-sdk-go-v2/internal/v4a L github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts L github.com/aws/aws-sdk-go-v2/service/internal/presigned-url from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/service/signin from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/service/signin/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/signin + L github.com/aws/aws-sdk-go-v2/service/signin/types from github.com/aws/aws-sdk-go-v2/credentials/logincreds+ L github.com/aws/aws-sdk-go-v2/service/ssm from tailscale.com/ipn/store/awsstore L github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm L github.com/aws/aws-sdk-go-v2/service/ssm/types from github.com/aws/aws-sdk-go-v2/service/ssm+ @@ -59,7 +65,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/aws-sdk-go-v2/service/sts from github.com/aws/aws-sdk-go-v2/config+ L github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sts L github.com/aws/aws-sdk-go-v2/service/sts/types from github.com/aws/aws-sdk-go-v2/credentials/stscreds+ - L github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ + L 💣 github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ L github.com/aws/smithy-go/auth from github.com/aws/aws-sdk-go-v2/internal/auth+ L github.com/aws/smithy-go/auth/bearer from github.com/aws/aws-sdk-go-v2/aws+ L github.com/aws/smithy-go/context from github.com/aws/smithy-go/auth/bearer @@ -69,7 +75,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/smithy-go/encoding/json from github.com/aws/aws-sdk-go-v2/service/ssm+ L github.com/aws/smithy-go/encoding/xml from github.com/aws/aws-sdk-go-v2/service/sts L github.com/aws/smithy-go/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/smithy-go/endpoints/private/rulesfn from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/smithy-go/endpoints/private/rulesfn from github.com/aws/aws-sdk-go-v2/service/sts+ + L github.com/aws/smithy-go/eventstream from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+ L github.com/aws/smithy-go/internal/sync/singleflight from github.com/aws/smithy-go/auth/bearer L github.com/aws/smithy-go/io from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+ L github.com/aws/smithy-go/logging from github.com/aws/aws-sdk-go-v2/aws+ @@ -78,15 +85,16 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/smithy-go/private/requestcompression from github.com/aws/aws-sdk-go-v2/config L github.com/aws/smithy-go/ptr from github.com/aws/aws-sdk-go-v2/aws+ L github.com/aws/smithy-go/rand from github.com/aws/aws-sdk-go-v2/aws/middleware+ + L github.com/aws/smithy-go/sync from github.com/aws/smithy-go/transport/http L github.com/aws/smithy-go/time from github.com/aws/aws-sdk-go-v2/service/ssm+ L github.com/aws/smithy-go/tracing from github.com/aws/aws-sdk-go-v2/aws/middleware+ + L github.com/aws/smithy-go/traits from github.com/aws/smithy-go/eventstream L github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws/middleware+ L github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http L github.com/aws/smithy-go/waiter from github.com/aws/aws-sdk-go-v2/service/ssm github.com/coder/websocket from tailscale.com/util/eventbus github.com/coder/websocket/internal/errd from github.com/coder/websocket github.com/coder/websocket/internal/util from github.com/coder/websocket - github.com/coder/websocket/internal/xsync from github.com/coder/websocket L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw github.com/creachadair/msync/trigger from tailscale.com/logtail LD 💣 github.com/creack/pty from tailscale.com/ssh/tailssh @@ -112,6 +120,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json/jsontext+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json/jsontext+ github.com/go-json-experiment/json/jsontext from tailscale.com/logtail+ + github.com/go-json-experiment/json/v1 from tailscale.com/feature/routecheck+ W 💣 github.com/go-ole/go-ole from github.com/go-ole/go-ole/oleutil+ W 💣 github.com/go-ole/go-ole/oleutil from tailscale.com/wgengine/winnet L 💣 github.com/godbus/dbus/v5 from tailscale.com/net/dns+ @@ -130,7 +139,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/google/nftables/expr from github.com/google/nftables+ L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ - DW github.com/google/uuid from tailscale.com/clientupdate+ + W github.com/google/uuid from tailscale.com/clientupdate github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ github.com/huin/goupnp from github.com/huin/goupnp/dcps/internetgateway2+ github.com/huin/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper @@ -173,16 +182,15 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/pires/go-proxyproto from tailscale.com/ipn/ipnlocal LD github.com/pkg/sftp from tailscale.com/ssh/tailssh LD github.com/pkg/sftp/internal/encoding/ssh/filexfer from github.com/pkg/sftp - D github.com/prometheus-community/pro-bing from tailscale.com/wgengine/netstack L 💣 github.com/safchain/ethtool from tailscale.com/net/netkernelconf+ - W 💣 github.com/tailscale/certstore from tailscale.com/control/controlclient + DW 💣 github.com/tailscale/certstore from tailscale.com/control/controlclient LD github.com/tailscale/gliderssh from tailscale.com/ssh/tailssh W 💣 github.com/tailscale/go-winio from tailscale.com/safesocket W 💣 github.com/tailscale/go-winio/internal/fs from github.com/tailscale/go-winio W 💣 github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - github.com/tailscale/hujson from tailscale.com/ipn/conffile + github.com/tailscale/hujson from tailscale.com/ipn/conffile+ L 💣 github.com/tailscale/netlink from tailscale.com/net/routetable+ L 💣 github.com/tailscale/netlink/nl from github.com/tailscale/netlink LD github.com/tailscale/peercred from tailscale.com/ipn/ipnauth @@ -253,12 +261,13 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com from tailscale.com/version tailscale.com/appc from tailscale.com/ipn/ipnlocal+ 💣 tailscale.com/atomicfile from tailscale.com/ipn+ - LD tailscale.com/chirp from tailscale.com/cmd/tailscaled + LD tailscale.com/chirp from tailscale.com/feature/bird tailscale.com/client/local from tailscale.com/client/web+ tailscale.com/client/tailscale/apitype from tailscale.com/client/local+ tailscale.com/client/web from tailscale.com/ipn/ipnlocal tailscale.com/clientupdate from tailscale.com/feature/clientupdate LW tailscale.com/clientupdate/distsign from tailscale.com/clientupdate + tailscale.com/cmd/tailscale/jsonoutput from tailscale.com/feature/tailnetlock tailscale.com/cmd/tailscaled/childproc from tailscale.com/cmd/tailscaled+ tailscale.com/cmd/tailscaled/tailscaledhooks from tailscale.com/cmd/tailscaled+ tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ @@ -284,13 +293,18 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/feature/wakeonlan+ tailscale.com/feature/ace from tailscale.com/feature/condregister + tailscale.com/feature/acme from tailscale.com/feature/condregister tailscale.com/feature/appconnectors from tailscale.com/feature/condregister + LD tailscale.com/feature/bird from tailscale.com/feature/condregister tailscale.com/feature/buildfeatures from tailscale.com/wgengine/magicsock+ tailscale.com/feature/c2n from tailscale.com/feature/condregister + tailscale.com/feature/captiveportal from tailscale.com/feature/condregister + tailscale.com/feature/captiveportal/netcheckhook from tailscale.com/feature/captiveportal tailscale.com/feature/capture from tailscale.com/feature/condregister tailscale.com/feature/clientupdate from tailscale.com/feature/condregister tailscale.com/feature/condlite/expvar from tailscale.com/wgengine/magicsock tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled + tailscale.com/feature/condregister/netlog from tailscale.com/feature/condregister tailscale.com/feature/condregister/portmapper from tailscale.com/feature/condregister tailscale.com/feature/condregister/useproxy from tailscale.com/feature/condregister tailscale.com/feature/conn25 from tailscale.com/feature/condregister @@ -299,14 +313,22 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/feature/drive from tailscale.com/feature/condregister L tailscale.com/feature/linkspeed from tailscale.com/feature/condregister L tailscale.com/feature/linuxdnsfight from tailscale.com/feature/condregister + tailscale.com/feature/netlog from tailscale.com/feature/condregister/netlog tailscale.com/feature/portlist from tailscale.com/feature/condregister tailscale.com/feature/portmapper from tailscale.com/feature/condregister/portmapper tailscale.com/feature/posture from tailscale.com/feature/condregister tailscale.com/feature/relayserver from tailscale.com/feature/condregister + tailscale.com/feature/remoteconfig from tailscale.com/feature/condregister + tailscale.com/feature/routecheck from tailscale.com/feature/condregister + tailscale.com/feature/runtimemetrics from tailscale.com/feature/condregister L tailscale.com/feature/sdnotify from tailscale.com/feature/condregister + tailscale.com/feature/serviceclientprefs from tailscale.com/feature/condregister + tailscale.com/feature/serviceclientprefs/serviceclient from tailscale.com/client/local+ LD tailscale.com/feature/ssh from tailscale.com/cmd/tailscaled - tailscale.com/feature/syspolicy from tailscale.com/feature/condregister+ + L tailscale.com/feature/syslog from tailscale.com/feature/condregister + tailscale.com/feature/syspolicy from tailscale.com/feature/condregister tailscale.com/feature/taildrop from tailscale.com/feature/condregister + tailscale.com/feature/tailnetlock from tailscale.com/feature/condregister L tailscale.com/feature/tap from tailscale.com/feature/condregister tailscale.com/feature/tpm from tailscale.com/feature/condregister L 💣 tailscale.com/feature/tundevstats from tailscale.com/feature/condregister @@ -344,7 +366,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/ace from tailscale.com/feature/ace tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+ 💣 tailscale.com/net/batching from tailscale.com/wgengine/magicsock+ - tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/captivedetection from tailscale.com/feature/captiveportal+ tailscale.com/net/dns from tailscale.com/cmd/tailscaled+ tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ tailscale.com/net/dns/resolvconffile from tailscale.com/net/dns+ @@ -369,6 +391,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/portmapper from tailscale.com/feature/portmapper+ tailscale.com/net/portmapper/portmappertype from tailscale.com/feature/portmapper+ tailscale.com/net/proxymux from tailscale.com/cmd/tailscaled + tailscale.com/net/routecheck from tailscale.com/feature/routecheck+ + tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ tailscale.com/net/routetable from tailscale.com/doctor/routetable 💣 tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock+ tailscale.com/net/socks5 from tailscale.com/cmd/tailscaled @@ -376,6 +401,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/stun from tailscale.com/ipn/localapi+ tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/ipn/ipnlocal+ tailscale.com/net/tsaddr from tailscale.com/client/web+ tailscale.com/net/tsdial from tailscale.com/cmd/tailscaled+ 💣 tailscale.com/net/tshttpproxy from tailscale.com/feature/useproxy @@ -393,7 +419,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de LD 💣 tailscale.com/ssh/tailssh from tailscale.com/feature/ssh tailscale.com/syncs from tailscale.com/cmd/tailscaled+ tailscale.com/tailcfg from tailscale.com/client/local+ - tailscale.com/tempfork/acme from tailscale.com/ipn/ipnlocal + tailscale.com/tempfork/acme from tailscale.com/feature/acme tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock tailscale.com/tempfork/httprec from tailscale.com/feature/c2n tailscale.com/tka from tailscale.com/client/local+ @@ -402,7 +428,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/tstime from tailscale.com/control/controlclient+ tailscale.com/tstime/mono from tailscale.com/net/tstun+ tailscale.com/tstime/rate from tailscale.com/wgengine/filter - tailscale.com/tsweb from tailscale.com/util/eventbus + tailscale.com/tsweb from tailscale.com/util/eventbus+ tailscale.com/tsweb/varz from tailscale.com/cmd/tailscaled+ tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+ tailscale.com/types/bools from tailscale.com/wgengine/netlog @@ -415,7 +441,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/types/lazy from tailscale.com/ipn/ipnlocal+ tailscale.com/types/logger from tailscale.com/appc+ tailscale.com/types/logid from tailscale.com/cmd/tailscaled+ - tailscale.com/types/mapx from tailscale.com/ipn/ipnext + tailscale.com/types/mapx from tailscale.com/ipn/ipnext+ tailscale.com/types/netlogfunc from tailscale.com/net/tstun+ tailscale.com/types/netlogtype from tailscale.com/wgengine/netlog tailscale.com/types/netmap from tailscale.com/control/controlclient+ @@ -437,6 +463,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/cmpver from tailscale.com/net/dns+ tailscale.com/util/ctxkey from tailscale.com/ipn/ipnlocal+ 💣 tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting + tailscale.com/util/def from tailscale.com/feature/debugportmapper+ L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics+ tailscale.com/util/dnsname from tailscale.com/appc+ tailscale.com/util/eventbus from tailscale.com/tsd+ @@ -456,7 +483,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W 💣 tailscale.com/util/osdiag/internal/wsc from tailscale.com/util/osdiag tailscale.com/util/osshare from tailscale.com/cmd/tailscaled+ tailscale.com/util/osuser from tailscale.com/ipn/ipnlocal+ - tailscale.com/util/progresstracking from tailscale.com/feature/taildrop + tailscale.com/util/progresstracking from tailscale.com/feature/taildrop+ tailscale.com/util/race from tailscale.com/net/dns/resolver tailscale.com/util/racebuild from tailscale.com/logpolicy tailscale.com/util/rands from tailscale.com/ipn/ipnlocal+ @@ -464,7 +491,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/set from tailscale.com/control/controlclient+ tailscale.com/util/singleflight from tailscale.com/control/controlclient+ tailscale.com/util/slicesx from tailscale.com/appc+ - tailscale.com/util/syspolicy from tailscale.com/feature/syspolicy + tailscale.com/util/syspolicy from tailscale.com/feature/syspolicy+ tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy/setting+ tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy/internal/metrics+ tailscale.com/util/syspolicy/internal/metrics from tailscale.com/util/syspolicy/source @@ -491,13 +518,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ 💣 tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/netlog from tailscale.com/wgengine + tailscale.com/wgengine/netlog from tailscale.com/feature/netlog tailscale.com/wgengine/netstack from tailscale.com/cmd/tailscaled tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ tailscale.com/wgengine/router from tailscale.com/cmd/tailscaled+ tailscale.com/wgengine/router/osrouter from tailscale.com/feature/condregister tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ - tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal 💣 tailscale.com/wgengine/wgint from tailscale.com/wgengine+ tailscale.com/wgengine/wglog from tailscale.com/wgengine W 💣 tailscale.com/wgengine/winnet from tailscale.com/wgengine/router/osrouter @@ -507,7 +533,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/chacha20poly1305 from github.com/tailscale/wireguard-go/device+ - golang.org/x/crypto/cryptobyte from tailscale.com/feature/tpm + golang.org/x/crypto/cryptobyte from tailscale.com/feature/tpm+ golang.org/x/crypto/cryptobyte/asn1 from golang.org/x/crypto/cryptobyte+ golang.org/x/crypto/curve25519 from golang.org/x/crypto/ssh+ golang.org/x/crypto/hkdf from tailscale.com/control/controlbase @@ -525,13 +551,13 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/net/dns/dnsmessage from tailscale.com/appc+ golang.org/x/net/http/httpguts from tailscale.com/ipn/ipnlocal golang.org/x/net/http/httpproxy from tailscale.com/net/tshttpproxy - golang.org/x/net/icmp from tailscale.com/net/ping+ + golang.org/x/net/icmp from tailscale.com/net/ping golang.org/x/net/idna from golang.org/x/net/http/httpguts+ golang.org/x/net/internal/iana from golang.org/x/net/icmp+ - golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ golang.org/x/net/internal/socks from golang.org/x/net/proxy - golang.org/x/net/ipv4 from github.com/prometheus-community/pro-bing+ - golang.org/x/net/ipv6 from github.com/prometheus-community/pro-bing+ + golang.org/x/net/ipv4 from github.com/tailscale/wireguard-go/conn+ + golang.org/x/net/ipv6 from github.com/tailscale/wireguard-go/conn+ golang.org/x/net/proxy from tailscale.com/net/netns D golang.org/x/net/route from tailscale.com/net/netmon+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ @@ -547,7 +573,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/text/secure/bidirule from golang.org/x/net/idna golang.org/x/text/transform from golang.org/x/text/secure/bidirule+ golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ - golang.org/x/text/unicode/norm from golang.org/x/net/idna + golang.org/x/text/unicode/norm from golang.org/x/net/idna+ golang.org/x/time/rate from gvisor.dev/gvisor/pkg/log+ vendor/golang.org/x/crypto/chacha20 from vendor/golang.org/x/crypto/chacha20poly1305 vendor/golang.org/x/crypto/chacha20poly1305 from crypto/hpke+ @@ -566,6 +592,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna archive/tar from tailscale.com/clientupdate + L archive/zip from tailscale.com/clientupdate bufio from compress/flate+ bytes from archive/tar+ cmp from slices+ @@ -642,7 +669,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/x509 from crypto/tls+ D crypto/x509/internal/macos from crypto/x509 crypto/x509/pkix from crypto/x509+ - DW database/sql/driver from github.com/google/uuid + W database/sql/driver from github.com/google/uuid W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/tailscale/web-client-prebuilt+ @@ -662,6 +689,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de hash from compress/zlib+ hash/adler32 from compress/zlib+ hash/crc32 from compress/gzip+ + hash/fnv from tailscale.com/net/traffic hash/maphash from go4.org/mem html from html/template+ html/template from tailscale.com/util/eventbus @@ -716,11 +744,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de internal/unsafeheader from internal/reflectlite+ io from archive/tar+ io/fs from archive/tar+ - io/ioutil from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ + io/ioutil from github.com/aws/aws-sdk-go-v2/service/sso+ iter from maps+ log from expvar+ log/internal from log - LD log/syslog from tailscale.com/ssh/tailssh + LD log/syslog from tailscale.com/ssh/tailssh+ maps from tailscale.com/clientupdate+ math from archive/tar+ math/big from crypto/dsa+ @@ -732,7 +760,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de mime/quotedprintable from mime/multipart net from crypto/tls+ net/http from expvar+ - net/http/httptrace from github.com/prometheus-community/pro-bing+ + net/http/httptrace from github.com/aws/smithy-go/transport/http+ net/http/httputil from github.com/aws/smithy-go/transport/http+ net/http/internal from net/http+ net/http/internal/ascii from net/http+ @@ -752,6 +780,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de regexp/syntax from regexp runtime from archive/tar+ runtime/debug from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+ + runtime/metrics from tailscale.com/feature/runtimemetrics runtime/pprof from net/http/pprof+ runtime/trace from net/http/pprof slices from tailscale.com/appc+ diff --git a/cmd/tailscaled/deps_test.go b/cmd/tailscaled/deps_test.go index be4f65a7d..a611644d1 100644 --- a/cmd/tailscaled/deps_test.go +++ b/cmd/tailscaled/deps_test.go @@ -13,6 +13,18 @@ import ( "tailscale.com/tstest/deptest" ) +func TestOmitServiceClientPrefs(t *testing.T) { + const msg = "unexpected with ts_omit_serviceclientprefs" + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_serviceclientprefs,ts_include_cli", + BadDeps: map[string]string{ + "tailscale.com/feature/serviceclientprefs": msg, + }, + }.Check(t) +} + func TestOmitSSH(t *testing.T) { const msg = "unexpected with ts_omit_ssh" deptest.DepChecker{ @@ -33,6 +45,21 @@ func TestOmitSSH(t *testing.T) { }.Check(t) } +func TestOmitSyslog(t *testing.T) { + const msg = "unexpected syslog usage with ts_omit_syslog" + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + // Tailscale SSH's incubator also uses log/syslog, so omit + // SSH too to lock down the standard library package. + Tags: "ts_omit_syslog,ts_omit_ssh,ts_include_cli", + BadDeps: map[string]string{ + "log/syslog": msg, + "tailscale.com/feature/syslog": msg, + }, + }.Check(t) +} + func TestOmitSyspolicy(t *testing.T) { const msg = "unexpected syspolicy usage with ts_omit_syspolicy" deptest.DepChecker{ @@ -137,6 +164,20 @@ func TestOmitCaptivePortal(t *testing.T) { }.Check(t) } +func TestOmitBird(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_bird,ts_include_cli", + OnDep: func(dep string) { + switch dep { + case "tailscale.com/chirp", "tailscale.com/feature/bird": + t.Errorf("unexpected dep with ts_omit_bird: %q", dep) + } + }, + }.Check(t) +} + func TestOmitAuth(t *testing.T) { deptest.DepChecker{ GOOS: "linux", @@ -202,6 +243,19 @@ func TestOmitPortlist(t *testing.T) { }.Check(t) } +func TestOmitRouteCheck(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_routecheck,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "routecheck") && !strings.HasSuffix(dep, "/peernode") { + t.Errorf("unexpected dep: %q", dep) + } + }, + }.Check(t) +} + func TestOmitGRO(t *testing.T) { deptest.DepChecker{ GOOS: "linux", diff --git a/cmd/tailscaled/install_darwin.go b/cmd/tailscaled/install_darwin.go index 15d9e5462..4de1f3122 100644 --- a/cmd/tailscaled/install_darwin.go +++ b/cmd/tailscaled/install_darwin.go @@ -123,7 +123,7 @@ func installSystemDaemonDarwin(args []string) (err error) { return err } - // Do not overwrite targetBin with the binary file if it it's already + // Do not overwrite targetBin with the binary file if it's already // pointing to it. This is primarily to handle Homebrew that writes // /usr/local/bin/tailscaled is a symlink to the actual binary. if !same { diff --git a/cmd/tailscaled/netstack.go b/cmd/tailscaled/netstack.go index d896f384f..4e180d36b 100644 --- a/cmd/tailscaled/netstack.go +++ b/cmd/tailscaled/netstack.go @@ -44,11 +44,6 @@ func newNetstack(logf logger.Logf, sys *tsd.System, onlyNetstack bool) (tsd.Nets dialer := sys.Dialer.Get() // must be set by caller already if onlyNetstack { - e := sys.Engine.Get() - dialer.UseNetstackForIP = func(ip netip.Addr) bool { - _, ok := e.PeerForIP(ip) - return ok - } dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { // Note: don't just return ns.DialContextTCP or we'll return // *gonet.TCPConn(nil) instead of a nil interface which trips up diff --git a/cmd/tailscaled/syspolicy.go b/cmd/tailscaled/syspolicy.go new file mode 100644 index 000000000..47e6294fd --- /dev/null +++ b/cmd/tailscaled/syspolicy.go @@ -0,0 +1,54 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_syspolicy + +package main + +import ( + "flag" + "log" + "os" + "path/filepath" + "runtime" + + "tailscale.com/util/syspolicy" +) + +// syspolicyFile is the path to a JSON syspolicy file, set via the +// --syspolicy-file flag. An empty value disables file-based syspolicy. +var syspolicyFile string + +// defaultSyspolicyFile returns the platform-specific default path for the +// --syspolicy-file flag. On Windows it sits next to the rest of Tailscale's +// machine state under %ProgramData%\Tailscale. On every other platform +// (Linux, the BSDs, illumos/Solaris, and tailscaled-without-the-GUI on +// macOS) it uses /etc/tailscale, which is where admin-provided +// configuration is conventionally placed. +// +// On Windows, when the file exists, its values take precedence over the +// HKLM registry-based platform store on a per-key basis (with the registry +// providing fallback values for keys the file does not set), because rsop +// merges later-registered same-scope sources over earlier ones. +func defaultSyspolicyFile() string { + if runtime.GOOS == "windows" { + if pd := os.Getenv("ProgramData"); pd != "" { + return filepath.Join(pd, "Tailscale", "syspolicy.json") + } + return "" + } + return "/etc/tailscale/syspolicy.json" +} + +func init() { + flag.StringVar(&syspolicyFile, "syspolicy-file", defaultSyspolicyFile(), + "path to a JSON syspolicy file applied as a device-scope policy source; empty disables") + loadSyspolicy.Set(func() { + if syspolicyFile == "" { + return + } + if err := syspolicy.LoadJSONPolicyFile("JSONFile", syspolicyFile); err != nil { + log.Printf("%v", err) + } + }) +} diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index fe18731ae..d7c19f51b 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -18,6 +18,7 @@ import ( "log" "net" "net/http" + "net/netip" "os" "os/signal" "path/filepath" @@ -87,6 +88,16 @@ func defaultTunName() string { // See https://github.com/tailscale/tailscale-synology/issues/35 return "tailscale0,userspace-networking" } + if buildfeatures.HasNetstack && distro.Get() == distro.Crostini { + // cros-garcon NULL-derefs on cold-boot netlink interface + // enumeration when tailscale0 is present, preventing the + // Crostini container and ChromeOS Terminal from starting + // cleanly. Default to userspace-networking until the + // upstream ChromiumOS bug is fixed. + // See https://github.com/tailscale/tailscale/issues/12090 + // See https://issuetracker.google.com/issues/517069318 + return "userspace-networking" + } } return "tailscale0" } @@ -131,9 +142,8 @@ var args struct { } var ( - installSystemDaemon func([]string) error // non-nil on some platforms - uninstallSystemDaemon func([]string) error // non-nil on some platforms - createBIRDClient func(string) (wgengine.BIRDClient, error) // non-nil on some platforms + installSystemDaemon func([]string) error // non-nil on some platforms + uninstallSystemDaemon func([]string) error // non-nil on some platforms ) // Note - we use function pointers for subcommands so that subcommands like @@ -173,6 +183,12 @@ var ( hookOutboundProxyListen feature.Hook[func() proxyStartFunc] ) +// loadSyspolicy, if set, loads a JSON-file-backed syspolicy source after +// command-line flags are parsed, using the path from --syspolicy-file. It +// is set when built without ts_omit_syspolicy (see syspolicy.go), and +// unset otherwise. +var loadSyspolicy feature.Hook[func()] + // proxyStartFunc is the type of the function returned by // outboundProxyListen, to start the servers on the Listeners // started by hookOutboundProxyListen. @@ -207,7 +223,7 @@ func main() { } flag.BoolVar(&printVersion, "version", false, "print version information and exit") flag.BoolVar(&args.disableLogs, "no-logs-no-support", false, "disable log uploads; this also disables any technical support") - flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2)") + flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2); prefix with 'optional:' to boot unconfigured when the source is absent instead of failing") if buildfeatures.HasTPM { flag.Var(&args.hardwareAttestation, "hardware-attestation", `use hardware-backed keys to bind node identity to this device when supported by the OS and hardware. Uses TPM 2.0 on Linux and Windows; SecureEnclave on @@ -217,6 +233,9 @@ store state on filesystem.`) if f, ok := hookRegisterOutboundProxyFlags.GetOk(); ok { f() } + if f, ok := feature.HookRegisterLogSinkFlags.GetOk(); ok { + f() + } if runtime.GOOS == "plan9" && os.Getenv("_NETSHELL_CHILD_") != "" { os.Args = []string{"tailscaled", "be-child", "plan9-netshell"} @@ -245,6 +264,10 @@ store state on filesystem.`) } } + if f, ok := feature.HookLogSink.GetOk(); ok { + f() // redirects the default logger (e.g. to syslog) if requested by flags + } + if fd, ok := envknob.LookupInt("TS_PARENT_DEATH_FD"); ok && fd > 2 { go dieOnPipeReadErrorOfFD(fd) } @@ -264,7 +287,7 @@ store state on filesystem.`) log.Fatalf("--socket is required") } - if buildfeatures.HasBird && args.birdSocketPath != "" && createBIRDClient == nil { + if buildfeatures.HasBird && args.birdSocketPath != "" && !wgengine.HookNewBird.IsSet() { log.SetFlags(0) log.Fatalf("--bird-socket is not supported on %s", runtime.GOOS) } @@ -285,6 +308,12 @@ store state on filesystem.`) } } + // If syspolicy is built in, load the JSON syspolicy file (if any) now + // so its settings are visible before anything queries them. + if f, ok := loadSyspolicy.GetOk(); ok { + f() + } + if buildfeatures.HasTPM { handleTPMFlags() } @@ -409,11 +438,24 @@ func run() (err error) { // Parse config, if specified, to fail early if it's invalid. var conf *conffile.Config if args.confFile != "" { - conf, err = conffile.Load(args.confFile) - if err != nil { + // An "optional:" prefix (e.g. "optional:vm:user-data") means it's fine + // for the config source to be absent: boot unconfigured and let the node + // be enrolled interactively instead of failing to start. A config that's + // present but invalid still fails, even when optional. + path := args.confFile + optional := false + if p, ok := strings.CutPrefix(path, "optional:"); ok { + optional, path = true, p + } + conf, err = conffile.Load(path) + switch { + case err == nil: + sys.InitialConfig = conf + case optional && errors.Is(err, conffile.ErrNoConfig): + logf("config: none present at %q; continuing unconfigured", path) + default: return fmt.Errorf("error reading config file: %w", err) } - sys.InitialConfig = conf } var netMon *netmon.Monitor @@ -428,6 +470,10 @@ func run() (err error) { var publicLogID logid.PublicID if buildfeatures.HasLogTail { + logpolicy.GetLogTarget.Set(func() string { + target, _ := sys.PolicyClientOrDefault().GetString(pkey.LogTarget, "") + return target + }) pol := logpolicy.Options{ Collection: logtail.CollectionNode, @@ -460,9 +506,6 @@ func run() (err error) { return nil } - if envknob.Bool("TS_DEBUG_MEMORY") { - logf = logger.RusagePrefixLog(logf) - } logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100) if envknob.Bool("TS_PLEASE_PANIC") { @@ -671,6 +714,12 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID if err != nil { return nil, fmt.Errorf("ipnlocal.NewLocalBackend: %w", err) } + if onlyNetstack { + dialer.UseNetstackForIP = func(ip netip.Addr) bool { + _, ok := lb.PeerForIP(ip) + return ok + } + } lb.SetVarRoot(opts.VarRoot) if logPol != nil { lb.SetLogFlusher(logPol.Logtail.StartFlush) @@ -761,12 +810,8 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo netstackSubnetRouter := onlyNetstack // but mutated later on some platforms netns.SetEnabled(!onlyNetstack) - if args.birdSocketPath != "" && createBIRDClient != nil { - log.Printf("Connecting to BIRD at %s ...", args.birdSocketPath) - conf.BIRDClient, err = createBIRDClient(args.birdSocketPath) - if err != nil { - return false, fmt.Errorf("createBIRDClient: %w", err) - } + if buildfeatures.HasBird && args.birdSocketPath != "" { + conf.BIRDSocket = args.birdSocketPath } if onlyNetstack { if runtime.GOOS == "linux" && distro.Get() == distro.Synology { @@ -828,7 +873,6 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo if err != nil { return onlyNetstack, err } - e = wgengine.NewWatchdog(e) sys.Set(e) sys.NetstackRouter.Set(netstackSubnetRouter) @@ -917,7 +961,9 @@ func handleTPMFlags() { case !args.hardwareAttestation.set: policyHWAttestation, _ := policyclient.Get().GetBoolean(pkey.HardwareAttestation, false) if err := canUseHardwareAttestation(); err != nil { - log.Printf("[unexpected] policy requires hardware attestation, but device does not support it: %v", err) + if policyHWAttestation { + log.Printf("[unexpected] policy requires hardware attestation, but device does not support it: %v", err) + } args.hardwareAttestation.v = false } else { args.hardwareAttestation.v = policyHWAttestation diff --git a/cmd/tailscaled/tailscaled_bird.go b/cmd/tailscaled/tailscaled_bird.go deleted file mode 100644 index c1c32d2bb..000000000 --- a/cmd/tailscaled/tailscaled_bird.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -//go:build go1.19 && (linux || darwin || freebsd || openbsd) && !ts_omit_bird - -package main - -import ( - "tailscale.com/chirp" - "tailscale.com/wgengine" -) - -func init() { - createBIRDClient = func(ctlSocket string) (wgengine.BIRDClient, error) { - return chirp.New(ctlSocket) - } -} diff --git a/cmd/testwrapper/flakytest/flakytest.go b/cmd/testwrapper/flakytest/flakytest.go index 5e1591e81..ff9b8e5bb 100644 --- a/cmd/testwrapper/flakytest/flakytest.go +++ b/cmd/testwrapper/flakytest/flakytest.go @@ -1,9 +1,14 @@ // Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -// Package flakytest contains test helpers for marking a test as flaky. For -// tests run using cmd/testwrapper, a failed flaky test will cause tests to be -// re-run a few time until they succeed or exceed our iteration limit. +// Package flakytest contains test helpers for marking a test as flaky. +// +// Marking a test with [Mark] is not required for cmd/testwrapper to retry +// failed tests; the wrapper retries any failure within a per-test time +// budget and reports a test as flaky if it ever passes on retry. Mark is +// useful for tracking a known-flaky test against a GitHub issue and for the +// TS_SKIP_FLAKY_TESTS skip behavior used to keep CI green when a flake is +// being investigated. package flakytest import ( diff --git a/cmd/testwrapper/testdata/A_baseline/raw.txt b/cmd/testwrapper/testdata/A_baseline/raw.txt new file mode 100644 index 000000000..36bfad518 --- /dev/null +++ b/cmd/testwrapper/testdata/A_baseline/raw.txt @@ -0,0 +1,6 @@ +=== RUN TestOne + a_test.go:6: hello from one +--- PASS: TestOne (0.00s) +=== RUN TestTwo +--- PASS: TestTwo (0.00s) +PASS diff --git a/cmd/testwrapper/testdata/A_baseline/src.go b/cmd/testwrapper/testdata/A_baseline/src.go new file mode 100644 index 000000000..5f90db1dd --- /dev/null +++ b/cmd/testwrapper/testdata/A_baseline/src.go @@ -0,0 +1,13 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package baseline + +import "testing" + +func TestOne(t *testing.T) { + t.Log("hello from one") +} + +func TestTwo(t *testing.T) { +} diff --git a/cmd/testwrapper/testdata/B_inbody/raw.txt b/cmd/testwrapper/testdata/B_inbody/raw.txt new file mode 100644 index 000000000..108a0ec59 --- /dev/null +++ b/cmd/testwrapper/testdata/B_inbody/raw.txt @@ -0,0 +1,30 @@ +=== RUN TestRace +================== +WARNING: DATA RACE +Read at 0x0000007f12a8 by goroutine 8: + racesurvey/B_inbody.TestRace.func1() + /tmp/racesurvey/B_inbody/b_test.go:13 +0x74 + +Previous write at 0x0000007f12a8 by goroutine 9: + racesurvey/B_inbody.TestRace.func2() + /tmp/racesurvey/B_inbody/b_test.go:14 +0x8c + +Goroutine 8 (running) created at: + racesurvey/B_inbody.TestRace() + /tmp/racesurvey/B_inbody/b_test.go:13 +0xbe + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 9 (finished) created at: + racesurvey/B_inbody.TestRace() + /tmp/racesurvey/B_inbody/b_test.go:14 +0x124 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 +================== + testing.go:1712: race detected during execution of test +--- FAIL: TestRace (0.00s) +FAIL diff --git a/cmd/testwrapper/testdata/B_inbody/src.go b/cmd/testwrapper/testdata/B_inbody/src.go new file mode 100644 index 000000000..f0abebb99 --- /dev/null +++ b/cmd/testwrapper/testdata/B_inbody/src.go @@ -0,0 +1,19 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package inbody + +import ( + "sync" + "testing" +) + +var counter int + +func TestRace(t *testing.T) { + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); counter++ }() + go func() { defer wg.Done(); counter++ }() + wg.Wait() +} diff --git a/cmd/testwrapper/testdata/C_spawnwait/caught.raw.txt b/cmd/testwrapper/testdata/C_spawnwait/caught.raw.txt new file mode 100644 index 000000000..c2dcf3655 --- /dev/null +++ b/cmd/testwrapper/testdata/C_spawnwait/caught.raw.txt @@ -0,0 +1,32 @@ +=== RUN TestSpawn +================== +WARNING: DATA RACE +Read at 0x0000007f12a8 by goroutine 9: + racesurvey/C_spawnwait.TestSpawn.func2() + /tmp/racesurvey/C_spawnwait/c_test.go:14 +0x70 + +Previous write at 0x0000007f12a8 by goroutine 8: + racesurvey/C_spawnwait.TestSpawn.func1() + /tmp/racesurvey/C_spawnwait/c_test.go:13 +0x88 + +Goroutine 9 (running) created at: + racesurvey/C_spawnwait.TestSpawn() + /tmp/racesurvey/C_spawnwait/c_test.go:14 +0x44 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 8 (finished) created at: + racesurvey/C_spawnwait.TestSpawn() + /tmp/racesurvey/C_spawnwait/c_test.go:13 +0x34 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 +================== + testing.go:1712: race detected during execution of test +--- FAIL: TestSpawn (0.00s) +=== RUN TestWait +--- PASS: TestWait (0.00s) +FAIL diff --git a/cmd/testwrapper/testdata/C_spawnwait/pass.raw.txt b/cmd/testwrapper/testdata/C_spawnwait/pass.raw.txt new file mode 100644 index 000000000..c7ba6ebf3 --- /dev/null +++ b/cmd/testwrapper/testdata/C_spawnwait/pass.raw.txt @@ -0,0 +1,31 @@ +=== RUN TestSpawn +--- PASS: TestSpawn (0.00s) +=== RUN TestWait +================== +WARNING: DATA RACE +Read at 0x0000007f12a8 by goroutine 8: + racesurvey/C_spawnwait.TestSpawn.func1() + /tmp/racesurvey/C_spawnwait/c_test.go:13 +0x70 + +Previous write at 0x0000007f12a8 by goroutine 9: + racesurvey/C_spawnwait.TestSpawn.func2() + /tmp/racesurvey/C_spawnwait/c_test.go:14 +0x88 + +Goroutine 8 (running) created at: + racesurvey/C_spawnwait.TestSpawn() + /tmp/racesurvey/C_spawnwait/c_test.go:13 +0x34 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 9 (finished) created at: + racesurvey/C_spawnwait.TestSpawn() + /tmp/racesurvey/C_spawnwait/c_test.go:14 +0x44 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 +================== +--- PASS: TestWait (0.00s) +FAIL diff --git a/cmd/testwrapper/testdata/C_spawnwait/src.go b/cmd/testwrapper/testdata/C_spawnwait/src.go new file mode 100644 index 000000000..42e13fef1 --- /dev/null +++ b/cmd/testwrapper/testdata/C_spawnwait/src.go @@ -0,0 +1,22 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package spawnwait + +import ( + "sync" + "testing" +) + +var counter int +var wg sync.WaitGroup + +func TestSpawn(t *testing.T) { + wg.Add(2) + go func() { defer wg.Done(); counter++ }() + go func() { defer wg.Done(); counter++ }() +} + +func TestWait(t *testing.T) { + wg.Wait() +} diff --git a/cmd/testwrapper/testdata/D_delayed/raw.txt b/cmd/testwrapper/testdata/D_delayed/raw.txt new file mode 100644 index 000000000..d7d22f21e --- /dev/null +++ b/cmd/testwrapper/testdata/D_delayed/raw.txt @@ -0,0 +1,32 @@ +=== RUN TestA +--- PASS: TestA (0.00s) +=== RUN TestSleep +================== +WARNING: DATA RACE +Read at 0x0000007f12a8 by goroutine 9: + racesurvey/D_postterminal.TestA.func2() + /tmp/racesurvey/D_postterminal/d_test.go:16 +0x8a + +Previous write at 0x0000007f12a8 by goroutine 8: + racesurvey/D_postterminal.TestA.func1() + /tmp/racesurvey/D_postterminal/d_test.go:15 +0xa4 + +Goroutine 9 (running) created at: + racesurvey/D_postterminal.TestA() + /tmp/racesurvey/D_postterminal/d_test.go:16 +0x44 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 8 (finished) created at: + racesurvey/D_postterminal.TestA() + /tmp/racesurvey/D_postterminal/d_test.go:15 +0x34 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 +================== + testing.go:1712: race detected during execution of test +--- FAIL: TestSleep (0.05s) +FAIL diff --git a/cmd/testwrapper/testdata/D_delayed/src.go b/cmd/testwrapper/testdata/D_delayed/src.go new file mode 100644 index 000000000..f0fed3cfc --- /dev/null +++ b/cmd/testwrapper/testdata/D_delayed/src.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package delayed + +import ( + "sync" + "testing" + "time" +) + +var counter int +var wg sync.WaitGroup +var trigger = make(chan struct{}) + +func TestA(t *testing.T) { + wg.Add(2) + go func() { defer wg.Done(); <-trigger; counter++ }() + go func() { defer wg.Done(); <-trigger; counter++ }() +} + +func TestSleep(t *testing.T) { + close(trigger) + // Sleep long enough that the goroutines race during this sleep, + // but TestSleep itself doesn't write to counter. + time.Sleep(50 * time.Millisecond) + wg.Wait() +} diff --git a/cmd/testwrapper/testdata/E_testmain/raw.txt b/cmd/testwrapper/testdata/E_testmain/raw.txt new file mode 100644 index 000000000..df96dbe1f --- /dev/null +++ b/cmd/testwrapper/testdata/E_testmain/raw.txt @@ -0,0 +1,26 @@ +=== RUN TestPass +--- PASS: TestPass (0.00s) +PASS +================== +WARNING: DATA RACE +Read at 0x0000007f52a8 by goroutine 9: + racesurvey/E_testmain.TestMain.func2() + /tmp/racesurvey/E_testmain/e_test.go:18 +0x74 + +Previous write at 0x0000007f52a8 by goroutine 8: + racesurvey/E_testmain.TestMain.func1() + /tmp/racesurvey/E_testmain/e_test.go:17 +0x8c + +Goroutine 9 (running) created at: + racesurvey/E_testmain.TestMain() + /tmp/racesurvey/E_testmain/e_test.go:18 +0x139 + main.main() + _testmain.go:48 +0x171 + +Goroutine 8 (finished) created at: + racesurvey/E_testmain.TestMain() + /tmp/racesurvey/E_testmain/e_test.go:17 +0xcc + main.main() + _testmain.go:48 +0x171 +================== +Found 1 data race(s) diff --git a/cmd/testwrapper/testdata/E_testmain/src.go b/cmd/testwrapper/testdata/E_testmain/src.go new file mode 100644 index 000000000..ec15be22b --- /dev/null +++ b/cmd/testwrapper/testdata/E_testmain/src.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package testmain + +import ( + "sync" + "testing" +) + +var counter int + +func TestPass(t *testing.T) { +} + +func TestMain(m *testing.M) { + code := m.Run() + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); counter++ }() + go func() { defer wg.Done(); counter++ }() + wg.Wait() + _ = code +} diff --git a/cmd/testwrapper/testdata/F_parallel/split.raw.txt b/cmd/testwrapper/testdata/F_parallel/split.raw.txt new file mode 100644 index 000000000..6c8e1137d --- /dev/null +++ b/cmd/testwrapper/testdata/F_parallel/split.raw.txt @@ -0,0 +1,57 @@ +=== RUN TestParA +=== PAUSE TestParA +=== RUN TestParB +=== PAUSE TestParB +=== CONT TestParA +=== CONT TestParB +--- PASS: TestParA (0.00s) +================== +WARNING: DATA RACE +Read at 0x0000007f02b0 by goroutine 8: + racesurvey/F_parallel.TestParB() + /tmp/racesurvey/F_parallel/f_test.go:17 +0x3b + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Previous write at 0x0000007f02b0 by goroutine 7: + racesurvey/F_parallel.TestParA() + /tmp/racesurvey/F_parallel/f_test.go:10 +0x53 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 8 (running) created at: + testing.(*T).Run() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0xb12 + testing.runTests.func1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2585 +0x84 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.runTests() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2583 +0x9e9 + testing.(*M).Run() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2443 +0xf4b + main.main() + _testmain.go:48 +0x164 + +Goroutine 7 (running) created at: + testing.(*T).Run() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0xb12 + testing.runTests.func1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2585 +0x84 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.runTests() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2583 +0x9e9 + testing.(*M).Run() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2443 +0xf4b + main.main() + _testmain.go:48 +0x164 +================== +=== NAME TestParB + testing.go:1712: race detected during execution of test +--- FAIL: TestParB (0.00s) +FAIL diff --git a/cmd/testwrapper/testdata/F_parallel/src.go b/cmd/testwrapper/testdata/F_parallel/src.go new file mode 100644 index 000000000..18cd968ff --- /dev/null +++ b/cmd/testwrapper/testdata/F_parallel/src.go @@ -0,0 +1,22 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package parallel + +import "testing" + +var counter int + +func TestParA(t *testing.T) { + t.Parallel() + for i := 0; i < 100; i++ { + counter++ + } +} + +func TestParB(t *testing.T) { + t.Parallel() + for i := 0; i < 100; i++ { + counter++ + } +} diff --git a/cmd/testwrapper/testdata/README.md b/cmd/testwrapper/testdata/README.md new file mode 100644 index 000000000..d15dc64c3 --- /dev/null +++ b/cmd/testwrapper/testdata/README.md @@ -0,0 +1,133 @@ +# Race-output test corpus + +This directory is a corpus of captured Go test binary outputs that +exercise the various ways the `-race` detector's `WARNING: DATA RACE` +text can land relative to `=== RUN` / `--- PASS:` / `--- FAIL:` / +`=== NAME` lines, and how `cmd/internal/test2json` attributes that +output to tests. + +Each scenario subdirectory contains: + +- `src.go` — the Go source code that was compiled and run to produce + the captured output. Reproduce via + `go test -race -c -o /tmp/scenario.test .//`. +- `raw.txt` (or scenario-specific name) — the raw stdout+stderr of + the resulting test binary when run as `./scenario.test -test.v`. + This is the byte stream that `go test -json` feeds to + `go tool test2json` in production. + +`go test -json` adds two things on top of what `test2json` sees, +which are NOT in these captures: a `FAIL\t\t