webnet/tailscale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/k8s-operator/e2e: run self-contained e2e tests with devcontrol (#17415)

Browse Source 
* cmd/k8s-operator/e2e: run self-contained e2e tests with devcontrol

Adds orchestration for more of the e2e testing setup requirements to
make it easier to run them in CI, but also run them locally in a way
that's consistent with CI. Requires running devcontrol, but otherwise
supports creating all the scaffolding required to exercise the operator
and proxies.

Updates tailscale/corp#32085

Change-Id: Ia7bff38af3801fd141ad17452aa5a68b7e724ca6
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

* cmd/k8s-operator/e2e: being more specific on tmp dir cleanup

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

---------

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
Co-authored-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
Tom Proctor
2026-01-08 12:01:12 +00:00
committed by GitHub
parent 522a6e385e
commit 73cb3b491e
18 changed files with 1680 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/k8s-operator/e2e/certs/pebble.minica.crt
+19
View File
@@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDCTCCAfGgAwIBAgIIJOLbes8sTr4wDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
AxMVbWluaWNhIHJvb3QgY2EgMjRlMmRiMCAXDTE3MTIwNjE5NDIxMFoYDzIxMTcx
MjA2MTk0MjEwWjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSAyNGUyZGIwggEi
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5WgZNoVJandj43kkLyU50vzCZ
alozvdRo3OFiKoDtmqKPNWRNO2hC9AUNxTDJco51Yc42u/WV3fPbbhSznTiOOVtn
Ajm6iq4I5nZYltGGZetGDOQWr78y2gWY+SG078MuOO2hyDIiKtVc3xiXYA+8Hluu
9F8KbqSS1h55yxZ9b87eKR+B0zu2ahzBCIHKmKWgc6N13l7aDxxY3D6uq8gtJRU0
toumyLbdzGcupVvjbjDP11nl07RESDWBLG1/g3ktJvqIa4BWgU2HMh4rND6y8OD3
Hy3H8MY6CElL+MOCbFJjWqhtOxeFyZZV9q3kYnk9CAuQJKMEGuN4GU6tzhW1AgMB
AAGjRTBDMA4GA1UdDwEB/wQEAwIChDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB
BQUHAwIwEgYDVR0TAQH/BAgwBgEB/wIBADANBgkqhkiG9w0BAQsFAAOCAQEAF85v
d40HK1ouDAtWeO1PbnWfGEmC5Xa478s9ddOd9Clvp2McYzNlAFfM7kdcj6xeiNhF
WPIfaGAi/QdURSL/6C1KsVDqlFBlTs9zYfh2g0UXGvJtj1maeih7zxFLvet+fqll
xseM4P9EVJaQxwuK/F78YBt0tCNfivC6JNZMgxKF59h0FBpH70ytUSHXdz7FKwix
Mfn3qEb9BXSk0Q3prNV5sOV3vgjEtB4THfDxSz9z3+DepVnW3vbbqwEbkXdk3j82
2muVldgOUgTwK8eT+XdofVdntzU/kzygSAtAQwLJfn51fS1GvEcYGBc1bDryIqmF
p9BI7gVKtWSZYegicA==
-----END CERTIFICATE-----
cmd/k8s-operator/e2e/doc.go
+28
View File
@@ -0,0 +1,28 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
// Package e2e runs end-to-end tests for the Tailscale Kubernetes operator.
//
// To run without arguments, it requires:
//
// * Kubernetes cluster with local kubeconfig for it (direct connection, no API server proxy)
// * Tailscale operator installed with --set apiServerProxyConfig.mode="true"
// * ACLs from acl.hujson
// * OAuth client secret in TS_API_CLIENT_SECRET env, with at least auth_keys write scope and tag:k8s tag
// * Default ProxyClass and operator env vars as appropriate to set the desired default proxy images.
//
// It also supports running against devcontrol, using the --devcontrol flag,
// which it expects to reach at http://localhost:31544. Use --cluster to create
// a dedicated kind cluster for the tests, and --build to build and test the
// operator and proxy images for the current checkout.
//
// To run with minimal dependencies, use:
//
// go test -count=1 -v ./cmd/k8s-operator/e2e/ --build --cluster --devcontrol --skip-cleanup
//
// Running like this, it requires:
//
// * go
// * container runtime with the docker daemon API available
// * devcontrol: ./tool/go run ./cmd/devcontrol --generate-test-devices=k8s-operator-e2e --scenario-output-dir=/tmp/k8s-operator-e2e --test-dns=http://localhost:8055
package e2e
cmd/k8s-operator/e2e/ingress_test.go
+7 -15
View File
@@ -14,8 +14,6 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
kube "tailscale.com/k8s-operator"
"tailscale.com/tstest"
"tailscale.com/types/ptr"
@@ -24,17 +22,12 @@ import (
// See [TestMain] for test requirements.
func TestIngress(t *testing.T) {
if apiClient == nil {
t.Skip("TestIngress requires TS_API_CLIENT_SECRET set")
if tnClient == nil {
t.Skip("TestIngress requires a working tailnet client")
}
cfg := config.GetConfigOrDie()
cl, err := client.New(cfg, client.Options{})
if err != nil {
t.Fatal(err)
}
// Apply nginx
createAndCleanup(t, cl,
createAndCleanup(t, kubeClient,
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "nginx",
@@ -73,8 +66,7 @@ func TestIngress(t *testing.T) {
Name: "test-ingress",
Namespace: "default",
Annotations: map[string]string{
"tailscale.com/expose": "true",
"tailscale.com/proxy-class": "prod",
"tailscale.com/expose": "true",
},
},
Spec: corev1.ServiceSpec{
@@ -90,12 +82,12 @@ func TestIngress(t *testing.T) {
},
},
}
createAndCleanup(t, cl, svc)
createAndCleanup(t, kubeClient, svc)
// TODO: instead of timing out only when test times out, cancel context after 60s or so.
if err := wait.PollUntilContextCancel(t.Context(), time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) {
maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")}
if err := get(ctx, cl, maybeReadySvc); err != nil {
if err := get(ctx, kubeClient, maybeReadySvc); err != nil {
return false, err
}
isReady := kube.SvcIsReady(maybeReadySvc)
@@ -118,7 +110,7 @@ func TestIngress(t *testing.T) {
}
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
resp, err = tailnetClient.HTTPClient().Do(req.WithContext(ctx))
resp, err = tnClient.HTTPClient().Do(req.WithContext(ctx))
return err
}); err != nil {
t.Fatalf("error trying to reach Service: %v", err)
cmd/k8s-operator/e2e/main_test.go
+6 -68
View File
@@ -5,34 +5,22 @@ package e2e
import (
"context"
"errors"
"flag"
"log"
"os"
"strings"
"testing"
"time"
"golang.org/x/oauth2/clientcredentials"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"tailscale.com/internal/client/tailscale"
"tailscale.com/ipn/store/mem"
"tailscale.com/tsnet"
)
// This test suite is currently not run in CI.
// It requires some setup not handled by this code:
// - Kubernetes cluster with local kubeconfig for it (direct connection, no API server proxy)
// - Tailscale operator installed with --set apiServerProxyConfig.mode="true"
// - ACLs from acl.hujson
// - OAuth client secret in TS_API_CLIENT_SECRET env, with at least auth_keys write scope and tag:k8s tag
var (
apiClient *tailscale.Client // For API calls to control.
tailnetClient *tsnet.Server // For testing real tailnet traffic.
)
func TestMain(m *testing.M) {
flag.Parse()
if !*fDevcontrol && os.Getenv("TS_API_CLIENT_SECRET") == "" {
log.Printf("Skipping setup: devcontrol is false and TS_API_CLIENT_SECRET is not set")
os.Exit(m.Run())
}
code, err := runTests(m)
if err != nil {
log.Printf("Error: %v", err)
@@ -41,56 +29,6 @@ func TestMain(m *testing.M) {
os.Exit(code)
}
func runTests(m *testing.M) (int, error) {
secret := os.Getenv("TS_API_CLIENT_SECRET")
if secret != "" {
secretParts := strings.Split(secret, "-")
if len(secretParts) != 4 {
return 0, errors.New("TS_API_CLIENT_SECRET is not valid")
}
ctx := context.Background()
credentials := clientcredentials.Config{
ClientID: secretParts[2],
ClientSecret: secret,
TokenURL: "https://login.tailscale.com/api/v2/oauth/token",
Scopes: []string{"auth_keys"},
}
apiClient = tailscale.NewClient("-", nil)
apiClient.HTTPClient = credentials.Client(ctx)
caps := tailscale.KeyCapabilities{
Devices: tailscale.KeyDeviceCapabilities{
Create: tailscale.KeyDeviceCreateCapabilities{
Reusable: false,
Preauthorized: true,
Ephemeral: true,
Tags: []string{"tag:k8s"},
},
},
}
authKey, authKeyMeta, err := apiClient.CreateKeyWithExpiry(ctx, caps, 10*time.Minute)
if err != nil {
return 0, err
}
defer apiClient.DeleteKey(context.Background(), authKeyMeta.ID)
tailnetClient = &tsnet.Server{
Hostname: "test-proxy",
Ephemeral: true,
Store: &mem.Store{},
AuthKey: authKey,
}
_, err = tailnetClient.Up(ctx)
if err != nil {
return 0, err
}
defer tailnetClient.Close()
}
return m.Run(), nil
}
func objectMeta(namespace, name string) metav1.ObjectMeta {
return metav1.ObjectMeta{
Namespace: namespace,
cmd/k8s-operator/e2e/pebble.go
+174
View File
@@ -0,0 +1,174 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package e2e
import (
"context"
"fmt"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
"tailscale.com/types/ptr"
)
func applyPebbleResources(ctx context.Context, cl client.Client) error {
owner := client.FieldOwner("k8s-test")
if err := cl.Patch(ctx, pebbleDeployment(pebbleTag), client.Apply, owner); err != nil {
return fmt.Errorf("failed to apply pebble Deployment: %w", err)
}
if err := cl.Patch(ctx, pebbleService(), client.Apply, owner); err != nil {
return fmt.Errorf("failed to apply pebble Service: %w", err)
}
if err := cl.Patch(ctx, tailscaleNamespace(), client.Apply, owner); err != nil {
return fmt.Errorf("failed to apply tailscale Namespace: %w", err)
}
if err := cl.Patch(ctx, pebbleExternalNameService(), client.Apply, owner); err != nil {
return fmt.Errorf("failed to apply pebble ExternalName Service: %w", err)
}
return nil
}
func pebbleDeployment(tag string) *appsv1.Deployment {
return &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "pebble",
Namespace: ns,
},
Spec: appsv1.DeploymentSpec{
Replicas: ptr.To[int32](1),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "pebble",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "pebble",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "pebble",
Image: fmt.Sprintf("ghcr.io/letsencrypt/pebble:%s", tag),
ImagePullPolicy: corev1.PullIfNotPresent,
Args: []string{
"-dnsserver=localhost:8053",
"-strict",
},
Ports: []corev1.ContainerPort{
{
Name: "acme",
ContainerPort: 14000,
},
{
Name: "pebble-api",
ContainerPort: 15000,
},
},
Env: []corev1.EnvVar{
{
Name: "PEBBLE_VA_NOSLEEP",
Value: "1",
},
},
},
{
Name: "challtestsrv",
Image: fmt.Sprintf("ghcr.io/letsencrypt/pebble-challtestsrv:%s", tag),
ImagePullPolicy: corev1.PullIfNotPresent,
Args: []string{"-defaultIPv6="},
Ports: []corev1.ContainerPort{
{
Name: "mgmt-api",
ContainerPort: 8055,
},
},
},
},
},
},
},
}
}
func pebbleService() *corev1.Service {
return &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "pebble",
Namespace: ns,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: map[string]string{
"app": "pebble",
},
Ports: []corev1.ServicePort{
{
Name: "acme",
Port: 14000,
TargetPort: intstr.FromInt(14000),
},
{
Name: "pebble-api",
Port: 15000,
TargetPort: intstr.FromInt(15000),
},
{
Name: "mgmt-api",
Port: 8055,
TargetPort: intstr.FromInt(8055),
},
},
},
}
}
func tailscaleNamespace() *corev1.Namespace {
return &corev1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "tailscale",
},
}
}
// pebbleExternalNameService ensures the operator in the tailscale namespace
// can reach pebble on a DNS name (pebble) that matches its TLS cert.
func pebbleExternalNameService() *corev1.Service {
return &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "pebble",
Namespace: "tailscale",
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeExternalName,
Selector: map[string]string{
"app": "pebble",
},
ExternalName: "pebble.default.svc.cluster.local",
},
}
}
cmd/k8s-operator/e2e/proxy_test.go
+21 -15
View File
@@ -4,8 +4,10 @@
package e2e
import (
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"testing"
"time"
@@ -14,25 +16,18 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"tailscale.com/ipn"
"tailscale.com/tstest"
)
// See [TestMain] for test requirements.
func TestProxy(t *testing.T) {
if apiClient == nil {
t.Skip("TestIngress requires TS_API_CLIENT_SECRET set")
}
cfg := config.GetConfigOrDie()
cl, err := client.New(cfg, client.Options{})
if err != nil {
t.Fatal(err)
if tnClient == nil {
t.Skip("TestProxy requires a working tailnet client")
}
// Create role and role binding to allow a group we'll impersonate to do stuff.
createAndCleanup(t, cl, &rbacv1.Role{
createAndCleanup(t, kubeClient, &rbacv1.Role{
ObjectMeta: objectMeta("tailscale", "read-secrets"),
Rules: []rbacv1.PolicyRule{{
APIGroups: []string{""},
@@ -40,7 +35,7 @@ func TestProxy(t *testing.T) {
Resources: []string{"secrets"},
}},
})
createAndCleanup(t, cl, &rbacv1.RoleBinding{
createAndCleanup(t, kubeClient, &rbacv1.RoleBinding{
ObjectMeta: objectMeta("tailscale", "read-secrets"),
Subjects: []rbacv1.Subject{{
Kind: "Group",
@@ -56,16 +51,25 @@ func TestProxy(t *testing.T) {
operatorSecret := corev1.Secret{
ObjectMeta: objectMeta("tailscale", "operator"),
}
if err := get(t.Context(), cl, &operatorSecret); err != nil {
if err := get(t.Context(), kubeClient, &operatorSecret); err != nil {
t.Fatal(err)
}
// Join tailnet as a client of the API server proxy.
proxyCfg := &rest.Config{
Host: fmt.Sprintf("https://%s:443", hostNameFromOperatorSecret(t, operatorSecret)),
Dial: tailnetClient.Dial,
}
proxyCl, err := client.New(proxyCfg, client.Options{})
proxyCl, err := client.New(proxyCfg, client.Options{
HTTPClient: &http.Client{
Timeout: 10 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: testCAs,
},
DialContext: tnClient.Dial,
},
},
})
if err != nil {
t.Fatal(err)
}
@@ -77,7 +81,9 @@ func TestProxy(t *testing.T) {
// Wait for up to a minute the first time we use the proxy, to give it time
// to provision the TLS certs.
if err := tstest.WaitFor(time.Minute, func() error {
return get(t.Context(), proxyCl, &allowedSecret)
err := get(t.Context(), proxyCl, &allowedSecret)
t.Logf("get Secret via proxy: %v", err)
return err
}); err != nil {
t.Fatal(err)
}
cmd/k8s-operator/e2e/setup.go
+680
View File
@@ -0,0 +1,680 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package e2e
import (
"context"
"crypto/rand"
"crypto/tls"
"crypto/x509"
_ "embed"
jsonv1 "encoding/json"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"os/signal"
"path/filepath"
"slices"
"strings"
"sync"
"syscall"
"testing"
"time"
"github.com/go-logr/zapr"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/daemon"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"go.uber.org/zap"
"golang.org/x/oauth2/clientcredentials"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/release"
"helm.sh/helm/v3/pkg/storage/driver"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/portforward"
"k8s.io/client-go/transport/spdy"
"sigs.k8s.io/controller-runtime/pkg/client"
klog "sigs.k8s.io/controller-runtime/pkg/log"
kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/kind/pkg/cluster"
"sigs.k8s.io/kind/pkg/cluster/nodeutils"
"sigs.k8s.io/kind/pkg/cmd"
"tailscale.com/client/tailscale/v2"
"tailscale.com/ipn"
"tailscale.com/ipn/store/mem"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tsnet"
)
const (
pebbleTag = "2.8.0"
ns = "default"
tmp = "/tmp/k8s-operator-e2e"
kindClusterName = "k8s-operator-e2e"
)
var (
tsClient = &tailscale.Client{Tailnet: "-"} // For API calls to control.
tnClient *tsnet.Server // For testing real tailnet traffic.
kubeClient client.WithWatch // For k8s API calls.
//go:embed certs/pebble.minica.crt
pebbleMiniCACert []byte
// Either nil (system) or pebble CAs if pebble is deployed for devcontrol.
// pebble has a static "mini" CA that its ACME directory URL serves a cert
// from, and also dynamically generates a different CA for issuing certs.
testCAs *x509.CertPool
//go:embed acl.hujson
requiredACLs []byte
fDevcontrol = flag.Bool("devcontrol", false, "if true, connect to devcontrol at http://localhost:31544. Run devcontrol with "+`
./tool/go run ./cmd/devcontrol \
--generate-test-devices=k8s-operator-e2e \
--dir=/tmp/devcontrol \
--scenario-output-dir=/tmp/k8s-operator-e2e \
--test-dns=http://localhost:8055`)
fSkipCleanup = flag.Bool("skip-cleanup", false, "if true, do not delete the kind cluster (if created) or tmp dir on exit")
fCluster = flag.Bool("cluster", false, "if true, create or use a pre-existing kind cluster named k8s-operator-e2e; otherwise assume a usable cluster already exists in kubeconfig")
fBuild = flag.Bool("build", false, "if true, build and deploy the operator and container images from the current checkout; otherwise assume the operator is already set up")
)
func runTests(m *testing.M) (int, error) {
logger := kzap.NewRaw().Sugar()
klog.SetLogger(zapr.NewLogger(logger.Desugar()))
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer cancel()
ossDir, err := gitRootDir()
if err != nil {
return 0, err
}
if err := os.MkdirAll(tmp, 0755); err != nil {
return 0, fmt.Errorf("failed to create temp dir: %w", err)
}
logger.Infof("temp dir: %q", tmp)
logger.Infof("oss dir: %q", ossDir)
var (
kubeconfig string
kindProvider *cluster.Provider
)
if *fCluster {
kubeconfig = filepath.Join(tmp, "kubeconfig")
kindProvider = cluster.NewProvider(
cluster.ProviderWithLogger(cmd.NewLogger()),
)
clusters, err := kindProvider.List()
if err != nil {
return 0, fmt.Errorf("failed to list kind clusters: %w", err)
}
if !slices.Contains(clusters, kindClusterName) {
if err := kindProvider.Create(kindClusterName,
cluster.CreateWithWaitForReady(5*time.Minute),
cluster.CreateWithKubeconfigPath(kubeconfig),
cluster.CreateWithNodeImage("kindest/node:v1.30.0"),
); err != nil {
return 0, fmt.Errorf("failed to create kind cluster: %w", err)
}
}
if !*fSkipCleanup {
defer kindProvider.Delete(kindClusterName, kubeconfig)
defer os.Remove(kubeconfig)
}
}
// Cluster client setup.
restCfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
if err != nil {
return 0, fmt.Errorf("error loading kubeconfig: %w", err)
}
kubeClient, err = client.NewWithWatch(restCfg, client.Options{Scheme: tsapi.GlobalScheme})
if err != nil {
return 0, fmt.Errorf("error creating Kubernetes client: %w", err)
}
var (
clusterLoginServer string // Login server from cluster Pod point of view.
clientID, clientSecret string // OAuth client for the operator to use.
caPaths []string // Extra CA cert file paths to add to images.
certsDir string = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images.
)
if *fDevcontrol {
// Deploy pebble and get its certs.
if err := applyPebbleResources(ctx, kubeClient); err != nil {
return 0, fmt.Errorf("failed to apply pebble resources: %w", err)
}
pebblePod, err := waitForPodReady(ctx, logger, kubeClient, ns, client.MatchingLabels{"app": "pebble"})
if err != nil {
return 0, fmt.Errorf("pebble pod not ready: %w", err)
}
if err := forwardLocalPortToPod(ctx, logger, restCfg, ns, pebblePod, 15000); err != nil {
return 0, fmt.Errorf("failed to set up port forwarding to pebble: %w", err)
}
testCAs = x509.NewCertPool()
if ok := testCAs.AppendCertsFromPEM(pebbleMiniCACert); !ok {
return 0, fmt.Errorf("failed to parse pebble minica cert")
}
var pebbleCAChain []byte
for _, path := range []string{"/intermediates/0", "/roots/0"} {
pem, err := pebbleGet(ctx, 15000, path)
if err != nil {
return 0, err
}
pebbleCAChain = append(pebbleCAChain, pem...)
}
if ok := testCAs.AppendCertsFromPEM(pebbleCAChain); !ok {
return 0, fmt.Errorf("failed to parse pebble ca chain cert")
}
if err := os.MkdirAll(certsDir, 0755); err != nil {
return 0, fmt.Errorf("failed to create certs dir: %w", err)
}
pebbleCAChainPath := filepath.Join(certsDir, "pebble-ca-chain.crt")
if err := os.WriteFile(pebbleCAChainPath, pebbleCAChain, 0644); err != nil {
return 0, fmt.Errorf("failed to write pebble CA chain: %w", err)
}
pebbleMiniCACertPath := filepath.Join(certsDir, "pebble.minica.crt")
if err := os.WriteFile(pebbleMiniCACertPath, pebbleMiniCACert, 0644); err != nil {
return 0, fmt.Errorf("failed to write pebble minica: %w", err)
}
caPaths = []string{pebbleCAChainPath, pebbleMiniCACertPath}
if !*fSkipCleanup {
defer os.RemoveAll(certsDir)
}
// Set up network connectivity between cluster and devcontrol.
//
// For devcontrol -> pebble (DNS mgmt for ACME challenges):
// * Port forward from localhost port 8055 to in-cluster pebble port 8055.
//
// For Pods -> devcontrol (tailscale clients joining the tailnet):
// * Create ssh-server Deployment in cluster.
// * Create reverse ssh tunnel that goes from ssh-server port 31544 to localhost:31544.
if err := forwardLocalPortToPod(ctx, logger, restCfg, ns, pebblePod, 8055); err != nil {
return 0, fmt.Errorf("failed to set up port forwarding to pebble: %w", err)
}
privateKey, publicKey, err := readOrGenerateSSHKey(tmp)
if err != nil {
return 0, fmt.Errorf("failed to read or generate SSH key: %w", err)
}
if !*fSkipCleanup {
defer os.Remove(privateKeyPath)
}
sshServiceIP, err := connectClusterToDevcontrol(ctx, logger, kubeClient, restCfg, privateKey, publicKey)
if err != nil {
return 0, fmt.Errorf("failed to set up cluster->devcontrol connection: %w", err)
}
if !*fSkipCleanup {
defer func() {
if err := cleanupSSHResources(context.Background(), kubeClient); err != nil {
logger.Infof("failed to clean up ssh-server resources: %v", err)
}
}()
}
// Address cluster workloads can reach devcontrol at. Must be a private
// IP to make sure tailscale client code recognises it shouldn't try an
// https fallback. See [controlclient.NewNoiseClient] for details.
clusterLoginServer = fmt.Sprintf("http://%s:31544", sshServiceIP)
b, err := os.ReadFile(filepath.Join(tmp, "api-key.json"))
if err != nil {
return 0, fmt.Errorf("failed to read api-key.json: %w", err)
}
var apiKeyData struct {
APIKey string `json:"apiKey"`
}
if err := jsonv1.Unmarshal(b, &apiKeyData); err != nil {
return 0, fmt.Errorf("failed to parse api-key.json: %w", err)
}
if apiKeyData.APIKey == "" {
return 0, fmt.Errorf("api-key.json did not contain an API key")
}
// Finish setting up tsClient.
baseURL, err := url.Parse("http://localhost:31544")
if err != nil {
return 0, fmt.Errorf("parse url: %w", err)
}
tsClient.BaseURL = baseURL
tsClient.APIKey = apiKeyData.APIKey
tsClient.HTTP = &http.Client{}
// Set ACLs and create OAuth client.
if err := tsClient.PolicyFile().Set(ctx, string(requiredACLs), ""); err != nil {
return 0, fmt.Errorf("failed to set ACLs: %w", err)
}
logger.Infof("ACLs configured")
key, err := tsClient.Keys().CreateOAuthClient(ctx, tailscale.CreateOAuthClientRequest{
Scopes: []string{"auth_keys", "devices:core", "services"},
Tags: []string{"tag:k8s-operator"},
Description: "k8s-operator client for e2e tests",
})
if err != nil {
return 0, fmt.Errorf("failed to create OAuth client: %w", err)
}
clientID = key.ID
clientSecret = key.Key
} else {
clientSecret = os.Getenv("TS_API_CLIENT_SECRET")
if clientSecret == "" {
return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator")
}
// Format is "tskey-client-<id>-<random>".
parts := strings.Split(clientSecret, "-")
if len(parts) != 4 {
return 0, fmt.Errorf("TS_API_CLIENT_SECRET is not valid")
}
clientID = parts[2]
credentials := clientcredentials.Config{
ClientID: clientID,
ClientSecret: clientSecret,
TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL),
Scopes: []string{"auth_keys"},
}
baseURL, _ := url.Parse(ipn.DefaultControlURL)
tsClient = &tailscale.Client{
Tailnet: "-",
HTTP: credentials.Client(ctx),
BaseURL: baseURL,
}
}
var ossTag string
if *fBuild {
// TODO(tomhjp): proper support for --build=false and layering pebble certs on top of existing images.
// TODO(tomhjp): support non-local platform.
// TODO(tomhjp): build tsrecorder as well.
// Build tailscale/k8s-operator, tailscale/tailscale, tailscale/k8s-proxy, with pebble CAs added.
ossTag, err = tagForRepo(ossDir)
if err != nil {
return 0, err
}
logger.Infof("using OSS image tag: %q", ossTag)
ossImageToTarget := map[string]string{
"local/k8s-operator": "publishdevoperator",
"local/tailscale": "publishdevimage",
"local/k8s-proxy": "publishdevproxy",
}
for img, target := range ossImageToTarget {
if err := buildImage(ctx, ossDir, img, target, ossTag, caPaths); err != nil {
return 0, err
}
nodes, err := kindProvider.ListInternalNodes(kindClusterName)
if err != nil {
return 0, fmt.Errorf("failed to list kind nodes: %w", err)
}
// TODO(tomhjp): can be made more efficient and portable if we
// stream built image tarballs straight to the node rather than
// going via the daemon.
// TODO(tomhjp): support --build with non-kind clusters.
imgRef, err := name.ParseReference(fmt.Sprintf("%s:%s", img, ossTag))
if err != nil {
return 0, fmt.Errorf("failed to parse image reference: %w", err)
}
img, err := daemon.Image(imgRef)
if err != nil {
return 0, fmt.Errorf("failed to get image from daemon: %w", err)
}
pr, pw := io.Pipe()
go func() {
defer pw.Close()
if err := tarball.Write(imgRef, img, pw); err != nil {
logger.Infof("failed to write image to pipe: %v", err)
}
}()
for _, n := range nodes {
if err := nodeutils.LoadImageArchive(n, pr); err != nil {
return 0, fmt.Errorf("failed to load image into node %q: %w", n.String(), err)
}
}
}
}
// Generate CRDs for the helm chart.
cmd := exec.CommandContext(ctx, "go", "run", "tailscale.com/cmd/k8s-operator/generate", "helmcrd")
cmd.Dir = ossDir
out, err := cmd.CombinedOutput()
if err != nil {
return 0, fmt.Errorf("failed to generate CRD: %v: %s", err, out)
}
// Load and install helm chart.
chart, err := loader.Load(filepath.Join(ossDir, "cmd", "k8s-operator", "deploy", "chart"))
if err != nil {
return 0, fmt.Errorf("failed to load helm chart: %w", err)
}
values := map[string]any{
"loginServer": clusterLoginServer,
"oauth": map[string]any{
"clientId": clientID,
"clientSecret": clientSecret,
},
"apiServerProxyConfig": map[string]any{
"mode": "true",
},
"operatorConfig": map[string]any{
"logging": "debug",
"extraEnv": []map[string]any{
{
"name": "K8S_PROXY_IMAGE",
"value": "local/k8s-proxy:" + ossTag,
},
{
"name": "TS_DEBUG_ACME_DIRECTORY_URL",
"value": "https://pebble:14000/dir",
},
},
"image": map[string]any{
"repo": "local/k8s-operator",
"tag": ossTag,
"pullPolicy": "IfNotPresent",
},
},
"proxyConfig": map[string]any{
"defaultProxyClass": "default",
"image": map[string]any{
"repository": "local/tailscale",
"tag": ossTag,
},
},
}
settings := cli.New()
settings.KubeConfig = kubeconfig
settings.SetNamespace("tailscale")
helmCfg := &action.Configuration{}
if err := helmCfg.Init(settings.RESTClientGetter(), "tailscale", "", logger.Infof); err != nil {
return 0, fmt.Errorf("failed to initialize helm action configuration: %w", err)
}
const relName = "tailscale-operator" // TODO(tomhjp): maybe configurable if others use a different value.
f := upgraderOrInstaller(helmCfg, relName)
if _, err := f(ctx, relName, chart, values); err != nil {
return 0, fmt.Errorf("failed to install %q via helm: %w", relName, err)
}
if err := applyDefaultProxyClass(ctx, kubeClient); err != nil {
return 0, fmt.Errorf("failed to apply default ProxyClass: %w", err)
}
caps := tailscale.KeyCapabilities{}
caps.Devices.Create.Preauthorized = true
caps.Devices.Create.Ephemeral = true
caps.Devices.Create.Tags = []string{"tag:k8s"}
authKey, err := tsClient.Keys().CreateAuthKey(ctx, tailscale.CreateKeyRequest{
Capabilities: caps,
ExpirySeconds: 600,
Description: "e2e test authkey",
})
if err != nil {
return 0, err
}
defer tsClient.Keys().Delete(context.Background(), authKey.ID)
tnClient = &tsnet.Server{
ControlURL: tsClient.BaseURL.String(),
Hostname: "test-proxy",
Ephemeral: true,
Store: &mem.Store{},
AuthKey: authKey.Key,
}
_, err = tnClient.Up(ctx)
if err != nil {
return 0, err
}
defer tnClient.Close()
return m.Run(), nil
}
func upgraderOrInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc {
hist := action.NewHistory(cfg)
hist.Max = 1
helmVersions, err := hist.Run(releaseName)
if err == driver.ErrReleaseNotFound || (len(helmVersions) > 0 && helmVersions[0].Info.Status == release.StatusUninstalled) {
return helmInstaller(cfg, releaseName)
} else {
return helmUpgrader(cfg)
}
}
func helmUpgrader(cfg *action.Configuration) helmInstallerFunc {
upgrade := action.NewUpgrade(cfg)
upgrade.Namespace = "tailscale"
upgrade.Install = true
upgrade.Wait = true
upgrade.Timeout = 5 * time.Minute
return upgrade.RunWithContext
}
func helmInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc {
install := action.NewInstall(cfg)
install.Namespace = "tailscale"
install.CreateNamespace = true
install.ReleaseName = releaseName
install.Wait = true
install.Timeout = 5 * time.Minute
install.Replace = true
return func(ctx context.Context, _ string, chart *chart.Chart, values map[string]any) (*release.Release, error) {
return install.RunWithContext(ctx, chart, values)
}
}
type helmInstallerFunc func(context.Context, string, *chart.Chart, map[string]any) (*release.Release, error)
// gitRootDir returns the top-level directory of the current git repo. Expects
// to be run from inside a git repo.
func gitRootDir() (string, error) {
top, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
if err != nil {
return "", fmt.Errorf("failed to find git top level (not in corp git?): %w", err)
}
return strings.TrimSpace(string(top)), nil
}
func tagForRepo(dir string) (string, error) {
cmd := exec.Command("git", "rev-parse", "--short", "HEAD")
cmd.Dir = dir
out, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("failed to get latest git tag for repo %q: %w", dir, err)
}
tag := strings.TrimSpace(string(out))
// If dirty, append an extra random tag to ensure unique image tags.
cmd = exec.Command("git", "status", "--porcelain")
cmd.Dir = dir
out, err = cmd.Output()
if err != nil {
return "", fmt.Errorf("failed to check git status for repo %q: %w", dir, err)
}
if strings.TrimSpace(string(out)) != "" {
tag += "-" + strings.ToLower(rand.Text())
}
return tag, nil
}
func applyDefaultProxyClass(ctx context.Context, cl client.Client) error {
pc := &tsapi.ProxyClass{
TypeMeta: metav1.TypeMeta{
APIVersion: tsapi.SchemeGroupVersion.String(),
Kind: tsapi.ProxyClassKind,
},
ObjectMeta: metav1.ObjectMeta{
Name: "default",
},
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Pod: &tsapi.Pod{
TailscaleInitContainer: &tsapi.Container{
ImagePullPolicy: "IfNotPresent",
},
TailscaleContainer: &tsapi.Container{
ImagePullPolicy: "IfNotPresent",
},
},
},
},
}
owner := client.FieldOwner("k8s-test")
if err := cl.Patch(ctx, pc, client.Apply, owner); err != nil {
return fmt.Errorf("failed to apply default ProxyClass: %w", err)
}
return nil
}
// forwardLocalPortToPod sets up port forwarding to the specified Pod and remote port.
// It runs until the provided ctx is done.
func forwardLocalPortToPod(ctx context.Context, logger *zap.SugaredLogger, cfg *rest.Config, ns, podName string, port int) error {
transport, upgrader, err := spdy.RoundTripperFor(cfg)
if err != nil {
return fmt.Errorf("failed to create round tripper: %w", err)
}
u, err := url.Parse(fmt.Sprintf("%s%s/api/v1/namespaces/%s/pods/%s/portforward", cfg.Host, cfg.APIPath, ns, podName))
if err != nil {
return fmt.Errorf("failed to parse URL: %w", err)
}
dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", u)
stopChan := make(chan struct{}, 1)
readyChan := make(chan struct{}, 1)
ports := []string{fmt.Sprintf("%d:%d", port, port)}
// TODO(tomhjp): work out how zap logger can be used instead of stdout/err.
pf, err := portforward.New(dialer, ports, stopChan, readyChan, os.Stdout, os.Stderr)
if err != nil {
return fmt.Errorf("failed to create port forwarder: %w", err)
}
go func() {
if err := pf.ForwardPorts(); err != nil {
logger.Infof("Port forwarding error: %v\n", err)
}
}()
var once sync.Once
go func() {
<-ctx.Done()
once.Do(func() { close(stopChan) })
}()
// Wait for port forwarding to be ready
select {
case <-readyChan:
logger.Infof("Port forwarding to Pod %s/%s ready", ns, podName)
case <-time.After(10 * time.Second):
once.Do(func() { close(stopChan) })
return fmt.Errorf("timeout waiting for port forward to be ready")
}
return nil
}
// waitForPodReady waits for at least 1 Pod matching the label selector to be
// in Ready state. It returns the name of the first ready Pod it finds.
func waitForPodReady(ctx context.Context, logger *zap.SugaredLogger, cl client.WithWatch, ns string, labelSelector client.MatchingLabels) (string, error) {
pods := &corev1.PodList{}
w, err := cl.Watch(ctx, pods, client.InNamespace(ns), client.MatchingLabels(labelSelector))
if err != nil {
return "", fmt.Errorf("failed to create pod watcher: %v", err)
}
defer w.Stop()
for {
select {
case event, ok := <-w.ResultChan():
if !ok {
return "", fmt.Errorf("watcher channel closed")
}
switch event.Type {
case watch.Added, watch.Modified:
if pod, ok := event.Object.(*corev1.Pod); ok {
for _, condition := range pod.Status.Conditions {
if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
logger.Infof("pod %s is ready", pod.Name)
return pod.Name, nil
}
}
}
case watch.Error:
return "", fmt.Errorf("watch error: %v", event.Object)
}
case <-ctx.Done():
return "", fmt.Errorf("timeout waiting for pod to be ready")
}
}
}
func pebbleGet(ctx context.Context, port uint16, path string) ([]byte, error) {
pebbleClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: testCAs,
},
},
Timeout: 10 * time.Second,
}
req, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://localhost:%d%s", port, path), nil)
resp, err := pebbleClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch pebble root CA: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d when fetching pebble root CA", resp.StatusCode)
}
b, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read pebble root CA response: %w", err)
}
return b, nil
}
func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts []string) error {
var files []string
for _, f := range extraCACerts {
files = append(files, fmt.Sprintf("%s:/etc/ssl/certs/%s", f, filepath.Base(f)))
}
cmd := exec.CommandContext(ctx, "make", target,
"PLATFORM=local",
fmt.Sprintf("TAGS=%s", tag),
fmt.Sprintf("REPO=%s", repo),
fmt.Sprintf("FILES=%s", strings.Join(files, ",")),
)
cmd.Dir = dir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to build image %q: %w", target, err)
}
return nil
}
cmd/k8s-operator/e2e/ssh.go
+352
View File
@@ -0,0 +1,352 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package e2e
import (
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/hex"
"encoding/pem"
"fmt"
"io"
"net"
"os"
"path/filepath"
"time"
"go.uber.org/zap"
"golang.org/x/crypto/ssh"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
tailscaleroot "tailscale.com"
"tailscale.com/types/ptr"
)
const (
keysFilePath = "/root/.ssh/authorized_keys"
sshdConfig = `
Port 8022
# Allow reverse tunnels
GatewayPorts yes
AllowTcpForwarding yes
# Auth
PermitRootLogin yes
PasswordAuthentication no
PubkeyAuthentication yes
AuthorizedKeysFile ` + keysFilePath
)
var privateKeyPath = filepath.Join(tmp, "id_ed25519")
func connectClusterToDevcontrol(ctx context.Context, logger *zap.SugaredLogger, cl client.WithWatch, restConfig *rest.Config, privKey ed25519.PrivateKey, pubKey []byte) (clusterIP string, _ error) {
logger.Info("Setting up SSH reverse tunnel from cluster to devcontrol...")
var err error
if clusterIP, err = applySSHResources(ctx, cl, tailscaleroot.AlpineDockerTag, pubKey); err != nil {
return "", fmt.Errorf("failed to apply ssh-server resources: %w", err)
}
sshPodName, err := waitForPodReady(ctx, logger, cl, ns, client.MatchingLabels{"app": "ssh-server"})
if err != nil {
return "", fmt.Errorf("ssh-server Pod not ready: %w", err)
}
if err := forwardLocalPortToPod(ctx, logger, restConfig, ns, sshPodName, 8022); err != nil {
return "", fmt.Errorf("failed to set up port forwarding to ssh-server: %w", err)
}
if err := reverseTunnel(ctx, logger, privKey, fmt.Sprintf("localhost:%d", 8022), 31544, "localhost:31544"); err != nil {
return "", fmt.Errorf("failed to set up reverse tunnel: %w", err)
}
return clusterIP, nil
}
func reverseTunnel(ctx context.Context, logger *zap.SugaredLogger, privateKey ed25519.PrivateKey, sshHost string, remotePort uint16, fwdTo string) error {
signer, err := ssh.NewSignerFromKey(privateKey)
if err != nil {
return fmt.Errorf("failed to create signer: %w", err)
}
config := &ssh.ClientConfig{
User: "root",
Auth: []ssh.AuthMethod{
ssh.PublicKeys(signer),
},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 30 * time.Second,
}
conn, err := ssh.Dial("tcp", sshHost, config)
if err != nil {
return fmt.Errorf("failed to connect to SSH server: %w", err)
}
logger.Infof("Connected to SSH server at %s\n", sshHost)
go func() {
defer conn.Close()
// Start listening on remote port.
remoteAddr := fmt.Sprintf("localhost:%d", remotePort)
remoteLn, err := conn.Listen("tcp", remoteAddr)
if err != nil {
logger.Infof("Failed to listen on remote port %d: %v", remotePort, err)
return
}
defer remoteLn.Close()
logger.Infof("Reverse tunnel ready on remote addr %s -> local addr %s", remoteAddr, fwdTo)
for {
remoteConn, err := remoteLn.Accept()
if err != nil {
logger.Infof("Failed to accept remote connection: %v", err)
return
}
go handleConnection(ctx, logger, remoteConn, fwdTo)
}
}()
return nil
}
func handleConnection(ctx context.Context, logger *zap.SugaredLogger, remoteConn net.Conn, fwdTo string) {
go func() {
<-ctx.Done()
remoteConn.Close()
}()
var d net.Dialer
localConn, err := d.DialContext(ctx, "tcp", fwdTo)
if err != nil {
logger.Infof("Failed to connect to local service %s: %v", fwdTo, err)
return
}
go func() {
<-ctx.Done()
localConn.Close()
}()
go func() {
if _, err := io.Copy(localConn, remoteConn); err != nil {
logger.Infof("Error copying remote->local: %v", err)
}
}()
go func() {
if _, err := io.Copy(remoteConn, localConn); err != nil {
logger.Infof("Error copying local->remote: %v", err)
}
}()
}
func readOrGenerateSSHKey(tmp string) (ed25519.PrivateKey, []byte, error) {
var privateKey ed25519.PrivateKey
b, err := os.ReadFile(privateKeyPath)
switch {
case os.IsNotExist(err):
_, privateKey, err = ed25519.GenerateKey(rand.Reader)
if err != nil {
return nil, nil, fmt.Errorf("failed to generate key: %w", err)
}
privKeyPEM, err := ssh.MarshalPrivateKey(privateKey, "")
if err != nil {
return nil, nil, fmt.Errorf("failed to marshal SSH private key: %w", err)
}
f, err := os.OpenFile(privateKeyPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return nil, nil, fmt.Errorf("failed to open SSH private key file: %w", err)
}
defer f.Close()
if err := pem.Encode(f, privKeyPEM); err != nil {
return nil, nil, fmt.Errorf("failed to write SSH private key: %w", err)
}
case err != nil:
return nil, nil, fmt.Errorf("failed to read SSH private key: %w", err)
default:
pKey, err := ssh.ParseRawPrivateKey(b)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse SSH private key: %w", err)
}
pKeyPointer, ok := pKey.(*ed25519.PrivateKey)
if !ok {
return nil, nil, fmt.Errorf("SSH private key is not ed25519: %T", pKey)
}
privateKey = *pKeyPointer
}
sshPublicKey, err := ssh.NewPublicKey(privateKey.Public())
if err != nil {
return nil, nil, fmt.Errorf("failed to create SSH public key: %w", err)
}
return privateKey, ssh.MarshalAuthorizedKey(sshPublicKey), nil
}
func applySSHResources(ctx context.Context, cl client.Client, alpineTag string, pubKey []byte) (string, error) {
owner := client.FieldOwner("k8s-test")
if err := cl.Patch(ctx, sshDeployment(alpineTag, pubKey), client.Apply, owner); err != nil {
return "", fmt.Errorf("failed to apply ssh-server Deployment: %w", err)
}
if err := cl.Patch(ctx, sshConfigMap(pubKey), client.Apply, owner); err != nil {
return "", fmt.Errorf("failed to apply ssh-server ConfigMap: %w", err)
}
svc := sshService()
if err := cl.Patch(ctx, svc, client.Apply, owner); err != nil {
return "", fmt.Errorf("failed to apply ssh-server Service: %w", err)
}
return svc.Spec.ClusterIP, nil
}
func cleanupSSHResources(ctx context.Context, cl client.Client) error {
noGrace := &client.DeleteOptions{
GracePeriodSeconds: ptr.To[int64](0),
}
if err := cl.Delete(ctx, sshDeployment("", nil), noGrace); err != nil {
return fmt.Errorf("failed to delete ssh-server Deployment: %w", err)
}
if err := cl.Delete(ctx, sshConfigMap(nil), noGrace); err != nil {
return fmt.Errorf("failed to delete ssh-server ConfigMap: %w", err)
}
if err := cl.Delete(ctx, sshService(), noGrace); err != nil {
return fmt.Errorf("failed to delete control Service: %w", err)
}
return nil
}
func sshDeployment(tag string, pubKey []byte) *appsv1.Deployment {
return &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "ssh-server",
Namespace: ns,
},
Spec: appsv1.DeploymentSpec{
Replicas: ptr.To[int32](1),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "ssh-server",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "ssh-server",
},
Annotations: map[string]string{
"pubkey": hex.EncodeToString(pubKey), // Ensure new key triggers rollout.
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ssh-server",
Image: fmt.Sprintf("alpine:%s", tag),
Command: []string{
"sh", "-c",
"apk add openssh-server; ssh-keygen -A; /usr/sbin/sshd -D -e",
},
Ports: []corev1.ContainerPort{
{
Name: "ctrl-port-fwd",
ContainerPort: 31544,
Protocol: corev1.ProtocolTCP,
},
{
Name: "ssh",
ContainerPort: 8022,
Protocol: corev1.ProtocolTCP,
},
},
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
TCPSocket: &corev1.TCPSocketAction{
Port: intstr.FromInt(8022),
},
},
InitialDelaySeconds: 1,
PeriodSeconds: 1,
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "sshd-config",
MountPath: "/etc/ssh/sshd_config.d/reverse-tunnel.conf",
SubPath: "reverse-tunnel.conf",
},
{
Name: "sshd-config",
MountPath: keysFilePath,
SubPath: "authorized_keys",
},
},
},
},
Volumes: []corev1.Volume{
{
Name: "sshd-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "ssh-server-config",
},
},
},
},
},
},
},
},
}
}
func sshConfigMap(pubKey []byte) *corev1.ConfigMap {
return &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "ssh-server-config",
Namespace: ns,
},
Data: map[string]string{
"reverse-tunnel.conf": sshdConfig,
"authorized_keys": string(pubKey),
},
}
}
func sshService() *corev1.Service {
return &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "control",
Namespace: ns,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: map[string]string{
"app": "ssh-server",
},
Ports: []corev1.ServicePort{
{
Name: "tunnel",
Port: 31544,
Protocol: corev1.ProtocolTCP,
},
},
},
}
}