cmd/k8s-operator/e2e: run self-contained e2e tests with devcontrol (#17415)
* cmd/k8s-operator/e2e: run self-contained e2e tests with devcontrol Adds orchestration for more of the e2e testing setup requirements to make it easier to run them in CI, but also run them locally in a way that's consistent with CI. Requires running devcontrol, but otherwise supports creating all the scaffolding required to exercise the operator and proxies. Updates tailscale/corp#32085 Change-Id: Ia7bff38af3801fd141ad17452aa5a68b7e724ca6 Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com> * cmd/k8s-operator/e2e: being more specific on tmp dir cleanup Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk> --------- Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com> Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk> Co-authored-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
Tom Proctor
GitHub
@@ -0,0 +1,352 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/crypto/ssh"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/rest"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
tailscaleroot "tailscale.com"
|
||||
"tailscale.com/types/ptr"
|
||||
)
|
||||
|
||||
const (
|
||||
keysFilePath = "/root/.ssh/authorized_keys"
|
||||
sshdConfig = `
|
||||
Port 8022
|
||||
|
||||
# Allow reverse tunnels
|
||||
GatewayPorts yes
|
||||
AllowTcpForwarding yes
|
||||
|
||||
# Auth
|
||||
PermitRootLogin yes
|
||||
PasswordAuthentication no
|
||||
PubkeyAuthentication yes
|
||||
AuthorizedKeysFile ` + keysFilePath
|
||||
)
|
||||
|
||||
var privateKeyPath = filepath.Join(tmp, "id_ed25519")
|
||||
|
||||
func connectClusterToDevcontrol(ctx context.Context, logger *zap.SugaredLogger, cl client.WithWatch, restConfig *rest.Config, privKey ed25519.PrivateKey, pubKey []byte) (clusterIP string, _ error) {
|
||||
logger.Info("Setting up SSH reverse tunnel from cluster to devcontrol...")
|
||||
var err error
|
||||
if clusterIP, err = applySSHResources(ctx, cl, tailscaleroot.AlpineDockerTag, pubKey); err != nil {
|
||||
return "", fmt.Errorf("failed to apply ssh-server resources: %w", err)
|
||||
}
|
||||
sshPodName, err := waitForPodReady(ctx, logger, cl, ns, client.MatchingLabels{"app": "ssh-server"})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ssh-server Pod not ready: %w", err)
|
||||
}
|
||||
if err := forwardLocalPortToPod(ctx, logger, restConfig, ns, sshPodName, 8022); err != nil {
|
||||
return "", fmt.Errorf("failed to set up port forwarding to ssh-server: %w", err)
|
||||
}
|
||||
if err := reverseTunnel(ctx, logger, privKey, fmt.Sprintf("localhost:%d", 8022), 31544, "localhost:31544"); err != nil {
|
||||
return "", fmt.Errorf("failed to set up reverse tunnel: %w", err)
|
||||
}
|
||||
|
||||
return clusterIP, nil
|
||||
}
|
||||
|
||||
func reverseTunnel(ctx context.Context, logger *zap.SugaredLogger, privateKey ed25519.PrivateKey, sshHost string, remotePort uint16, fwdTo string) error {
|
||||
signer, err := ssh.NewSignerFromKey(privateKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create signer: %w", err)
|
||||
}
|
||||
config := &ssh.ClientConfig{
|
||||
User: "root",
|
||||
Auth: []ssh.AuthMethod{
|
||||
ssh.PublicKeys(signer),
|
||||
},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
conn, err := ssh.Dial("tcp", sshHost, config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to SSH server: %w", err)
|
||||
}
|
||||
logger.Infof("Connected to SSH server at %s\n", sshHost)
|
||||
|
||||
go func() {
|
||||
defer conn.Close()
|
||||
|
||||
// Start listening on remote port.
|
||||
remoteAddr := fmt.Sprintf("localhost:%d", remotePort)
|
||||
remoteLn, err := conn.Listen("tcp", remoteAddr)
|
||||
if err != nil {
|
||||
logger.Infof("Failed to listen on remote port %d: %v", remotePort, err)
|
||||
return
|
||||
}
|
||||
defer remoteLn.Close()
|
||||
logger.Infof("Reverse tunnel ready on remote addr %s -> local addr %s", remoteAddr, fwdTo)
|
||||
|
||||
for {
|
||||
remoteConn, err := remoteLn.Accept()
|
||||
if err != nil {
|
||||
logger.Infof("Failed to accept remote connection: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
go handleConnection(ctx, logger, remoteConn, fwdTo)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleConnection(ctx context.Context, logger *zap.SugaredLogger, remoteConn net.Conn, fwdTo string) {
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
remoteConn.Close()
|
||||
}()
|
||||
|
||||
var d net.Dialer
|
||||
localConn, err := d.DialContext(ctx, "tcp", fwdTo)
|
||||
if err != nil {
|
||||
logger.Infof("Failed to connect to local service %s: %v", fwdTo, err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
localConn.Close()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
if _, err := io.Copy(localConn, remoteConn); err != nil {
|
||||
logger.Infof("Error copying remote->local: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
if _, err := io.Copy(remoteConn, localConn); err != nil {
|
||||
logger.Infof("Error copying local->remote: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func readOrGenerateSSHKey(tmp string) (ed25519.PrivateKey, []byte, error) {
|
||||
var privateKey ed25519.PrivateKey
|
||||
b, err := os.ReadFile(privateKeyPath)
|
||||
switch {
|
||||
case os.IsNotExist(err):
|
||||
_, privateKey, err = ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to generate key: %w", err)
|
||||
}
|
||||
privKeyPEM, err := ssh.MarshalPrivateKey(privateKey, "")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to marshal SSH private key: %w", err)
|
||||
}
|
||||
f, err := os.OpenFile(privateKeyPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to open SSH private key file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
if err := pem.Encode(f, privKeyPEM); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to write SSH private key: %w", err)
|
||||
}
|
||||
case err != nil:
|
||||
return nil, nil, fmt.Errorf("failed to read SSH private key: %w", err)
|
||||
default:
|
||||
pKey, err := ssh.ParseRawPrivateKey(b)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to parse SSH private key: %w", err)
|
||||
}
|
||||
pKeyPointer, ok := pKey.(*ed25519.PrivateKey)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("SSH private key is not ed25519: %T", pKey)
|
||||
}
|
||||
privateKey = *pKeyPointer
|
||||
}
|
||||
|
||||
sshPublicKey, err := ssh.NewPublicKey(privateKey.Public())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create SSH public key: %w", err)
|
||||
}
|
||||
|
||||
return privateKey, ssh.MarshalAuthorizedKey(sshPublicKey), nil
|
||||
}
|
||||
|
||||
func applySSHResources(ctx context.Context, cl client.Client, alpineTag string, pubKey []byte) (string, error) {
|
||||
owner := client.FieldOwner("k8s-test")
|
||||
|
||||
if err := cl.Patch(ctx, sshDeployment(alpineTag, pubKey), client.Apply, owner); err != nil {
|
||||
return "", fmt.Errorf("failed to apply ssh-server Deployment: %w", err)
|
||||
}
|
||||
if err := cl.Patch(ctx, sshConfigMap(pubKey), client.Apply, owner); err != nil {
|
||||
return "", fmt.Errorf("failed to apply ssh-server ConfigMap: %w", err)
|
||||
}
|
||||
svc := sshService()
|
||||
if err := cl.Patch(ctx, svc, client.Apply, owner); err != nil {
|
||||
return "", fmt.Errorf("failed to apply ssh-server Service: %w", err)
|
||||
}
|
||||
|
||||
return svc.Spec.ClusterIP, nil
|
||||
}
|
||||
|
||||
func cleanupSSHResources(ctx context.Context, cl client.Client) error {
|
||||
noGrace := &client.DeleteOptions{
|
||||
GracePeriodSeconds: ptr.To[int64](0),
|
||||
}
|
||||
if err := cl.Delete(ctx, sshDeployment("", nil), noGrace); err != nil {
|
||||
return fmt.Errorf("failed to delete ssh-server Deployment: %w", err)
|
||||
}
|
||||
if err := cl.Delete(ctx, sshConfigMap(nil), noGrace); err != nil {
|
||||
return fmt.Errorf("failed to delete ssh-server ConfigMap: %w", err)
|
||||
}
|
||||
if err := cl.Delete(ctx, sshService(), noGrace); err != nil {
|
||||
return fmt.Errorf("failed to delete control Service: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sshDeployment(tag string, pubKey []byte) *appsv1.Deployment {
|
||||
return &appsv1.Deployment{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Deployment",
|
||||
APIVersion: "apps/v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ssh-server",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: appsv1.DeploymentSpec{
|
||||
Replicas: ptr.To[int32](1),
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"app": "ssh-server",
|
||||
},
|
||||
},
|
||||
Template: corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"app": "ssh-server",
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"pubkey": hex.EncodeToString(pubKey), // Ensure new key triggers rollout.
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "ssh-server",
|
||||
Image: fmt.Sprintf("alpine:%s", tag),
|
||||
Command: []string{
|
||||
"sh", "-c",
|
||||
"apk add openssh-server; ssh-keygen -A; /usr/sbin/sshd -D -e",
|
||||
},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{
|
||||
Name: "ctrl-port-fwd",
|
||||
ContainerPort: 31544,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "ssh",
|
||||
ContainerPort: 8022,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
ReadinessProbe: &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
TCPSocket: &corev1.TCPSocketAction{
|
||||
Port: intstr.FromInt(8022),
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 1,
|
||||
PeriodSeconds: 1,
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{
|
||||
Name: "sshd-config",
|
||||
MountPath: "/etc/ssh/sshd_config.d/reverse-tunnel.conf",
|
||||
SubPath: "reverse-tunnel.conf",
|
||||
},
|
||||
{
|
||||
Name: "sshd-config",
|
||||
MountPath: keysFilePath,
|
||||
SubPath: "authorized_keys",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Volumes: []corev1.Volume{
|
||||
{
|
||||
Name: "sshd-config",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
ConfigMap: &corev1.ConfigMapVolumeSource{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: "ssh-server-config",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func sshConfigMap(pubKey []byte) *corev1.ConfigMap {
|
||||
return &corev1.ConfigMap{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "ConfigMap",
|
||||
APIVersion: "v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ssh-server-config",
|
||||
Namespace: ns,
|
||||
},
|
||||
Data: map[string]string{
|
||||
"reverse-tunnel.conf": sshdConfig,
|
||||
"authorized_keys": string(pubKey),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func sshService() *corev1.Service {
|
||||
return &corev1.Service{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Service",
|
||||
APIVersion: "v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "control",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Type: corev1.ServiceTypeClusterIP,
|
||||
Selector: map[string]string{
|
||||
"app": "ssh-server",
|
||||
},
|
||||
Ports: []corev1.ServicePort{
|
||||
{
|
||||
Name: "tunnel",
|
||||
Port: 31544,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user