cmd/k8s-operator: simplify scope of e2e tests (#17076)
Removes ACL edits from e2e tests in favour of trying to simplify the tests and separate the actual test logic from the environment setup logic as much as possible. Also aims to fit in with the requirements that will generally be filled anyway for most devs working on the operator; in particular using tags that fit in with our documentation. Updates tailscale/corp#32085 Change-Id: I7659246e39ec0b7bcc4ec0a00c6310f25fe6fac2 Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
This commit is contained in:
Tom Proctor
GitHub
@@ -6,169 +6,91 @@ package e2e
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"github.com/tailscale/hujson"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"golang.org/x/oauth2/clientcredentials"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
"tailscale.com/internal/client/tailscale"
|
||||
)
|
||||
|
||||
const (
|
||||
e2eManagedComment = "// This is managed by the k8s-operator e2e tests"
|
||||
)
|
||||
|
||||
var (
|
||||
tsClient *tailscale.Client
|
||||
testGrants = map[string]string{
|
||||
"test-proxy": `{
|
||||
"src": ["tag:e2e-test-proxy"],
|
||||
"dst": ["tag:k8s-operator"],
|
||||
"app": {
|
||||
"tailscale.com/cap/kubernetes": [{
|
||||
"impersonate": {
|
||||
"groups": ["ts:e2e-test-proxy"],
|
||||
},
|
||||
}],
|
||||
},
|
||||
}`,
|
||||
}
|
||||
"tailscale.com/ipn/store/mem"
|
||||
"tailscale.com/tsnet"
|
||||
)
|
||||
|
||||
// This test suite is currently not run in CI.
|
||||
// It requires some setup not handled by this code:
|
||||
// - Kubernetes cluster with tailscale operator installed
|
||||
// - Current kubeconfig context set to connect to that cluster (directly, no operator proxy)
|
||||
// - Operator installed with --set apiServerProxyConfig.mode="true"
|
||||
// - ACLs that define tag:e2e-test-proxy tag. TODO(tomhjp): Can maybe replace this prereq onwards with an API key
|
||||
// - OAuth client ID and secret in TS_API_CLIENT_ID and TS_API_CLIENT_SECRET env
|
||||
// - OAuth client must have auth_keys and policy_file write for tag:e2e-test-proxy tag
|
||||
// - Kubernetes cluster with local kubeconfig for it (direct connection, no API server proxy)
|
||||
// - Tailscale operator installed with --set apiServerProxyConfig.mode="true"
|
||||
// - ACLs from acl.hujson
|
||||
// - OAuth client secret in TS_API_CLIENT_SECRET env, with at least auth_keys write scope and tag:k8s tag
|
||||
var (
|
||||
apiClient *tailscale.Client // For API calls to control.
|
||||
tailnetClient *tsnet.Server // For testing real tailnet traffic.
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
code, err := runTests(m)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
log.Printf("Error: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func runTests(m *testing.M) (int, error) {
|
||||
zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar()
|
||||
logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
|
||||
secret := os.Getenv("TS_API_CLIENT_SECRET")
|
||||
if secret != "" {
|
||||
secretParts := strings.Split(secret, "-")
|
||||
if len(secretParts) != 4 {
|
||||
return 0, errors.New("TS_API_CLIENT_SECRET is not valid")
|
||||
}
|
||||
ctx := context.Background()
|
||||
credentials := clientcredentials.Config{
|
||||
ClientID: secretParts[2],
|
||||
ClientSecret: secret,
|
||||
TokenURL: "https://login.tailscale.com/api/v2/oauth/token",
|
||||
Scopes: []string{"auth_keys"},
|
||||
}
|
||||
apiClient = tailscale.NewClient("-", nil)
|
||||
apiClient.HTTPClient = credentials.Client(ctx)
|
||||
|
||||
if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" {
|
||||
cleanup, err := setupClientAndACLs()
|
||||
caps := tailscale.KeyCapabilities{
|
||||
Devices: tailscale.KeyDeviceCapabilities{
|
||||
Create: tailscale.KeyDeviceCreateCapabilities{
|
||||
Reusable: false,
|
||||
Preauthorized: true,
|
||||
Ephemeral: true,
|
||||
Tags: []string{"tag:k8s"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
authKey, authKeyMeta, err := apiClient.CreateKeyWithExpiry(ctx, caps, 10*time.Minute)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
err = errors.Join(err, cleanup())
|
||||
}()
|
||||
defer apiClient.DeleteKey(context.Background(), authKeyMeta.ID)
|
||||
|
||||
tailnetClient = &tsnet.Server{
|
||||
Hostname: "test-proxy",
|
||||
Ephemeral: true,
|
||||
Store: &mem.Store{},
|
||||
AuthKey: authKey,
|
||||
}
|
||||
_, err = tailnetClient.Up(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tailnetClient.Close()
|
||||
}
|
||||
|
||||
return m.Run(), nil
|
||||
}
|
||||
|
||||
func setupClientAndACLs() (cleanup func() error, _ error) {
|
||||
ctx := context.Background()
|
||||
credentials := clientcredentials.Config{
|
||||
ClientID: os.Getenv("TS_API_CLIENT_ID"),
|
||||
ClientSecret: os.Getenv("TS_API_CLIENT_SECRET"),
|
||||
TokenURL: "https://login.tailscale.com/api/v2/oauth/token",
|
||||
Scopes: []string{"auth_keys", "policy_file"},
|
||||
}
|
||||
tsClient = tailscale.NewClient("-", nil)
|
||||
tsClient.HTTPClient = credentials.Client(ctx)
|
||||
|
||||
if err := patchACLs(ctx, tsClient, func(acls *hujson.Value) {
|
||||
for test, grant := range testGrants {
|
||||
deleteTestGrants(test, acls)
|
||||
addTestGrant(test, grant, acls)
|
||||
}
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return func() error {
|
||||
return patchACLs(ctx, tsClient, func(acls *hujson.Value) {
|
||||
for test := range testGrants {
|
||||
deleteTestGrants(test, acls)
|
||||
}
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
func patchACLs(ctx context.Context, tsClient *tailscale.Client, patchFn func(*hujson.Value)) error {
|
||||
acls, err := tsClient.ACLHuJSON(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hj, err := hujson.Parse([]byte(acls.ACL))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
patchFn(&hj)
|
||||
|
||||
hj.Format()
|
||||
acls.ACL = hj.String()
|
||||
if _, err := tsClient.SetACLHuJSON(ctx, *acls, true); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func addTestGrant(test, grant string, acls *hujson.Value) error {
|
||||
v, err := hujson.Parse([]byte(grant))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Add the managed comment to the first line of the grant object contents.
|
||||
v.Value.(*hujson.Object).Members[0].Name.BeforeExtra = hujson.Extra(fmt.Sprintf("%s: %s\n", e2eManagedComment, test))
|
||||
|
||||
if err := acls.Patch([]byte(fmt.Sprintf(`[{"op": "add", "path": "/grants/-", "value": %s}]`, v.String()))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteTestGrants(test string, acls *hujson.Value) error {
|
||||
grants := acls.Find("/grants")
|
||||
|
||||
var patches []string
|
||||
for i, g := range grants.Value.(*hujson.Array).Elements {
|
||||
members := g.Value.(*hujson.Object).Members
|
||||
if len(members) == 0 {
|
||||
continue
|
||||
}
|
||||
comment := strings.TrimSpace(string(members[0].Name.BeforeExtra))
|
||||
if name, found := strings.CutPrefix(comment, e2eManagedComment+": "); found && name == test {
|
||||
patches = append(patches, fmt.Sprintf(`{"op": "remove", "path": "/grants/%d"}`, i))
|
||||
}
|
||||
}
|
||||
|
||||
// Remove in reverse order so we don't affect the found indices as we mutate.
|
||||
slices.Reverse(patches)
|
||||
|
||||
if err := acls.Patch([]byte(fmt.Sprintf("[%s]", strings.Join(patches, ",")))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func objectMeta(namespace, name string) metav1.ObjectMeta {
|
||||
return metav1.ObjectMeta{
|
||||
Namespace: namespace,
|
||||
@@ -176,13 +98,25 @@ func objectMeta(namespace, name string) metav1.ObjectMeta {
|
||||
}
|
||||
}
|
||||
|
||||
func createAndCleanup(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) {
|
||||
func createAndCleanup(t *testing.T, cl client.Client, obj client.Object) {
|
||||
t.Helper()
|
||||
if err := cl.Create(ctx, obj); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
// Try to create the object first
|
||||
err := cl.Create(t.Context(), obj)
|
||||
if err != nil {
|
||||
if apierrors.IsAlreadyExists(err) {
|
||||
if updateErr := cl.Update(t.Context(), obj); updateErr != nil {
|
||||
t.Fatal(updateErr)
|
||||
}
|
||||
} else {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
if err := cl.Delete(ctx, obj); err != nil {
|
||||
// Use context.Background() for cleanup, as t.Context() is cancelled
|
||||
// just before cleanup functions are called.
|
||||
if err := cl.Delete(context.Background(), obj); err != nil {
|
||||
t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err)
|
||||
}
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user