cmd/k8s-operator: migrate to tailscale-client-go-v2 (#19010)

This commit modifies the kubernetes operator to use the `tailscale-client-go-v2`
package instead of the internal tailscale client it was previously using. This
now gives us the ability to expand out custom resources and features as they
become available via the API module.

The tailnet reconciler has also been modified to manage clients as tailnets
are created and removed, providing each subsequent reconciler with a single
`ClientProvider` that obtains a tailscale client for the respective tailnet
by name, or the operator's default when presented with a blank string.

Fixes: https://github.com/tailscale/corp/issues/38418

Signed-off-by: David Bond <davidsbond93@gmail.com>

k8s-operator/reconciler/tailnet/mocks_test.go

@@ -9,7 +9,9 @@ import (
 	"context"
 	"io"
 
-	"tailscale.com/internal/client/tailscale"
+	"tailscale.com/client/tailscale/v2"
+
+	"tailscale.com/k8s-operator/tsclient"
 )
 
 type (
@@ -18,28 +20,62 @@ type (
 		ErrorOnKeys     bool
 		ErrorOnServices bool
 	}
+
+	MockDeviceResource struct {
+		tsclient.DeviceResource
+
+		Error bool
+	}
+
+	MockKeyResource struct {
+		tsclient.KeyResource
+
+		Error bool
+	}
+
+	MockVIPServiceResource struct {
+		tsclient.VIPServiceResource
+
+		Error bool
+	}
 )
 
-func (m MockTailnetClient) Devices(_ context.Context, _ *tailscale.DeviceFieldsOpts) ([]*tailscale.Device, error) {
-	if m.ErrorOnDevices {
+func (m MockKeyResource) List(_ context.Context, _ bool) ([]tailscale.Key, error) {
+	if m.Error {
 		return nil, io.EOF
 	}
 
 	return nil, nil
 }
 
-func (m MockTailnetClient) Keys(_ context.Context) ([]string, error) {
-	if m.ErrorOnKeys {
+func (m MockDeviceResource) List(_ context.Context, _ ...tailscale.ListDevicesOptions) ([]tailscale.Device, error) {
+	if m.Error {
 		return nil, io.EOF
 	}
 
 	return nil, nil
 }
 
-func (m MockTailnetClient) ListVIPServices(_ context.Context) (*tailscale.VIPServiceList, error) {
-	if m.ErrorOnServices {
+func (m MockVIPServiceResource) List(_ context.Context) ([]tailscale.VIPService, error) {
+	if m.Error {
 		return nil, io.EOF
 	}
 
 	return nil, nil
 }
+
+func (m MockTailnetClient) Devices() tsclient.DeviceResource {
+	return MockDeviceResource{Error: m.ErrorOnDevices}
+}
+
+func (m MockTailnetClient) Keys() tsclient.KeyResource {
+	return MockKeyResource{Error: m.ErrorOnKeys}
+}
+
+func (m MockTailnetClient) VIPServices() tsclient.VIPServiceResource {
+	return MockVIPServiceResource{Error: m.ErrorOnServices}
+}
+
+func (m MockTailnetClient) LoginURL() string {
+	return ""
+}

k8s-operator/reconciler/tailnet/tailnet.go

@@ -12,12 +12,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/url"
 	"sync"
 	"time"
 
 	"go.uber.org/zap"
-	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/clientcredentials"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,12 +25,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/client/tailscale/v2"
 
-	"tailscale.com/internal/client/tailscale"
 	"tailscale.com/ipn"
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/k8s-operator/reconciler"
+	"tailscale.com/k8s-operator/tsclient"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
@@ -47,7 +47,8 @@ type (
 		tailscaleNamespace string
 		clock              tstime.Clock
 		logger             *zap.SugaredLogger
-		clientFunc         func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient
+		clientFunc         func(*tsapi.Tailnet, *corev1.Secret) tsclient.Client
+		registry           ClientRegistry
 
 		// Metrics related fields
 		mu       sync.Mutex
@@ -68,14 +69,18 @@ type (
 		Logger *zap.SugaredLogger
 		// ClientFunc is a function that takes tailscale credentials and returns an implementation for the Tailscale
 		// HTTP API. This should generally be nil unless needed for testing.
-		ClientFunc func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient
+		ClientFunc func(*tsapi.Tailnet, *corev1.Secret) tsclient.Client
+		// Registry is used to store and share initialized tailscale clients for use by other reconcilers.
+		Registry ClientRegistry
 	}
 
-	// The TailscaleClient interface describes types that interact with the Tailscale HTTP API.
-	TailscaleClient interface {
-		Devices(context.Context, *tailscale.DeviceFieldsOpts) ([]*tailscale.Device, error)
-		Keys(ctx context.Context) ([]string, error)
-		ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error)
+	// The ClientRegistry interface describes types that can store initialized tailscale clients for use by other
+	// reconcilers.
+	ClientRegistry interface {
+		// Add should store the given tsclient.Client implementation for a specified tailnet.
+		Add(tailnet string, client tsclient.Client, ready bool)
+		// Remove should remove any tsclient.Client implementation for a specified tailnet.
+		Remove(tailnet string)
 	}
 )
 
@@ -90,6 +95,7 @@ func NewReconciler(options ReconcilerOptions) *Reconciler {
 		clock:              options.Clock,
 		logger:             options.Logger.Named(reconcilerName),
 		clientFunc:         options.ClientFunc,
+		registry:           options.Registry,
 	}
 }
 
@@ -137,6 +143,7 @@ func (r *Reconciler) delete(ctx context.Context, tailnet *tsapi.Tailnet) (reconc
 	r.tailnets.Remove(tailnet.UID)
 	r.mu.Unlock()
 	gaugeTailnetResources.Set(int64(r.tailnets.Len()))
+	r.registry.Remove(tailnet.Name)
 
 	return reconcile.Result{}, nil
 }
@@ -193,11 +200,16 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet)
 		return reconcile.Result{RequeueAfter: time.Minute / 2}, nil
 	}
 
-	tsClient := r.createClient(ctx, tailnet, &secret)
+	tsClient, err := r.createClient(tailnet, &secret)
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to create tailnet client: %w", err)
+	}
 
 	// Second, we ensure the OAuth credentials supplied in the secret are valid and have the required scopes to access
 	// the various API endpoints required by the operator.
 	if ok := r.ensurePermissions(ctx, tsClient, tailnet); !ok {
+		r.registry.Add(tailnet.Name, tsClient, false)
+
 		if err = r.Status().Update(ctx, tailnet); err != nil {
 			return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
 		}
@@ -226,6 +238,8 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet)
 		return reconcile.Result{}, fmt.Errorf("failed to add finalizer to Tailnet %q: %w", tailnet.Name, err)
 	}
 
+	r.registry.Add(tailnet.Name, tsClient, true)
+
 	return reconcile.Result{}, nil
 }
 
@@ -235,9 +249,9 @@ const (
 	clientSecretKey = "client_secret"
 )
 
-func (r *Reconciler) createClient(ctx context.Context, tailnet *tsapi.Tailnet, secret *corev1.Secret) TailscaleClient {
+func (r *Reconciler) createClient(tailnet *tsapi.Tailnet, secret *corev1.Secret) (tsclient.Client, error) {
 	if r.clientFunc != nil {
-		return r.clientFunc(tailnet, secret)
+		return r.clientFunc(tailnet, secret), nil
 	}
 
 	baseURL := ipn.DefaultControlURL
@@ -245,38 +259,36 @@ func (r *Reconciler) createClient(ctx context.Context, tailnet *tsapi.Tailnet, s
 		baseURL = tailnet.Spec.LoginURL
 	}
 
-	credentials := clientcredentials.Config{
-		ClientID:     string(secret.Data[clientIDKey]),
-		ClientSecret: string(secret.Data[clientSecretKey]),
-		TokenURL:     baseURL + "/api/v2/oauth/token",
+	base, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse base URL %q: %w", baseURL, err)
 	}
 
-	source := credentials.TokenSource(ctx)
-	httpClient := oauth2.NewClient(ctx, source)
-
-	tsClient := tailscale.NewClient("-", nil)
-	tsClient.UserAgent = "tailscale-k8s-operator"
-	tsClient.HTTPClient = httpClient
-	tsClient.BaseURL = baseURL
-
-	return tsClient
+	return tsclient.Wrap(&tailscale.Client{
+		BaseURL:   base,
+		UserAgent: "tailscale-k8s-operator",
+		Auth: &tailscale.OAuth{
+			ClientID:     string(secret.Data[clientIDKey]),
+			ClientSecret: string(secret.Data[clientSecretKey]),
+		},
+	}), nil
 }
 
-func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient TailscaleClient, tailnet *tsapi.Tailnet) bool {
+func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient tsclient.Client, tailnet *tsapi.Tailnet) bool {
 	// Perform basic list requests here to confirm that the OAuth credentials referenced on the Tailnet resource
 	// can perform the basic operations required for the operator to function. This has a caveat of only performing
 	// read actions, as we don't want to create arbitrary keys and VIP services. However, it will catch when a user
 	// has completely forgotten an entire scope that's required.
 	var errs error
-	if _, err := tsClient.Devices(ctx, nil); err != nil {
+	if _, err := tsClient.Devices().List(ctx); err != nil {
 		errs = errors.Join(errs, fmt.Errorf("failed to list devices: %w", err))
 	}
 
-	if _, err := tsClient.Keys(ctx); err != nil {
+	if _, err := tsClient.Keys().List(ctx, false); err != nil {
 		errs = errors.Join(errs, fmt.Errorf("failed to list auth keys: %w", err))
 	}
 
-	if _, err := tsClient.ListVIPServices(ctx); err != nil {
+	if _, err := tsClient.VIPServices().List(ctx); err != nil {
 		errs = errors.Join(errs, fmt.Errorf("failed to list tailscale services: %w", err))
 	}
 

k8s-operator/reconciler/tailnet/tailnet_test.go

@@ -18,6 +18,7 @@ import (
 
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/k8s-operator/reconciler/tailnet"
+	"tailscale.com/k8s-operator/tsclient"
 	"tailscale.com/tstest"
 )
 
@@ -36,7 +37,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 		Secret             *corev1.Secret
 		ExpectsError       bool
 		ExpectedConditions []metav1.Condition
-		ClientFunc         func(*tsapi.Tailnet, *corev1.Secret) tailnet.TailscaleClient
+		ClientFunc         func(*tsapi.Tailnet, *corev1.Secret) tsclient.Client
 	}{
 		{
 			Name: "ignores-unknown-tailnet-requests",
@@ -201,7 +202,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					"client_secret": []byte("test"),
 				},
 			},
-			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
+			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tsclient.Client {
 				return &MockTailnetClient{ErrorOnDevices: true}
 			},
 			ExpectedConditions: []metav1.Condition{
@@ -240,7 +241,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					"client_secret": []byte("test"),
 				},
 			},
-			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
+			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tsclient.Client {
 				return &MockTailnetClient{ErrorOnServices: true}
 			},
 			ExpectedConditions: []metav1.Condition{
@@ -279,7 +280,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					"client_secret": []byte("test"),
 				},
 			},
-			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
+			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tsclient.Client {
 				return &MockTailnetClient{ErrorOnKeys: true}
 			},
 			ExpectedConditions: []metav1.Condition{
@@ -318,7 +319,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					"client_secret": []byte("test"),
 				},
 			},
-			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
+			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tsclient.Client {
 				return &MockTailnetClient{}
 			},
 			ExpectedConditions: []metav1.Condition{
@@ -349,6 +350,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 				Logger:             logger.Sugar(),
 				ClientFunc:         tc.ClientFunc,
 				TailscaleNamespace: "tailscale",
+				Registry:           tsclient.NewProvider(nil),
 			}
 
 			reconciler := tailnet.NewReconciler(opts)

k8s-operator/tsclient/client.go

@@ -0,0 +1,83 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package tsclient provides a mockable wrapper around the tailscale-client-go-v2 package for use by the Kubernetes
+// operator. It also contains the Provider type used to manage multiple instances of tailscale clients for different
+// tailnets.
+package tsclient
+
+import (
+	"context"
+
+	"tailscale.com/client/tailscale/v2"
+)
+
+type (
+	// The Client interface describes types that interact with the Tailscale API.
+	Client interface {
+		// LoginURL should return the url of the Tailscale control plane.
+		LoginURL() string
+		// Devices should return a DeviceResource implementation used to interact with the devices API.
+		Devices() DeviceResource
+		// Keys should return a KeyResource implementation used to interact with the keys API.
+		Keys() KeyResource
+		// VIPServices should return a VIPServiceResource implementation used to interact with the VIP services API.
+		VIPServices() VIPServiceResource
+	}
+
+	// The DeviceResource interface describes types that expose device related API endpoints.
+	DeviceResource interface {
+		// Delete should delete a device with a matching id.
+		Delete(ctx context.Context, id string) error
+		// List should return all devices based on the specified options.
+		List(ctx context.Context, opts ...tailscale.ListDevicesOptions) ([]tailscale.Device, error)
+		// Get should return the device with the matching identifier.
+		Get(ctx context.Context, id string) (*tailscale.Device, error)
+	}
+
+	// The KeyResource interface describes types that expose key related API endpoints.
+	KeyResource interface {
+		// CreateAuthKey should create and return a new auth key used to authenticate a device.
+		CreateAuthKey(ctx context.Context, ckr tailscale.CreateKeyRequest) (*tailscale.Key, error)
+		// List should return keys created by the caller or all keys if the provided boolean is set to true.
+		List(ctx context.Context, all bool) ([]tailscale.Key, error)
+	}
+
+	// The VIPServiceResource interface describes types that expose vip service related API endpoints.
+	VIPServiceResource interface {
+		// List should return all existing vip services within the tailnet.
+		List(ctx context.Context) ([]tailscale.VIPService, error)
+		// Delete should remove a named service from the tailnet.
+		Delete(ctx context.Context, name string) error
+		// Get should return the vip service associated with the given name.
+		Get(ctx context.Context, name string) (*tailscale.VIPService, error)
+		// CreateOrUpdate should update the provided vip service, creating it if it does not exist.
+		CreateOrUpdate(ctx context.Context, svc tailscale.VIPService) error
+	}
+
+	clientWrapper struct {
+		loginURL string
+		client   *tailscale.Client
+	}
+)
+
+// Wrap converts a given tailscale.Client into a Client.
+func Wrap(client *tailscale.Client) Client {
+	return &clientWrapper{client: client, loginURL: client.BaseURL.String()}
+}
+
+func (c *clientWrapper) Devices() DeviceResource {
+	return c.client.Devices()
+}
+
+func (c *clientWrapper) Keys() KeyResource {
+	return c.client.Keys()
+}
+
+func (c *clientWrapper) VIPServices() VIPServiceResource {
+	return c.client.VIPServices()
+}
+
+func (c *clientWrapper) LoginURL() string {
+	return c.loginURL
+}

k8s-operator/tsclient/provider.go

@@ -0,0 +1,77 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package tsclient
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+type (
+	// The Provider type is used to manage multiple Client implementations for different tailnets.
+	Provider struct {
+		defaultClient Client
+		mu            sync.RWMutex
+		clients       map[string]Client
+		readiness     map[string]bool
+	}
+)
+
+var (
+	// ErrClientNotFound is the error given when calling Provider.For with a tailnet that has not yet been registered
+	// with the provider.
+	ErrClientNotFound = errors.New("client not found")
+	// ErrNotReady is the error given when calling Provider.For with a tailnet that has not yet been declared as
+	// ready to use by the operator.
+	ErrNotReady = errors.New("tailnet not ready")
+)
+
+// NewProvider returns a new instance of the Provider type that uses the given Client implementation as the default
+// client. This client will be given when calling Provider.For with a blank tailnet name.
+func NewProvider(defaultClient Client) *Provider {
+	return &Provider{
+		defaultClient: defaultClient,
+		clients:       make(map[string]Client),
+		readiness:     make(map[string]bool),
+	}
+}
+
+// Add a Client implementation for a given tailnet.
+func (p *Provider) Add(tailnet string, client Client, ready bool) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.clients[tailnet] = client
+	p.readiness[tailnet] = ready
+}
+
+// Remove the Client implementation associated with the given tailnet.
+func (p *Provider) Remove(tailnet string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	delete(p.clients, tailnet)
+}
+
+// For returns a Client implementation associated with the given tailnet. Returns ErrClientNotFound if the given
+// tailnet does not exist. Use a blank tailnet name to obtain the default Client.
+func (p *Provider) For(tailnet string) (Client, error) {
+	if tailnet == "" {
+		return p.defaultClient, nil
+	}
+
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+
+	if client, ok := p.clients[tailnet]; ok {
+		if ready, _ := p.readiness[tailnet]; !ready {
+			return nil, fmt.Errorf("%w: %s", ErrNotReady, tailnet)
+		}
+
+		return client, nil
+	}
+
+	return nil, fmt.Errorf("%w: %s", ErrClientNotFound, tailnet)
+}