webnet/tailscale
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

all-kube: create Tailscale Service for HA kube-apiserver ProxyGroup (#16572)

Browse Source 
Adds a new reconciler for ProxyGroups of type kube-apiserver that will
provision a Tailscale Service for each replica to advertise. Adds two
new condition types to the ProxyGroup, TailscaleServiceValid and
TailscaleServiceConfigured, to post updates on the state of that
reconciler in a way that's consistent with the service-pg reconciler.
The created Tailscale Service name is configurable via a new ProxyGroup
field spec.kubeAPISserver.ServiceName, which expects a string of the
form "svc:<dns-label>".

Lots of supporting changes were needed to implement this in a way that's
consistent with other operator workflows, including:

* Pulled containerboot's ensureServicesUnadvertised and certManager into
  kube/ libraries to be shared with k8s-proxy. Use those in k8s-proxy to
  aid Service cert sharing between replicas and graceful Service shutdown.
* For certManager, add an initial wait to the cert loop to wait until
  the domain appears in the devices's netmap to avoid a guaranteed error
  on the first issue attempt when it's quick to start.
* Made several methods in ingress-for-pg.go and svc-for-pg.go into
  functions to share with the new reconciler
* Added a Resource struct to the owner refs stored in Tailscale Service
  annotations to be able to distinguish between Ingress- and ProxyGroup-
  based Services that need cleaning up in the Tailscale API.
* Added a ListVIPServices method to the internal tailscale client to aid
  cleaning up orphaned Services
* Support for reading config from a kube Secret, and partial support for
  config reloading, to prevent us having to force Pod restarts when
  config changes.
* Fixed up the zap logger so it's possible to set debug log level.

Updates #13358

Change-Id: Ia9607441157dd91fb9b6ecbc318eecbef446e116
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
This commit is contained in:
Tom Proctor
2025-07-21 11:03:21 +01:00
committed by GitHub
parent 5adde9e3f3
commit f421907c38
39 changed files with 2551 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/k8s-operator/api-server-proxy-pg.go
+479
View File
@@ -0,0 +1,479 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/internal/client/tailscale"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/tstime"
)
const (
proxyPGFinalizerName = "tailscale.com/kube-apiserver-finalizer"
// Reasons for KubeAPIServerProxyValid condition.
reasonKubeAPIServerProxyInvalid = "KubeAPIServerProxyInvalid"
reasonKubeAPIServerProxyValid = "KubeAPIServerProxyValid"
// Reasons for KubeAPIServerProxyConfigured condition.
reasonKubeAPIServerProxyConfigured = "KubeAPIServerProxyConfigured"
reasonKubeAPIServerProxyNoBackends = "KubeAPIServerProxyNoBackends"
)
// KubeAPIServerTSServiceReconciler reconciles the Tailscale Services required for an
// HA deployment of the API Server Proxy.
type KubeAPIServerTSServiceReconciler struct {
client.Client
recorder record.EventRecorder
logger *zap.SugaredLogger
tsClient tsClient
tsNamespace string
lc localClient
defaultTags []string
operatorID string // stableID of the operator's Tailscale device
clock tstime.Clock
}
// Reconcile is the entry point for the controller.
func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
logger := r.logger.With("ProxyGroup", req.Name)
logger.Debugf("starting reconcile")
defer logger.Debugf("reconcile finished")
pg := new(tsapi.ProxyGroup)
err = r.Get(ctx, req.NamespacedName, pg)
if apierrors.IsNotFound(err) {
// Request object not found, could have been deleted after reconcile request.
logger.Debugf("ProxyGroup not found, assuming it was deleted")
return res, nil
} else if err != nil {
return res, fmt.Errorf("failed to get ProxyGroup: %w", err)
}
serviceName := serviceNameForAPIServerProxy(pg)
logger = logger.With("Tailscale Service", serviceName)
if markedForDeletion(pg) {
logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up")
if err = r.maybeCleanup(ctx, serviceName, pg, logger); err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
return res, nil
}
return res, err
}
err = r.maybeProvision(ctx, serviceName, pg, logger)
if err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
// maybeProvision ensures that a Tailscale Service for this ProxyGroup exists
// and is up to date.
//
// Returns true if the operation resulted in a Tailscale Service update.
func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (err error) {
var dnsName string
oldPGStatus := pg.Status.DeepCopy()
defer func() {
podsAdvertising, podsErr := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName)
if podsErr != nil {
err = errors.Join(err, fmt.Errorf("failed to get number of advertised Pods: %w", podsErr))
// Continue, updating the status with the best available information.
}
// Update the ProxyGroup status with the Tailscale Service information
// Update the condition based on how many pods are advertising the service
conditionStatus := metav1.ConditionFalse
conditionReason := reasonKubeAPIServerProxyNoBackends
conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, pgReplicas(pg))
pg.Status.URL = ""
if podsAdvertising > 0 {
// At least one pod is advertising the service, consider it configured
conditionStatus = metav1.ConditionTrue
conditionReason = reasonKubeAPIServerProxyConfigured
if dnsName != "" {
pg.Status.URL = "https://" + dnsName
}
}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger)
if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) {
// An error encountered here should get returned by the Reconcile function.
err = errors.Join(err, r.Client.Status().Update(ctx, pg))
}
}()
if !tsoperator.ProxyGroupAvailable(pg) {
return nil
}
if !slices.Contains(pg.Finalizers, proxyPGFinalizerName) {
// This log line is printed exactly once during initial provisioning,
// because once the finalizer is in place this block gets skipped. So,
// this is a nice place to tell the operator that the high level,
// multi-reconcile operation is underway.
logger.Info("provisioning Tailscale Service for ProxyGroup")
pg.Finalizers = append(pg.Finalizers, proxyPGFinalizerName)
if err := r.Update(ctx, pg); err != nil {
return fmt.Errorf("failed to add finalizer: %w", err)
}
}
// 1. Check there isn't a Tailscale Service with the same hostname
// already created and not owned by this ProxyGroup.
existingTSSvc, err := r.tsClient.GetVIPService(ctx, serviceName)
if isErrorFeatureFlagNotEnabled(err) {
logger.Warn(msgFeatureFlagNotEnabled)
r.recorder.Event(pg, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msgFeatureFlagNotEnabled)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msgFeatureFlagNotEnabled, pg.Generation, r.clock, logger)
return nil
}
if err != nil && !isErrorTailscaleServiceNotFound(err) {
return fmt.Errorf("error getting Tailscale Service %q: %w", serviceName, err)
}
updatedAnnotations, err := exclusiveOwnerAnnotations(pg, r.operatorID, existingTSSvc)
if err != nil {
const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different Service name in the ProxyGroup's spec.kubeAPIServer.serviceName field"
msg := fmt.Sprintf("error ensuring exclusive ownership of Tailscale Service %s: %v. %s", serviceName, err, instr)
logger.Warn(msg)
r.recorder.Event(pg, corev1.EventTypeWarning, "InvalidTailscaleService", msg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger)
return nil
}
// After getting this far, we know the Tailscale Service is valid.
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger)
// Service tags are limited to matching the ProxyGroup's tags until we have
// support for querying peer caps for a Service-bound request.
serviceTags := r.defaultTags
if len(pg.Spec.Tags) > 0 {
serviceTags = pg.Spec.Tags.Stringify()
}
tsSvc := &tailscale.VIPService{
Name: serviceName,
Tags: serviceTags,
Ports: []string{"tcp:443"},
Comment: managedTSServiceComment,
Annotations: updatedAnnotations,
}
if existingTSSvc != nil {
tsSvc.Addrs = existingTSSvc.Addrs
}
// 2. Ensure the Tailscale Service exists and is up to date.
if existingTSSvc == nil ||
!slices.Equal(tsSvc.Tags, existingTSSvc.Tags) ||
!ownersAreSetAndEqual(tsSvc, existingTSSvc) ||
!slices.Equal(tsSvc.Ports, existingTSSvc.Ports) {
logger.Infof("Ensuring Tailscale Service exists and is up to date")
if err := r.tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil {
return fmt.Errorf("error creating Tailscale Service: %w", err)
}
}
// 3. Ensure that TLS Secret and RBAC exists.
tcd, err := tailnetCertDomain(ctx, r.lc)
if err != nil {
return fmt.Errorf("error determining DNS name base: %w", err)
}
dnsName = serviceName.WithoutPrefix() + "." + tcd
if err = r.ensureCertResources(ctx, pg, dnsName); err != nil {
return fmt.Errorf("error ensuring cert resources: %w", err)
}
// 4. Configure the Pods to advertise the Tailscale Service.
if err = r.maybeAdvertiseServices(ctx, pg, serviceName, logger); err != nil {
return fmt.Errorf("error updating advertised Tailscale Services: %w", err)
}
// 5. Clean up any stale Tailscale Services from previous resource versions.
if err = r.maybeDeleteStaleServices(ctx, pg, logger); err != nil {
return fmt.Errorf("failed to delete stale Tailscale Services: %w", err)
}
return nil
}
// maybeCleanup ensures that any resources, such as a Tailscale Service created for this Service, are cleaned up when the
// Service is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the Tailscale Service is only
// deleted if it does not contain any other owner references. If it does, the cleanup only removes the owner reference
// corresponding to this Service.
func (r *KubeAPIServerTSServiceReconciler) maybeCleanup(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (err error) {
ix := slices.Index(pg.Finalizers, proxyPGFinalizerName)
if ix < 0 {
logger.Debugf("no finalizer, nothing to do")
return nil
}
logger.Infof("Ensuring that Service %q is cleaned up", serviceName)
defer func() {
if err == nil {
err = r.deleteFinalizer(ctx, pg, logger)
}
}()
if _, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger); err != nil {
return fmt.Errorf("error deleting Tailscale Service: %w", err)
}
if err = cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg.Name, serviceName); err != nil {
return fmt.Errorf("failed to clean up cert resources: %w", err)
}
return nil
}
// maybeDeleteStaleServices deletes Services that have previously been created for
// this ProxyGroup but are no longer needed.
func (r *KubeAPIServerTSServiceReconciler) maybeDeleteStaleServices(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error {
serviceName := serviceNameForAPIServerProxy(pg)
svcs, err := r.tsClient.ListVIPServices(ctx)
if err != nil {
return fmt.Errorf("error listing Tailscale Services: %w", err)
}
for _, svc := range svcs.VIPServices {
if svc.Name == serviceName {
continue
}
owners, err := parseOwnerAnnotation(&svc)
if err != nil {
logger.Warnf("error parsing owner annotation for Tailscale Service %s: %v", svc.Name, err)
continue
}
if owners == nil || len(owners.OwnerRefs) != 1 || owners.OwnerRefs[0].OperatorID != r.operatorID {
continue
}
owner := owners.OwnerRefs[0]
if owner.Resource == nil || owner.Resource.Kind != "ProxyGroup" || owner.Resource.UID != string(pg.UID) {
continue
}
logger.Infof("Deleting Tailscale Service %s", svc.Name)
if err := r.tsClient.DeleteVIPService(ctx, svc.Name); err != nil && !isErrorTailscaleServiceNotFound(err) {
return fmt.Errorf("error deleting Tailscale Service %s: %w", svc.Name, err)
}
if err = cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg.Name, svc.Name); err != nil {
return fmt.Errorf("failed to clean up cert resources: %w", err)
}
}
return nil
}
func (r *KubeAPIServerTSServiceReconciler) deleteFinalizer(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error {
pg.Finalizers = slices.DeleteFunc(pg.Finalizers, func(f string) bool {
return f == proxyPGFinalizerName
})
logger.Debugf("ensure %q finalizer is removed", proxyPGFinalizerName)
if err := r.Update(ctx, pg); err != nil {
return fmt.Errorf("failed to remove finalizer %q: %w", proxyPGFinalizerName, err)
}
return nil
}
func (r *KubeAPIServerTSServiceReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string) error {
secret := certSecret(pg.Name, r.tsNamespace, domain, pg)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) {
s.Labels = secret.Labels
}); err != nil {
return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
}
role := certSecretRole(pg.Name, r.tsNamespace, domain)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
r.Labels = role.Labels
r.Rules = role.Rules
}); err != nil {
return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
}
rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) {
rb.Labels = rolebinding.Labels
rb.Subjects = rolebinding.Subjects
rb.RoleRef = rolebinding.RoleRef
}); err != nil {
return fmt.Errorf("failed to create or update RoleBinding %s: %w", rolebinding.Name, err)
}
return nil
}
func (r *KubeAPIServerTSServiceReconciler) maybeAdvertiseServices(ctx context.Context, pg *tsapi.ProxyGroup, serviceName tailcfg.ServiceName, logger *zap.SugaredLogger) error {
// Get all config Secrets for this ProxyGroup
cfgSecrets := &corev1.SecretList{}
if err := r.List(ctx, cfgSecrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil {
return fmt.Errorf("failed to list config Secrets: %w", err)
}
// Only advertise a Tailscale Service once the TLS certs required for
// serving it are available.
shouldBeAdvertised, err := hasCerts(ctx, r.Client, r.lc, r.tsNamespace, serviceName)
if err != nil {
return fmt.Errorf("error checking TLS credentials provisioned for Tailscale Service %q: %w", serviceName, err)
}
var advertiseServices []string
if shouldBeAdvertised {
advertiseServices = []string{serviceName.String()}
}
for _, s := range cfgSecrets.Items {
if len(s.Data[kubetypes.KubeAPIServerConfigFile]) == 0 {
continue
}
// Parse the existing config.
cfg, err := conf.Load(s.Data[kubetypes.KubeAPIServerConfigFile])
if err != nil {
return fmt.Errorf("error loading config from Secret %q: %w", s.Name, err)
}
if cfg.Parsed.APIServerProxy == nil {
return fmt.Errorf("config Secret %q does not contain APIServerProxy config", s.Name)
}
existingCfgSecret := s.DeepCopy()
var updated bool
if cfg.Parsed.APIServerProxy.ServiceName == nil || *cfg.Parsed.APIServerProxy.ServiceName != serviceName {
cfg.Parsed.APIServerProxy.ServiceName = &serviceName
updated = true
}
// Update the services to advertise if required.
if !slices.Equal(cfg.Parsed.AdvertiseServices, advertiseServices) {
cfg.Parsed.AdvertiseServices = advertiseServices
updated = true
}
if !updated {
continue
}
// Update the config Secret.
cfgB, err := json.Marshal(conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &cfg.Parsed,
})
if err != nil {
return err
}
s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB
if !apiequality.Semantic.DeepEqual(existingCfgSecret, s) {
logger.Debugf("Updating the Tailscale Services in ProxyGroup config Secret %s", s.Name)
if err := r.Update(ctx, &s); err != nil {
return err
}
}
}
return nil
}
func serviceNameForAPIServerProxy(pg *tsapi.ProxyGroup) tailcfg.ServiceName {
if pg.Spec.KubeAPIServer != nil && pg.Spec.KubeAPIServer.Hostname != "" {
return tailcfg.ServiceName("svc:" + pg.Spec.KubeAPIServer.Hostname)
}
return tailcfg.ServiceName("svc:" + pg.Name)
}
// exclusiveOwnerAnnotations returns the updated annotations required to ensure this
// instance of the operator is the exclusive owner. If the Tailscale Service is not
// nil, but does not contain an owner reference we return an error as this likely means
// that the Service was created by something other than a Tailscale Kubernetes operator.
// We also error if it is already owned by another operator instance, as we do not
// want to load balance a kube-apiserver ProxyGroup across multiple clusters.
func exclusiveOwnerAnnotations(pg *tsapi.ProxyGroup, operatorID string, svc *tailscale.VIPService) (map[string]string, error) {
ref := OwnerRef{
OperatorID: operatorID,
Resource: &Resource{
Kind: "ProxyGroup",
Name: pg.Name,
UID: string(pg.UID),
},
}
if svc == nil {
c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}}
json, err := json.Marshal(c)
if err != nil {
return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service's owner annotation contents: %w, please report this", err)
}
return map[string]string{
ownerAnnotation: string(json),
}, nil
}
o, err := parseOwnerAnnotation(svc)
if err != nil {
return nil, err
}
if o == nil || len(o.OwnerRefs) == 0 {
return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name)
}
if len(o.OwnerRefs) > 1 || o.OwnerRefs[0].OperatorID != operatorID {
return nil, fmt.Errorf("Tailscale Service %s is already owned by other operator(s) and cannot be shared across multiple clusters; configure a difference Service name to continue", svc.Name)
}
if o.OwnerRefs[0].Resource == nil {
return nil, fmt.Errorf("Tailscale Service %s exists, but does not reference an owning resource; not proceeding as this is likely a Service already owned by an Ingress", svc.Name)
}
if o.OwnerRefs[0].Resource.Kind != "ProxyGroup" || o.OwnerRefs[0].Resource.UID != string(pg.UID) {
return nil, fmt.Errorf("Tailscale Service %s is already owned by another resource: %#v; configure a difference Service name to continue", svc.Name, o.OwnerRefs[0].Resource)
}
if o.OwnerRefs[0].Resource.Name != pg.Name {
// ProxyGroup name can be updated in place.
o.OwnerRefs[0].Resource.Name = pg.Name
}
oBytes, err := json.Marshal(o)
if err != nil {
return nil, err
}
newAnnots := make(map[string]string, len(svc.Annotations)+1)
maps.Copy(newAnnots, svc.Annotations)
newAnnots[ownerAnnotation] = string(oBytes)
return newAnnots, nil
}
cmd/k8s-operator/api-server-proxy-pg_test.go
+384
View File
@@ -0,0 +1,384 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package main
import (
"encoding/json"
"reflect"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"tailscale.com/internal/client/tailscale"
"tailscale.com/ipn/ipnstate"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/tstest"
"tailscale.com/types/opt"
"tailscale.com/types/ptr"
)
func TestAPIServerProxyReconciler(t *testing.T) {
const (
pgName = "test-pg"
pgUID = "test-pg-uid"
ns = "operator-ns"
defaultDomain = "test-pg.ts.net"
)
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: pgName,
Generation: 1,
UID: pgUID,
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeKubernetesAPIServer,
},
Status: tsapi.ProxyGroupStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ProxyGroupAvailable),
Status: metav1.ConditionTrue,
ObservedGeneration: 1,
},
},
},
}
initialCfg := &conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("test-key"),
APIServerProxy: &conf.APIServerProxyConfig{
Enabled: opt.NewBool(true),
},
},
}
expectedCfg := *initialCfg
initialCfgB, err := json.Marshal(initialCfg)
if err != nil {
t.Fatalf("marshaling initial config: %v", err)
}
pgCfgSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pgName, 0),
Namespace: ns,
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig),
},
Data: map[string][]byte{
// Existing config should be preserved.
kubetypes.KubeAPIServerConfigFile: initialCfgB,
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pg, pgCfgSecret).
WithStatusSubresource(pg).
Build()
expectCfg := func(c *conf.VersionedConfig) {
t.Helper()
cBytes, err := json.Marshal(c)
if err != nil {
t.Fatalf("marshaling expected config: %v", err)
}
pgCfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cBytes
expectEqual(t, fc, pgCfgSecret)
}
ft := &fakeTSClient{}
ingressTSSvc := &tailscale.VIPService{
Name: "svc:some-ingress-hostname",
Comment: managedTSServiceComment,
Annotations: map[string]string{
// No resource field.
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`,
},
Ports: []string{"tcp:443"},
Tags: []string{"tag:k8s"},
Addrs: []string{"5.6.7.8"},
}
ft.CreateOrUpdateVIPService(t.Context(), ingressTSSvc)
lc := &fakeLocalClient{
status: &ipnstate.Status{
CurrentTailnet: &ipnstate.TailnetStatus{
MagicDNSSuffix: "ts.net",
},
},
}
r := &KubeAPIServerTSServiceReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
tsNamespace: ns,
logger: zap.Must(zap.NewDevelopment()).Sugar(),
recorder: record.NewFakeRecorder(10),
lc: lc,
clock: tstest.NewClock(tstest.ClockOpts{}),
operatorID: "self-id",
}
// Create a Tailscale Service that will conflict with the initial config.
if err := ft.CreateOrUpdateVIPService(t.Context(), &tailscale.VIPService{
Name: "svc:" + pgName,
}); err != nil {
t.Fatalf("creating initial Tailscale Service: %v", err)
}
expectReconciled(t, r, "", pgName)
pg.ObjectMeta.Finalizers = []string{proxyPGFinalizerName}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
expectMissing[corev1.Secret](t, fc, ns, defaultDomain)
expectMissing[rbacv1.Role](t, fc, ns, defaultDomain)
expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain)
expectEqual(t, fc, pgCfgSecret) // Unchanged.
// Delete Tailscale Service; should see Service created and valid condition updated to true.
if err := ft.DeleteVIPService(t.Context(), "svc:"+pgName); err != nil {
t.Fatalf("deleting initial Tailscale Service: %v", err)
}
expectReconciled(t, r, "", pgName)
tsSvc, err := ft.GetVIPService(t.Context(), "svc:"+pgName)
if err != nil {
t.Fatalf("getting Tailscale Service: %v", err)
}
if tsSvc == nil {
t.Fatalf("expected Tailscale Service to be created, but got nil")
}
expectedTSSvc := &tailscale.VIPService{
Name: "svc:" + pgName,
Comment: managedTSServiceComment,
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"test-pg-uid"}}]}`,
},
Ports: []string{"tcp:443"},
Tags: []string{"tag:k8s"},
Addrs: []string{"5.6.7.8"},
}
if !reflect.DeepEqual(tsSvc, expectedTSSvc) {
t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc)
}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
expectedCfg.APIServerProxy.ServiceName = ptr.To(tailcfg.ServiceName("svc:" + pgName))
expectCfg(&expectedCfg)
expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg))
expectEqual(t, fc, certSecretRole(pgName, ns, defaultDomain))
expectEqual(t, fc, certSecretRoleBinding(pg, ns, defaultDomain))
// Simulate certs being issued; should observe AdvertiseServices config change.
if err := populateTLSSecret(t.Context(), fc, pgName, defaultDomain); err != nil {
t.Fatalf("populating TLS Secret: %v", err)
}
expectReconciled(t, r, "", pgName)
expectedCfg.AdvertiseServices = []string{"svc:" + pgName}
expectCfg(&expectedCfg)
expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Unchanged status.
// Simulate Pod prefs updated with advertised services; should see Configured condition updated to true.
mustCreate(t, fc, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg-0",
Namespace: ns,
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState),
},
Data: map[string][]byte{
"_current-profile": []byte("profile-foo"),
"profile-foo": []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`),
},
})
expectReconciled(t, r, "", pgName)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
pg.Status.URL = "https://" + defaultDomain
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Rename the Tailscale Service - old one + cert resources should be cleaned up.
updatedServiceName := tailcfg.ServiceName("svc:test-pg-renamed")
updatedDomain := "test-pg-renamed.ts.net"
pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{
Hostname: updatedServiceName.WithoutPrefix(),
}
mustUpdate(t, fc, "", pgName, func(p *tsapi.ProxyGroup) {
p.Spec.KubeAPIServer = pg.Spec.KubeAPIServer
})
expectReconciled(t, r, "", pgName)
_, err = ft.GetVIPService(t.Context(), "svc:"+pgName)
if !isErrorTailscaleServiceNotFound(err) {
t.Fatalf("Expected 404, got: %v", err)
}
tsSvc, err = ft.GetVIPService(t.Context(), updatedServiceName)
if err != nil {
t.Fatalf("Expected renamed svc, got error: %v", err)
}
expectedTSSvc.Name = updatedServiceName
if !reflect.DeepEqual(tsSvc, expectedTSSvc) {
t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc)
}
// Check cfg and status reset until TLS certs are available again.
expectedCfg.APIServerProxy.ServiceName = ptr.To(updatedServiceName)
expectedCfg.AdvertiseServices = nil
expectCfg(&expectedCfg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
pg.Status.URL = ""
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg))
expectEqual(t, fc, certSecretRole(pgName, ns, updatedDomain))
expectEqual(t, fc, certSecretRoleBinding(pg, ns, updatedDomain))
expectMissing[corev1.Secret](t, fc, ns, defaultDomain)
expectMissing[rbacv1.Role](t, fc, ns, defaultDomain)
expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain)
// Check we get the new hostname in the status once ready.
if err := populateTLSSecret(t.Context(), fc, pgName, updatedDomain); err != nil {
t.Fatalf("populating TLS Secret: %v", err)
}
mustUpdate(t, fc, "operator-ns", "test-pg-0", func(s *corev1.Secret) {
s.Data["profile-foo"] = []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`)
})
expectReconciled(t, r, "", pgName)
expectedCfg.AdvertiseServices = []string{updatedServiceName.String()}
expectCfg(&expectedCfg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
pg.Status.URL = "https://" + updatedDomain
// Delete the ProxyGroup and verify Tailscale Service and cert resources are cleaned up.
if err := fc.Delete(t.Context(), pg); err != nil {
t.Fatalf("deleting ProxyGroup: %v", err)
}
expectReconciled(t, r, "", pgName)
expectMissing[corev1.Secret](t, fc, ns, updatedDomain)
expectMissing[rbacv1.Role](t, fc, ns, updatedDomain)
expectMissing[rbacv1.RoleBinding](t, fc, ns, updatedDomain)
_, err = ft.GetVIPService(t.Context(), updatedServiceName)
if !isErrorTailscaleServiceNotFound(err) {
t.Fatalf("Expected 404, got: %v", err)
}
// Ingress Tailscale Service should not be affected.
svc, err := ft.GetVIPService(t.Context(), ingressTSSvc.Name)
if err != nil {
t.Fatalf("getting ingress Tailscale Service: %v", err)
}
if !reflect.DeepEqual(svc, ingressTSSvc) {
t.Fatalf("expected ingress Tailscale Service to be unmodified %+v, got %+v", ingressTSSvc, svc)
}
}
func TestExclusiveOwnerAnnotations(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "pg1",
UID: "pg1-uid",
},
}
const (
selfOperatorID = "self-id"
pg1Owner = `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg1","uid":"pg1-uid"}}]}`
)
for name, tc := range map[string]struct {
svc *tailscale.VIPService
wantErr string
}{
"no_svc": {
svc: nil,
},
"empty_svc": {
svc: &tailscale.VIPService{},
wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator",
},
"already_owner": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: pg1Owner,
},
},
},
"already_owner_name_updated": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"old-pg1-name","uid":"pg1-uid"}}]}`,
},
},
},
"preserves_existing_annotations": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
"existing": "annotation",
ownerAnnotation: pg1Owner,
},
},
},
"owned_by_another_operator": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`,
},
},
wantErr: "already owned by other operator(s)",
},
"owned_by_an_ingress": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, // Ingress doesn't set Resource field (yet).
},
},
wantErr: "does not reference an owning resource",
},
"owned_by_another_pg": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg2","uid":"pg2-uid"}}]}`,
},
},
wantErr: "already owned by another resource",
},
} {
t.Run(name, func(t *testing.T) {
got, err := exclusiveOwnerAnnotations(pg, "self-id", tc.svc)
if tc.wantErr != "" {
if !strings.Contains(err.Error(), tc.wantErr) {
t.Errorf("exclusiveOwnerAnnotations() error = %v, wantErr %v", err, tc.wantErr)
}
} else if diff := cmp.Diff(pg1Owner, got[ownerAnnotation]); diff != "" {
t.Errorf("exclusiveOwnerAnnotations() mismatch (-want +got):\n%s", diff)
}
if tc.svc == nil {
return // Don't check annotations being preserved.
}
for k, v := range tc.svc.Annotations {
if k == ownerAnnotation {
continue
}
if got[k] != v {
t.Errorf("exclusiveOwnerAnnotations() did not preserve annotation %q: got %q, want %q", k, got[k], v)
}
}
})
}
}
func omitPGStatusConditionMessages(p *tsapi.ProxyGroup) {
for i := range p.Status.Conditions {
// Don't bother validating the message.
p.Status.Conditions[i].Message = ""
}
}
cmd/k8s-operator/proxy.go → cmd/k8s-operator/api-server-proxy.go
View File
cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml
+45 -11
View File
@@ -20,6 +20,10 @@ spec:
jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
name: Status
type: string
- description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver.
jsonPath: .status.url
name: URL
type: string
- description: ProxyGroup type.
jsonPath: .spec.type
name: Type
@@ -32,15 +36,22 @@ spec:
openAPIV3Schema:
description: |-
ProxyGroup defines a set of Tailscale devices that will act as proxies.
Currently only egress ProxyGroups are supported.
Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver
proxies. In addition to running a highly available set of proxies, ingress
and egress ProxyGroups also allow for serving many annotated Services from a
single set of proxies to minimise resource consumption.
Use the tailscale.com/proxy-group annotation on a Service to specify that
the egress proxy should be implemented by a ProxyGroup instead of a single
dedicated proxy. In addition to running a highly available set of proxies,
ProxyGroup also allows for serving many annotated Services from a single
set of proxies to minimise resource consumption.
For ingress and egress, use the tailscale.com/proxy-group annotation on a
Service to specify that the proxy should be implemented by a ProxyGroup
instead of a single dedicated proxy.
More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
More info:
* https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
* https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
For kube-apiserver, the ProxyGroup is a standalone resource. Use the
spec.kubeAPIServer field to configure options specific to the kube-apiserver
ProxyGroup type.
type: object
required:
- spec
@@ -83,6 +94,14 @@ spec:
ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
type: object
properties:
hostname:
description: |-
Hostname is the hostname with which to expose the Kubernetes API server
proxies. Must be a valid DNS label no longer than 63 characters. If not
specified, the name of the ProxyGroup is used as the hostname. Must be
unique across the whole tailnet.
type: string
pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$
mode:
description: |-
Mode to run the API server proxy in. Supported modes are auth and noauth.
@@ -141,10 +160,20 @@ spec:
conditions:
description: |-
List of status conditions to indicate the status of the ProxyGroup
resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`.
`ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled
and ready. `ProxyGroupAvailable` indicates that at least one proxy is
ready to serve traffic.
resources. Known condition types include `ProxyGroupReady` and
`ProxyGroupAvailable`.
* `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and
all expected conditions are true.
* `ProxyGroupAvailable` indicates that at least one proxy is ready to
serve traffic.
For ProxyGroups of type kube-apiserver, there are two additional conditions:
* `KubeAPIServerProxyConfigured` indicates that at least one API server
proxy is configured and ready to serve traffic.
* `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is
valid.
type: array
items:
description: Condition contains details for one aspect of the current state of this API Resource.
@@ -231,6 +260,11 @@ spec:
x-kubernetes-list-map-keys:
- hostname
x-kubernetes-list-type: map
url:
description: |-
URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if
any. Only applies to ProxyGroups of type kube-apiserver.
type: string
served: true
storage: true
subresources:
cmd/k8s-operator/deploy/manifests/operator.yaml
+45 -11
View File
@@ -2873,6 +2873,10 @@ spec:
jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
name: Status
type: string
- description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver.
jsonPath: .status.url
name: URL
type: string
- description: ProxyGroup type.
jsonPath: .spec.type
name: Type
@@ -2885,15 +2889,22 @@ spec:
openAPIV3Schema:
description: |-
ProxyGroup defines a set of Tailscale devices that will act as proxies.
Currently only egress ProxyGroups are supported.
Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver
proxies. In addition to running a highly available set of proxies, ingress
and egress ProxyGroups also allow for serving many annotated Services from a
single set of proxies to minimise resource consumption.
Use the tailscale.com/proxy-group annotation on a Service to specify that
the egress proxy should be implemented by a ProxyGroup instead of a single
dedicated proxy. In addition to running a highly available set of proxies,
ProxyGroup also allows for serving many annotated Services from a single
set of proxies to minimise resource consumption.
For ingress and egress, use the tailscale.com/proxy-group annotation on a
Service to specify that the proxy should be implemented by a ProxyGroup
instead of a single dedicated proxy.
More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
More info:
* https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
* https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
For kube-apiserver, the ProxyGroup is a standalone resource. Use the
spec.kubeAPIServer field to configure options specific to the kube-apiserver
ProxyGroup type.
properties:
apiVersion:
description: |-
@@ -2929,6 +2940,14 @@ spec:
KubeAPIServer contains configuration specific to the kube-apiserver
ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
properties:
hostname:
description: |-
Hostname is the hostname with which to expose the Kubernetes API server
proxies. Must be a valid DNS label no longer than 63 characters. If not
specified, the name of the ProxyGroup is used as the hostname. Must be
unique across the whole tailnet.
pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$
type: string
mode:
description: |-
Mode to run the API server proxy in. Supported modes are auth and noauth.
@@ -2990,10 +3009,20 @@ spec:
conditions:
description: |-
List of status conditions to indicate the status of the ProxyGroup
resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`.
`ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled
and ready. `ProxyGroupAvailable` indicates that at least one proxy is
ready to serve traffic.
resources. Known condition types include `ProxyGroupReady` and
`ProxyGroupAvailable`.
* `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and
all expected conditions are true.
* `ProxyGroupAvailable` indicates that at least one proxy is ready to
serve traffic.
For ProxyGroups of type kube-apiserver, there are two additional conditions:
* `KubeAPIServerProxyConfigured` indicates that at least one API server
proxy is configured and ready to serve traffic.
* `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is
valid.
items:
description: Condition contains details for one aspect of the current state of this API Resource.
properties:
@@ -3080,6 +3109,11 @@ spec:
x-kubernetes-list-map-keys:
- hostname
x-kubernetes-list-type: map
url:
description: |-
URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if
any. Only applies to ProxyGroups of type kube-apiserver.
type: string
type: object
required:
- spec
cmd/k8s-operator/egress-eps_test.go
+2 -1
View File
@@ -20,6 +20,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/fake"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/util/mak"
)
@@ -200,7 +201,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-0", pg),
Namespace: "operator-ns",
Labels: pgSecretLabels(pg, "state"),
Labels: pgSecretLabels(pg, kubetypes.LabelSecretTypeState),
},
}
return p, s
cmd/k8s-operator/ingress-for-pg.go
+45 -32
View File
@@ -248,7 +248,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
return false, nil
}
// 3. Ensure that TLS Secret and RBAC exists
tcd, err := r.tailnetCertDomain(ctx)
tcd, err := tailnetCertDomain(ctx, r.lc)
if err != nil {
return false, fmt.Errorf("error determining DNS name base: %w", err)
}
@@ -358,7 +358,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
}
// 6. Update Ingress status if ProxyGroup Pods are ready.
count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName)
count, err := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName)
if err != nil {
return false, fmt.Errorf("failed to check if any Pods are configured: %w", err)
}
@@ -370,7 +370,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
ing.Status.LoadBalancer.Ingress = nil
default:
var ports []networkingv1.IngressPortStatus
hasCerts, err := r.hasCerts(ctx, serviceName)
hasCerts, err := hasCerts(ctx, r.Client, r.lc, r.tsNamespace, serviceName)
if err != nil {
return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err)
}
@@ -481,7 +481,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
delete(cfg.Services, tsSvcName)
serveConfigChanged = true
}
if err := r.cleanupCertResources(ctx, proxyGroupName, tsSvcName); err != nil {
if err := cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, proxyGroupName, tsSvcName); err != nil {
return false, fmt.Errorf("failed to clean up cert resources: %w", err)
}
}
@@ -557,7 +557,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
}
// 3. Clean up any cluster resources
if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil {
if err := cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg, serviceName); err != nil {
return false, fmt.Errorf("failed to clean up cert resources: %w", err)
}
@@ -634,8 +634,8 @@ type localClient interface {
}
// tailnetCertDomain returns the base domain (TCD) of the current tailnet.
func (r *HAIngressReconciler) tailnetCertDomain(ctx context.Context) (string, error) {
st, err := r.lc.StatusWithoutPeers(ctx)
func tailnetCertDomain(ctx context.Context, lc localClient) (string, error) {
st, err := lc.StatusWithoutPeers(ctx)
if err != nil {
return "", fmt.Errorf("error getting tailscale status: %w", err)
}
@@ -761,7 +761,7 @@ const (
func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, logger *zap.SugaredLogger) (err error) {
// Get all config Secrets for this ProxyGroup.
secrets := &corev1.SecretList{}
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil {
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil {
return fmt.Errorf("failed to list config Secrets: %w", err)
}
@@ -773,7 +773,7 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
// The only exception is Ingresses with an HTTP endpoint enabled - if an
// Ingress has an HTTP endpoint enabled, it will be advertised even if the
// TLS cert is not yet provisioned.
hasCert, err := a.hasCerts(ctx, serviceName)
hasCert, err := hasCerts(ctx, a.Client, a.lc, a.tsNamespace, serviceName)
if err != nil {
return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err)
}
@@ -822,10 +822,10 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
return nil
}
func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) {
func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, pgName string, serviceName tailcfg.ServiceName) (int, error) {
// Get all state Secrets for this ProxyGroup.
secrets := &corev1.SecretList{}
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil {
if err := cl.List(ctx, secrets, client.InNamespace(tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil {
return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err)
}
@@ -859,7 +859,14 @@ type ownerAnnotationValue struct {
// Kubernetes operator instance.
type OwnerRef struct {
// OperatorID is the stable ID of the operator's Tailscale device.
OperatorID string `json:"operatorID,omitempty"`
OperatorID string `json:"operatorID,omitempty"`
Resource *Resource `json:"resource,omitempty"` // optional, used to identify the ProxyGroup that owns this Tailscale Service.
}
type Resource struct {
Kind string `json:"kind,omitempty"` // "ProxyGroup"
Name string `json:"name,omitempty"` // Name of the ProxyGroup that owns this Tailscale Service. Informational only.
UID string `json:"uid,omitempty"` // UID of the ProxyGroup that owns this Tailscale Service.
}
// ownerAnnotations returns the updated annotations required to ensure this
@@ -891,6 +898,9 @@ func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string]
if slices.Contains(o.OwnerRefs, ref) { // up to date
return svc.Annotations, nil
}
if o.OwnerRefs[0].Resource != nil {
return nil, fmt.Errorf("Tailscale Service %s is owned by another resource: %#v; cannot be reused for an Ingress", svc.Name, o.OwnerRefs[0].Resource)
}
o.OwnerRefs = append(o.OwnerRefs, ref)
json, err := json.Marshal(o)
if err != nil {
@@ -949,7 +959,7 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi
}); err != nil {
return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
}
rolebinding := certSecretRoleBinding(pg.Name, r.tsNamespace, domain)
rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) {
// Labels and subjects might have changed if the Ingress has been updated to use a
// different ProxyGroup.
@@ -963,19 +973,19 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi
// cleanupCertResources ensures that the TLS Secret and associated RBAC
// resources that allow proxies to read/write to the Secret are deleted.
func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error {
domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name))
func cleanupCertResources(ctx context.Context, cl client.Client, lc localClient, tsNamespace, pgName string, serviceName tailcfg.ServiceName) error {
domainName, err := dnsNameForService(ctx, lc, serviceName)
if err != nil {
return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", name, err)
return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", serviceName, err)
}
labels := certResourceLabels(pgName, domainName)
if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
if err := cl.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil {
return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err)
}
if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
if err := cl.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil {
return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err)
}
if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
if err := cl.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil {
return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err)
}
return nil
@@ -1018,17 +1028,17 @@ func certSecretRole(pgName, namespace, domain string) *rbacv1.Role {
// certSecretRoleBinding creates a RoleBinding for Role that will allow proxies
// to manage the TLS Secret for the given domain. Domain must be a valid
// Kubernetes resource name.
func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding {
func certSecretRoleBinding(pg *tsapi.ProxyGroup, namespace, domain string) *rbacv1.RoleBinding {
return &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: domain,
Namespace: namespace,
Labels: certResourceLabels(pgName, domain),
Labels: certResourceLabels(pg.Name, domain),
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: pgName,
Name: pgServiceAccountName(pg),
Namespace: namespace,
},
},
@@ -1041,14 +1051,17 @@ func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding
// certSecret creates a Secret that will store the TLS certificate and private
// key for the given domain. Domain must be a valid Kubernetes resource name.
func certSecret(pgName, namespace, domain string, ing *networkingv1.Ingress) *corev1.Secret {
func certSecret(pgName, namespace, domain string, parent client.Object) *corev1.Secret {
labels := certResourceLabels(pgName, domain)
labels[kubetypes.LabelSecretType] = "certs"
labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts
// Labels that let us identify the Ingress resource lets us reconcile
// the Ingress when the TLS Secret is updated (for example, when TLS
// certs have been provisioned).
labels[LabelParentName] = ing.Name
labels[LabelParentNamespace] = ing.Namespace
labels[LabelParentType] = strings.ToLower(parent.GetObjectKind().GroupVersionKind().Kind)
labels[LabelParentName] = parent.GetName()
if ns := parent.GetNamespace(); ns != "" {
labels[LabelParentNamespace] = ns
}
return &corev1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
@@ -1076,9 +1089,9 @@ func certResourceLabels(pgName, domain string) map[string]string {
}
// dnsNameForService returns the DNS name for the given Tailscale Service's name.
func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
func dnsNameForService(ctx context.Context, lc localClient, svc tailcfg.ServiceName) (string, error) {
s := svc.WithoutPrefix()
tcd, err := r.tailnetCertDomain(ctx)
tcd, err := tailnetCertDomain(ctx, lc)
if err != nil {
return "", fmt.Errorf("error determining DNS name base: %w", err)
}
@@ -1086,14 +1099,14 @@ func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg
}
// hasCerts checks if the TLS Secret for the given service has non-zero cert and key data.
func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceName) (bool, error) {
domain, err := r.dnsNameForService(ctx, svc)
func hasCerts(ctx context.Context, cl client.Client, lc localClient, ns string, svc tailcfg.ServiceName) (bool, error) {
domain, err := dnsNameForService(ctx, lc, svc)
if err != nil {
return false, fmt.Errorf("failed to get DNS name for service: %w", err)
}
secret := &corev1.Secret{}
err = r.Get(ctx, client.ObjectKey{
Namespace: r.tsNamespace,
err = cl.Get(ctx, client.ObjectKey{
Namespace: ns,
Name: domain,
}, secret)
if err != nil {
cmd/k8s-operator/ingress-for-pg_test.go
+25 -7
View File
@@ -75,8 +75,13 @@ func TestIngressPGReconciler(t *testing.T) {
// Verify that Role and RoleBinding have been created for the first Ingress.
// Do not verify the cert Secret as that was already verified implicitly above.
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg",
},
}
expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net"))
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
@@ -137,7 +142,7 @@ func TestIngressPGReconciler(t *testing.T) {
// Verify that Role and RoleBinding have been created for the second Ingress.
// Do not verify the cert Secret as that was already verified implicitly above.
expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-other-svc.ts.net"))
// Verify first Ingress is still working
verifyServeConfig(t, fc, "svc:my-svc", false)
@@ -186,7 +191,12 @@ func TestIngressPGReconciler(t *testing.T) {
})
expectReconciled(t, ingPGR, "default", "test-ingress")
expectEqual(t, fc, certSecretRole("test-pg-second", "operator-ns", "my-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding("test-pg-second", "operator-ns", "my-svc.ts.net"))
pg = &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg-second",
},
}
expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net"))
// Delete the first Ingress and verify cleanup
if err := fc.Delete(context.Background(), ing); err != nil {
@@ -515,7 +525,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg-0",
Namespace: "operator-ns",
Labels: pgSecretLabels("test-pg", "state"),
Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeState),
},
Data: map[string][]byte{
"_current-profile": []byte("profile-foo"),
@@ -686,6 +696,14 @@ func TestOwnerAnnotations(t *testing.T) {
ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"},{"operatorID":"self-id"}]}`,
},
},
"owned_by_proxygroup": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"1234-UID"}}]}`,
},
},
wantErr: "owned by another resource",
},
} {
t.Run(name, func(t *testing.T) {
got, err := ownerAnnotations("self-id", tc.svc)
@@ -708,7 +726,7 @@ func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain stri
kubetypes.LabelManaged: "true",
labelProxyGroup: pgName,
labelDomain: domain,
kubetypes.LabelSecretType: "certs",
kubetypes.LabelSecretType: kubetypes.LabelSecretTypeCerts,
},
},
Type: corev1.SecretTypeTLS,
@@ -806,7 +824,7 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expec
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pgName, 0),
Namespace: "operator-ns",
Labels: pgSecretLabels(pgName, "config"),
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig),
},
Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)),
@@ -845,7 +863,7 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) {
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pgName, 0),
Namespace: "operator-ns",
Labels: pgSecretLabels(pgName, "config"),
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig),
},
Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte("{}"),
cmd/k8s-operator/operator.go
+66 -4
View File
@@ -123,7 +123,7 @@ func main() {
defer s.Close()
restConfig := config.GetConfigOrDie()
if mode != apiServerProxyModeDisabled {
ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled)
ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled, true)
if err != nil {
zlog.Fatalf("error creating API server proxy: %v", err)
}
@@ -633,6 +633,32 @@ func runReconcilers(opts reconcilerOpts) {
startlog.Fatalf("could not create Recorder reconciler: %v", err)
}
// kube-apiserver's Tailscale Service reconciler.
err = builder.
ControllerManagedBy(mgr).
For(&tsapi.ProxyGroup{}, builder.WithPredicates(
predicate.NewPredicateFuncs(func(obj client.Object) bool {
pg, ok := obj.(*tsapi.ProxyGroup)
return ok && pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer
}),
)).
Named("kube-apiserver-ts-service-reconciler").
Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(kubeAPIServerPGsFromSecret(mgr.GetClient(), startlog))).
Complete(&KubeAPIServerTSServiceReconciler{
Client: mgr.GetClient(),
recorder: eventRecorder,
logger: opts.log.Named("kube-apiserver-ts-service-reconciler"),
tsClient: opts.tsClient,
tsNamespace: opts.tailscaleNamespace,
lc: lc,
defaultTags: strings.Split(opts.proxyTags, ","),
operatorID: id,
clock: tstime.DefaultClock{},
})
if err != nil {
startlog.Fatalf("could not create Kubernetes API server Tailscale Service reconciler: %v", err)
}
// ProxyGroup reconciler.
ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
@@ -1214,7 +1240,7 @@ func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
return nil
}
if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" {
if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState {
return nil
}
pg, ok := o.GetLabels()[LabelParentName]
@@ -1304,7 +1330,7 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.
func isTLSSecret(secret *corev1.Secret) bool {
return secret.Type == corev1.SecretTypeTLS &&
secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" &&
secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "certs" &&
secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeCerts &&
secret.ObjectMeta.Labels[labelDomain] != "" &&
secret.ObjectMeta.Labels[labelProxyGroup] != ""
}
@@ -1312,7 +1338,7 @@ func isTLSSecret(secret *corev1.Secret) bool {
func isPGStateSecret(secret *corev1.Secret) bool {
return secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" &&
secret.ObjectMeta.Labels[LabelParentType] == "proxygroup" &&
secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "state"
secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeState
}
// HAIngressesFromSecret returns a handler that returns reconcile requests for
@@ -1394,6 +1420,42 @@ func HAServicesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.M
}
}
// kubeAPIServerPGsFromSecret finds ProxyGroups of type "kube-apiserver" that
// need to be reconciled after a ProxyGroup-owned Secret is updated.
func kubeAPIServerPGsFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
secret, ok := o.(*corev1.Secret)
if !ok {
logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret")
return nil
}
if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" ||
secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" {
return nil
}
var pg tsapi.ProxyGroup
if err := cl.Get(ctx, types.NamespacedName{Name: secret.ObjectMeta.Labels[LabelParentName]}, &pg); err != nil {
logger.Infof("error getting ProxyGroup %s: %v", secret.ObjectMeta.Labels[LabelParentName], err)
return nil
}
if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: secret.ObjectMeta.Labels[LabelParentNamespace],
Name: secret.ObjectMeta.Labels[LabelParentName],
},
},
}
}
}
// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
// user-created ExternalName Services that should be exposed on this ProxyGroup.
func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
cmd/k8s-operator/proxygroup.go
+84 -47
View File
@@ -68,8 +68,7 @@ const (
//
// tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was
// first introduced.
pgMinCapabilityVersion = 106
kubeAPIServerConfigFile = "config.hujson"
pgMinCapabilityVersion = 106
)
var (
@@ -127,6 +126,10 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
}
if done, err := r.maybeCleanup(ctx, pg); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
} else if !done {
logger.Debugf("ProxyGroup resource cleanup not yet finished, will retry...")
@@ -158,7 +161,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
logger.Infof("ensuring ProxyGroup is set up")
pg.Finalizers = append(pg.Finalizers, FinalizerName)
if err := r.Update(ctx, pg); err != nil {
return r.notReadyErrf(pg, "error adding finalizer: %w", err)
return r.notReadyErrf(pg, logger, "error adding finalizer: %w", err)
}
}
@@ -174,31 +177,25 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
if apierrors.IsNotFound(err) {
msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q does not (yet) exist", proxyClassName)
logger.Info(msg)
return r.notReady(reasonProxyGroupCreating, msg)
return notReady(reasonProxyGroupCreating, msg)
}
if err != nil {
return r.notReadyErrf(pg, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err)
return r.notReadyErrf(pg, logger, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName)
logger.Info(msg)
return r.notReady(reasonProxyGroupCreating, msg)
return notReady(reasonProxyGroupCreating, msg)
}
}
if err := r.validate(ctx, pg, proxyClass, logger); err != nil {
return r.notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err))
return notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err))
}
staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass)
if err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
msg := fmt.Sprintf("optimistic lock error, retrying: %s", nrr.message)
logger.Info(msg)
return r.notReady(reasonProxyGroupCreating, msg)
} else {
return nil, nrr, err
}
return nil, nrr, err
}
return staticEndpoints, nrr, nil
@@ -299,9 +296,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
reason := reasonProxyGroupCreationFailed
msg := fmt.Sprintf("error provisioning NodePort Services for static endpoints: %v", err)
r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg)
return r.notReady(reason, msg)
return notReady(reason, msg)
}
return r.notReadyErrf(pg, "error provisioning NodePort Services for static endpoints: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning NodePort Services for static endpoints: %w", err)
}
}
@@ -312,9 +309,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
reason := reasonProxyGroupCreationFailed
msg := fmt.Sprintf("error provisioning config Secrets: %v", err)
r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg)
return r.notReady(reason, msg)
return notReady(reason, msg)
}
return r.notReadyErrf(pg, "error provisioning config Secrets: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning config Secrets: %w", err)
}
// State secrets are precreated so we can use the ProxyGroup CR as their owner ref.
@@ -325,7 +322,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning state Secrets: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning state Secrets: %w", err)
}
}
@@ -339,7 +336,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning ServiceAccount: %w", err)
}
}
@@ -350,7 +347,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
r.Rules = role.Rules
}); err != nil {
return r.notReadyErrf(pg, "error provisioning Role: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning Role: %w", err)
}
roleBinding := pgRoleBinding(pg, r.tsNamespace)
@@ -361,7 +358,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
r.RoleRef = roleBinding.RoleRef
r.Subjects = roleBinding.Subjects
}); err != nil {
return r.notReadyErrf(pg, "error provisioning RoleBinding: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning RoleBinding: %w", err)
}
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
@@ -371,7 +368,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp)
}); err != nil {
return r.notReadyErrf(pg, "error provisioning egress ConfigMap %q: %w", cm.Name, err)
return r.notReadyErrf(pg, logger, "error provisioning egress ConfigMap %q: %w", cm.Name, err)
}
}
@@ -381,7 +378,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning ingress ConfigMap %q: %w", cm.Name, err)
return r.notReadyErrf(pg, logger, "error provisioning ingress ConfigMap %q: %w", cm.Name, err)
}
}
@@ -391,7 +388,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
}
ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass)
if err != nil {
return r.notReadyErrf(pg, "error generating StatefulSet spec: %w", err)
return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err)
}
cfg := &tailscaleSTSConfig{
proxyType: string(pg.Spec.Type),
@@ -404,7 +401,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning StatefulSet: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning StatefulSet: %w", err)
}
mo := &metricsOpts{
@@ -414,11 +411,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
proxyType: "proxygroup",
}
if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil {
return r.notReadyErrf(pg, "error reconciling metrics resources: %w", err)
return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err)
}
if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil {
return r.notReadyErrf(pg, "error cleaning up dangling resources: %w", err)
return r.notReadyErrf(pg, logger, "error cleaning up dangling resources: %w", err)
}
logger.Info("ProxyGroup resources synced")
@@ -430,6 +427,10 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
defer func() {
if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) {
if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
if strings.Contains(updateErr.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error updating status, retrying: %s", updateErr)
updateErr = nil
}
err = errors.Join(err, updateErr)
}
}
@@ -457,6 +458,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger)
// Set ProxyGroupReady condition.
tsSvcValid, tsSvcSet := tsoperator.KubeAPIServerProxyValid(pg)
status = metav1.ConditionFalse
reason = reasonProxyGroupCreating
switch {
@@ -464,9 +466,15 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
// If we failed earlier, that reason takes precedence.
reason = nrr.reason
message = nrr.message
case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && tsSvcSet && !tsSvcValid:
reason = reasonProxyGroupInvalid
message = "waiting for config in spec.kubeAPIServer to be marked valid"
case len(devices) < desiredReplicas:
case len(devices) > desiredReplicas:
message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(devices)-desiredReplicas)
case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !tsoperator.KubeAPIServerProxyConfigured(pg):
reason = reasonProxyGroupCreating
message = "waiting for proxies to start advertising the kube-apiserver proxy's hostname"
default:
status = metav1.ConditionTrue
reason = reasonProxyGroupReady
@@ -714,7 +722,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pg.Name, i),
Namespace: r.tsNamespace,
Labels: pgSecretLabels(pg.Name, "config"),
Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig),
OwnerReferences: pgOwnerReference(pg),
},
}
@@ -775,13 +783,6 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
}
}
// AdvertiseServices config is set by ingress-pg-reconciler, so make sure we
// don't overwrite it if already set.
existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret)
if err != nil {
return nil, err
}
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
hostname := pgHostname(pg, i)
@@ -795,7 +796,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
}
if !deviceAuthed {
existingCfg := conf.ConfigV1Alpha1{}
if err := json.Unmarshal(existingCfgSecret.Data[kubeAPIServerConfigFile], &existingCfg); err != nil {
if err := json.Unmarshal(existingCfgSecret.Data[kubetypes.KubeAPIServerConfigFile], &existingCfg); err != nil {
return nil, fmt.Errorf("error unmarshalling existing config: %w", err)
}
if existingCfg.AuthKey != nil {
@@ -803,19 +804,42 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
}
}
}
cfg := conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
Hostname: &hostname,
AuthKey: authKey,
State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))),
App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer),
AuthKey: authKey,
KubeAPIServer: &conf.KubeAPIServer{
LogLevel: ptr.To(logger.Level().String()),
// Reloadable fields.
Hostname: &hostname,
APIServerProxy: &conf.APIServerProxyConfig{
Enabled: opt.NewBool(true),
AuthMode: opt.NewBool(isAuthAPIServerProxy(pg)),
// The first replica is elected as the cert issuer, same
// as containerboot does for ingress-pg-reconciler.
IssueCerts: opt.NewBool(i == 0),
},
},
}
// Copy over config that the apiserver-proxy-service-reconciler sets.
if existingCfgSecret != nil {
if k8sProxyCfg, ok := cfgSecret.Data[kubetypes.KubeAPIServerConfigFile]; ok {
k8sCfg := &conf.ConfigV1Alpha1{}
if err := json.Unmarshal(k8sProxyCfg, k8sCfg); err != nil {
return nil, fmt.Errorf("failed to unmarshal kube-apiserver config: %w", err)
}
cfg.AdvertiseServices = k8sCfg.AdvertiseServices
if k8sCfg.APIServerProxy != nil {
cfg.APIServerProxy.ServiceName = k8sCfg.APIServerProxy.ServiceName
}
}
}
if r.loginServer != "" {
cfg.ServerURL = &r.loginServer
}
@@ -832,8 +856,15 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
if err != nil {
return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err)
}
mak.Set(&cfgSecret.Data, kubeAPIServerConfigFile, cfgB)
mak.Set(&cfgSecret.Data, kubetypes.KubeAPIServerConfigFile, cfgB)
} else {
// AdvertiseServices config is set by ingress-pg-reconciler, so make sure we
// don't overwrite it if already set.
existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret)
if err != nil {
return nil, err
}
configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer)
if err != nil {
return nil, fmt.Errorf("error creating tailscaled config: %w", err)
@@ -1024,16 +1055,16 @@ func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error)
return nil, nil
}
conf, err := latestConfigFromSecret(cfgSecret)
cfg, err := latestConfigFromSecret(cfgSecret)
if err != nil {
return nil, err
}
if conf == nil {
if cfg == nil {
return nil, nil
}
return conf.AdvertiseServices, nil
return cfg.AdvertiseServices, nil
}
// getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by
@@ -1045,7 +1076,7 @@ func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error)
func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) {
// List all state Secrets owned by this ProxyGroup.
secrets := &corev1.SecretList{}
if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil {
if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState))); err != nil {
return nil, fmt.Errorf("failed to list state Secrets: %w", err)
}
for _, secret := range secrets.Items {
@@ -1140,15 +1171,21 @@ type nodeMetadata struct {
dnsName string
}
func (r *ProxyGroupReconciler) notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) {
func notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) {
return nil, &notReadyReason{
reason: reason,
message: msg,
}, nil
}
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
err := fmt.Errorf(format, a...)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error())
logger.Info(msg)
return notReady(reasonProxyGroupCreating, msg)
}
r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
return nil, &notReadyReason{
reason: reasonProxyGroupCreationFailed,
cmd/k8s-operator/proxygroup_specs.go
+8 -33
View File
@@ -7,7 +7,6 @@ package main
import (
"fmt"
"path/filepath"
"slices"
"strconv"
"strings"
@@ -16,6 +15,7 @@ import (
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/yaml"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
@@ -341,8 +341,11 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
},
},
{
Name: "TS_K8S_PROXY_CONFIG",
Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile),
Name: "TS_K8S_PROXY_CONFIG",
Value: "kube:" + types.NamespacedName{
Namespace: namespace,
Name: "$(POD_NAME)-config",
}.String(),
},
}
@@ -355,20 +358,6 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
return envs
}(),
VolumeMounts: func() []corev1.VolumeMount {
var mounts []corev1.VolumeMount
// TODO(tomhjp): Read config directly from the Secret instead.
for i := range pgReplicas(pg) {
mounts = append(mounts, corev1.VolumeMount{
Name: fmt.Sprintf("k8s-proxy-config-%d", i),
ReadOnly: true,
MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i),
})
}
return mounts
}(),
Ports: []corev1.ContainerPort{
{
Name: "k8s-proxy",
@@ -378,21 +367,6 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
},
},
},
Volumes: func() []corev1.Volume {
var volumes []corev1.Volume
for i := range pgReplicas(pg) {
volumes = append(volumes, corev1.Volume{
Name: fmt.Sprintf("k8s-proxy-config-%d", i),
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: pgConfigSecretName(pg.Name, i),
},
},
})
}
return volumes
}(),
},
},
},
@@ -426,6 +400,7 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
Resources: []string{"secrets"},
Verbs: []string{
"list",
"watch", // For k8s-proxy.
},
},
{
@@ -508,7 +483,7 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S
ObjectMeta: metav1.ObjectMeta{
Name: pgStateSecretName(pg.Name, i),
Namespace: namespace,
Labels: pgSecretLabels(pg.Name, "state"),
Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState),
OwnerReferences: pgOwnerReference(pg),
},
})
cmd/k8s-operator/proxygroup_test.go
+161 -1
View File
@@ -31,8 +31,11 @@ import (
kube "tailscale.com/k8s-operator"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/tstest"
"tailscale.com/types/opt"
"tailscale.com/types/ptr"
)
@@ -1256,6 +1259,163 @@ func TestProxyGroupTypes(t *testing.T) {
})
}
func TestKubeAPIServerStatusConditionFlow(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-k8s-apiserver",
UID: "test-k8s-apiserver-uid",
Generation: 1,
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeKubernetesAPIServer,
Replicas: ptr.To[int32](1),
KubeAPIServer: &tsapi.KubeAPIServerConfig{
Mode: ptr.To(tsapi.APIServerProxyModeNoAuth),
},
},
}
stateSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgStateSecretName(pg.Name, 0),
Namespace: tsNamespace,
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pg, stateSecret).
WithStatusSubresource(pg).
Build()
r := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
l: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
}
expectReconciled(t, r, "", pg.Name)
pg.ObjectMeta.Finalizers = append(pg.ObjectMeta.Finalizers, FinalizerName)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.l)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set kube-apiserver valid.
mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.l)
})
expectReconciled(t, r, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.l)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set available.
addNodeIDToStateSecrets(t, fc, pg)
expectReconciled(t, r, "", pg.Name)
pg.Status.Devices = []tsapi.TailnetDevice{
{
Hostname: "hostname-nodeid-0",
TailnetIPs: []string{"1.2.3.4", "::1"},
},
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.l)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set kube-apiserver configured.
mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.l)
})
expectReconciled(t, r, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.l)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.l)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
}
func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) {
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithStatusSubresource(&tsapi.ProxyGroup{}).
Build()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
l: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
}
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-k8s-apiserver",
UID: "test-k8s-apiserver-uid",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeKubernetesAPIServer,
Replicas: ptr.To[int32](1),
KubeAPIServer: &tsapi.KubeAPIServerConfig{
Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), // Avoid needing to pre-create the static ServiceAccount.
},
},
}
if err := fc.Create(t.Context(), pg); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
cfg := conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("secret-authkey"),
State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))),
App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer),
LogLevel: ptr.To("debug"),
Hostname: ptr.To("test-k8s-apiserver-0"),
APIServerProxy: &conf.APIServerProxyConfig{
Enabled: opt.NewBool(true),
AuthMode: opt.NewBool(false),
IssueCerts: opt.NewBool(true),
},
},
}
cfgB, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("failed to marshal config: %v", err)
}
cfgSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pg.Name, 0),
Namespace: tsNamespace,
Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig),
OwnerReferences: pgOwnerReference(pg),
},
Data: map[string][]byte{
kubetypes.KubeAPIServerConfigFile: cfgB,
},
}
expectEqual(t, fc, cfgSecret)
// Now simulate the kube-apiserver services reconciler updating config,
// then check the proxygroup reconciler doesn't overwrite it.
cfg.APIServerProxy.ServiceName = ptr.To(tailcfg.ServiceName("svc:some-svc-name"))
cfg.AdvertiseServices = []string{"svc:should-not-be-overwritten"}
cfgB, err = json.Marshal(cfg)
if err != nil {
t.Fatalf("failed to marshal config: %v", err)
}
mustUpdate(t, fc, tsNamespace, cfgSecret.Name, func(s *corev1.Secret) {
s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB
})
expectReconciled(t, reconciler, "", pg.Name)
cfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cfgB
expectEqual(t, fc, cfgSecret)
}
func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
@@ -1660,7 +1820,7 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG
if _, err := createOrUpdate(t.Context(), fc, "tailscale", pod, nil); err != nil {
t.Fatalf("failed to create or update Pod %s: %v", pod.Name, err)
}
mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) {
mustUpdate(t, fc, tsNamespace, pgStateSecretName(pg.Name, i), func(s *corev1.Secret) {
s.Data = map[string][]byte{
currentProfileKey: []byte(key),
key: bytes,
cmd/k8s-operator/svc-for-pg.go
+9 -9
View File
@@ -41,7 +41,7 @@ import (
)
const (
finalizerName = "tailscale.com/service-pg-finalizer"
svcPGFinalizerName = "tailscale.com/service-pg-finalizer"
reasonIngressSvcInvalid = "IngressSvcInvalid"
reasonIngressSvcValid = "IngressSvcValid"
@@ -174,13 +174,13 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
return false, nil
}
if !slices.Contains(svc.Finalizers, finalizerName) {
if !slices.Contains(svc.Finalizers, svcPGFinalizerName) {
// This log line is printed exactly once during initial provisioning,
// because once the finalizer is in place this block gets skipped. So,
// this is a nice place to tell the operator that the high level,
// multi-reconcile operation is underway.
logger.Infof("exposing Service over tailscale")
svc.Finalizers = append(svc.Finalizers, finalizerName)
svc.Finalizers = append(svc.Finalizers, svcPGFinalizerName)
if err := r.Update(ctx, svc); err != nil {
return false, fmt.Errorf("failed to add finalizer: %w", err)
}
@@ -378,7 +378,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
// corresponding to this Service.
func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, svc *corev1.Service, logger *zap.SugaredLogger) (svcChanged bool, err error) {
logger.Debugf("Ensuring any resources for Service are cleaned up")
ix := slices.Index(svc.Finalizers, finalizerName)
ix := slices.Index(svc.Finalizers, svcPGFinalizerName)
if ix < 0 {
logger.Debugf("no finalizer, nothing to do")
return false, nil
@@ -485,12 +485,12 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
func (r *HAServiceReconciler) deleteFinalizer(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
svc.Finalizers = slices.DeleteFunc(svc.Finalizers, func(f string) bool {
return f == finalizerName
return f == svcPGFinalizerName
})
logger.Debugf("ensure %q finalizer is removed", finalizerName)
logger.Debugf("ensure %q finalizer is removed", svcPGFinalizerName)
if err := r.Update(ctx, svc); err != nil {
return fmt.Errorf("failed to remove finalizer %q: %w", finalizerName, err)
return fmt.Errorf("failed to remove finalizer %q: %w", svcPGFinalizerName, err)
}
r.mu.Lock()
defer r.mu.Unlock()
@@ -653,7 +653,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
// Get all config Secrets for this ProxyGroup.
// Get all Pods
secrets := &corev1.SecretList{}
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil {
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil {
return fmt.Errorf("failed to list config Secrets: %w", err)
}
@@ -720,7 +720,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) {
// Get all state Secrets for this ProxyGroup.
secrets := &corev1.SecretList{}
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil {
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil {
return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err)
}
cmd/k8s-operator/svc-for-pg_test.go
+7 -6
View File
@@ -26,6 +26,7 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/ingressservices"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
@@ -139,7 +140,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName("test-pg", 0),
Namespace: "operator-ns",
Labels: pgSecretLabels("test-pg", "config"),
Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeConfig),
},
Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(`{"Version":""}`),
@@ -298,12 +299,12 @@ func TestServicePGReconciler_MultiCluster(t *testing.T) {
t.Fatalf("getting Tailscale Service: %v", err)
}
if len(tsSvcs) != 1 {
t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs))
if len(tsSvcs.VIPServices) != 1 {
t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs.VIPServices))
}
for name := range tsSvcs {
t.Logf("found Tailscale Service with name %q", name.String())
for _, svc := range tsSvcs.VIPServices {
t.Logf("found Tailscale Service with name %q", svc.Name)
}
}
}
@@ -336,7 +337,7 @@ func TestIgnoreRegularService(t *testing.T) {
tsSvcs, err := ft.ListVIPServices(context.Background())
if err == nil {
if len(tsSvcs) > 0 {
if len(tsSvcs.VIPServices) > 0 {
t.Fatal("unexpected Tailscale Services found")
}
}
cmd/k8s-operator/testutils_test.go
+6 -2
View File
@@ -891,13 +891,17 @@ func (c *fakeTSClient) GetVIPService(ctx context.Context, name tailcfg.ServiceNa
return svc, nil
}
func (c *fakeTSClient) ListVIPServices(ctx context.Context) (map[tailcfg.ServiceName]*tailscale.VIPService, error) {
func (c *fakeTSClient) ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error) {
c.Lock()
defer c.Unlock()
if c.vipServices == nil {
return nil, &tailscale.ErrResponse{Status: http.StatusNotFound}
}
return c.vipServices, nil
result := &tailscale.VIPServiceList{}
for _, svc := range c.vipServices {
result.VIPServices = append(result.VIPServices, *svc)
}
return result, nil
}
func (c *fakeTSClient) CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error {
cmd/k8s-operator/tsclient.go
+2
View File
@@ -56,6 +56,8 @@ type tsClient interface {
DeleteDevice(ctx context.Context, nodeStableID string) error
// GetVIPService is a method for getting a Tailscale Service. VIPService is the original name for Tailscale Service.
GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error)
// ListVIPServices is a method for listing all Tailscale Services. VIPService is the original name for Tailscale Service.
ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error)
// CreateOrUpdateVIPService is a method for creating or updating a Tailscale Service.
CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error
// DeleteVIPService is a method for deleting a Tailscale Service.