webnet/tailscale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/{k8s-operator,k8s-proxy}: add kube-apiserver ProxyGroup type (#16266)

Browse Source 
Adds a new k8s-proxy command to convert operator's in-process proxy to
a separately deployable type of ProxyGroup: kube-apiserver. k8s-proxy
reads in a new config file written by the operator, modelled on tailscaled's
conffile but with some modifications to ensure multiple versions of the
config can co-exist within a file. This should make it much easier to
support reading that config file from a Kube Secret with a stable file name.

To avoid needing to give the operator ClusterRole{,Binding} permissions,
the helm chart now optionally deploys a new static ServiceAccount for
the API Server proxy to use if in auth mode.

Proxies deployed by kube-apiserver ProxyGroups currently work the same as
the operator's in-process proxy. They do not yet leverage Tailscale Services
for presenting a single HA DNS name.

Updates #13358

Change-Id: Ib6ead69b2173c5e1929f3c13fb48a9a5362195d8
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
This commit is contained in:
Tom Proctor
2025-07-09 09:21:56 +01:00
committed by GitHub
parent 90bf0a97b3
commit 4dfed6b146
31 changed files with 1788 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/k8s-operator/svc-for-pg.go
+8 -52
View File
@@ -60,7 +60,6 @@ type HAServiceReconciler struct {
recorder record.EventRecorder
logger *zap.SugaredLogger
tsClient tsClient
tsnetServer tsnetServer
tsNamespace string
lc localClient
defaultTags []string
@@ -221,7 +220,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
// This checks and ensures that Tailscale Service's owner references are updated
// for this Service and errors if that is not possible (i.e. because it
// appears that the Tailscale Service has been created by a non-operator actor).
updatedAnnotations, err := r.ownerAnnotations(existingTSSvc)
updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc)
if err != nil {
instr := fmt.Sprintf("To proceed, you can either manually delete the existing Tailscale Service or choose a different hostname with the '%s' annotaion", AnnotationHostname)
msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr)
@@ -395,7 +394,7 @@ func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string,
serviceName := tailcfg.ServiceName("svc:" + hostname)
// 1. Clean up the Tailscale Service.
svcChanged, err = r.cleanupTailscaleService(ctx, serviceName, logger)
svcChanged, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger)
if err != nil {
return false, fmt.Errorf("error deleting Tailscale Service: %w", err)
}
@@ -456,7 +455,7 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
}
svcsChanged, err = r.cleanupTailscaleService(ctx, tailcfg.ServiceName(tsSvcName), logger)
svcsChanged, err = cleanupTailscaleService(ctx, r.tsClient, tailcfg.ServiceName(tsSvcName), r.operatorID, logger)
if err != nil {
return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err)
}
@@ -529,8 +528,8 @@ func (r *HAServiceReconciler) tailnetCertDomain(ctx context.Context) (string, er
// If a Tailscale Service is found, but contains other owner references, only removes this operator's owner reference.
// If a Tailscale Service by the given name is not found or does not contain this operator's owner reference, do nothing.
// It returns true if an existing Tailscale Service was updated to remove owner reference, as well as any error that occurred.
func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name tailcfg.ServiceName, logger *zap.SugaredLogger) (updated bool, err error) {
svc, err := r.tsClient.GetVIPService(ctx, name)
func cleanupTailscaleService(ctx context.Context, tsClient tsClient, name tailcfg.ServiceName, operatorID string, logger *zap.SugaredLogger) (updated bool, err error) {
svc, err := tsClient.GetVIPService(ctx, name)
if isErrorFeatureFlagNotEnabled(err) {
msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled)
logger.Warn(msg)
@@ -563,14 +562,14 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name
// cluster before deleting the Ingress. Perhaps the comparison could be
// 'if or.OperatorID == r.operatorID || or.ingressUID == r.ingressUID'.
ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool {
return or.OperatorID == r.operatorID
return or.OperatorID == operatorID
})
if ix == -1 {
return false, nil
}
if len(o.OwnerRefs) == 1 {
logger.Infof("Deleting Tailscale Service %q", name)
return false, r.tsClient.DeleteVIPService(ctx, name)
return false, tsClient.DeleteVIPService(ctx, name)
}
o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1)
logger.Infof("Updating Tailscale Service %q", name)
@@ -579,7 +578,7 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name
return false, fmt.Errorf("error marshalling updated Tailscale Service owner reference: %w", err)
}
svc.Annotations[ownerAnnotation] = string(json)
return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc)
return true, tsClient.CreateOrUpdateVIPService(ctx, svc)
}
func (a *HAServiceReconciler) backendRoutesSetup(ctx context.Context, serviceName, replicaName, pgName string, wantsCfg *ingressservices.Config, logger *zap.SugaredLogger) (bool, error) {
@@ -742,49 +741,6 @@ func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName
return count, nil
}
// ownerAnnotations returns the updated annotations required to ensure this
// instance of the operator is included as an owner. If the Tailscale Service is not
// nil, but does not contain an owner we return an error as this likely means
// that the Tailscale Service was created by something other than a Tailscale
// Kubernetes operator.
func (r *HAServiceReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) {
ref := OwnerRef{
OperatorID: r.operatorID,
}
if svc == nil {
c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}}
json, err := json.Marshal(c)
if err != nil {
return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service owner annotation contents: %w, please report this", err)
}
return map[string]string{
ownerAnnotation: string(json),
}, nil
}
o, err := parseOwnerAnnotation(svc)
if err != nil {
return nil, err
}
if o == nil || len(o.OwnerRefs) == 0 {
return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name)
}
if slices.Contains(o.OwnerRefs, ref) { // up to date
return svc.Annotations, nil
}
o.OwnerRefs = append(o.OwnerRefs, ref)
json, err := json.Marshal(o)
if err != nil {
return nil, fmt.Errorf("error marshalling updated owner references: %w", err)
}
newAnnots := make(map[string]string, len(svc.Annotations)+1)
for k, v := range svc.Annotations {
newAnnots[k] = v
}
newAnnots[ownerAnnotation] = string(json)
return newAnnots, nil
}
// dnsNameForService returns the DNS name for the given Tailscale Service name.
func (r *HAServiceReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
s := svc.WithoutPrefix()