webnet/tailscale
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

k8s-operator,sessionrecording: fixing race condition between resize (#16454)

Browse Source 
messages and cast headers when recording `kubectl attach` sessions

Updates #16490

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
Tom Meadows
2025-07-14 15:17:20 +01:00
committed by GitHub
parent f23e4279c4
commit bcaea4f245
10 changed files with 351 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s-operator/sessionrecording/spdy/conn.go
+57 -41
View File
@@ -4,11 +4,12 @@
//go:build !plan9
// Package spdy contains functionality for parsing SPDY streaming sessions. This
// is used for 'kubectl exec' session recording.
// is used for 'kubectl exec/attach' session recording.
package spdy
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"fmt"
@@ -24,29 +25,50 @@ import (
)
// New wraps the provided network connection and returns a connection whose reads and writes will get triggered as data is received on the hijacked connection.
// The connection must be a hijacked connection for a 'kubectl exec' session using SPDY.
// The connection must be a hijacked connection for a 'kubectl exec/attach' session using SPDY.
// The hijacked connection is used to transmit SPDY streams between Kubernetes client ('kubectl') and the destination container.
// Data read from the underlying network connection is data sent via one of the SPDY streams from the client to the container.
// Data written to the underlying connection is data sent from the container to the client.
// We parse the data and send everything for the stdout/stderr streams to the configured tsrecorder as an asciinema recording with the provided header.
// https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/4006-transition-spdy-to-websockets#background-remotecommand-subprotocol
func New(nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) net.Conn {
return &conn{
Conn: nc,
rec: rec,
ch: ch,
log: log,
hasTerm: hasTerm,
initialTermSizeSet: make(chan struct{}),
func New(ctx context.Context, nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) (net.Conn, error) {
lc := &conn{
Conn: nc,
ctx: ctx,
rec: rec,
ch: ch,
log: log,
hasTerm: hasTerm,
initialCastHeaderSent: make(chan struct{}, 1),
}
// if there is no term, we don't need to wait for a resize message
if !hasTerm {
var err error
lc.writeCastHeaderOnce.Do(func() {
// If this is a session with a terminal attached,
// we must wait for the terminal width and
// height to be parsed from a resize message
// before sending CastHeader, else tsrecorder
// will not be able to play this recording.
err = lc.rec.WriteCastHeader(ch)
close(lc.initialCastHeaderSent)
})
if err != nil {
return nil, fmt.Errorf("error writing CastHeader: %w", err)
}
}
return lc, nil
}
// conn is a wrapper around net.Conn. It reads the bytestream for a 'kubectl
// exec' session streamed using SPDY protocol, sends session recording data to
// exec/attach' session streamed using SPDY protocol, sends session recording data to
// the configured recorder and forwards the raw bytes to the original
// destination.
type conn struct {
net.Conn
ctx context.Context
// rec knows how to send data written to it to a tsrecorder instance.
rec *tsrecorder.Client
@@ -63,7 +85,7 @@ type conn struct {
// CastHeader must be sent before any payload. If the session has a
// terminal attached, the CastHeader must have '.Width' and '.Height'
// fields set for the tsrecorder UI to be able to play the recording.
// For 'kubectl exec' sessions, terminal width and height are sent as a
// For 'kubectl exec/attach' sessions, terminal width and height are sent as a
// resize message on resize stream from the client when the session
// starts as well as at any time the client detects a terminal change.
// We can intercept the resize message on Read calls. As there is no
@@ -79,15 +101,10 @@ type conn struct {
// writeCastHeaderOnce is used to ensure CastHeader gets sent to tsrecorder once.
writeCastHeaderOnce sync.Once
hasTerm bool // whether the session had TTY attached
// initialTermSizeSet channel gets sent a value once, when the Read has
// received a resize message and set the initial terminal size. It must
// be set to a buffered channel to prevent Reads being blocked on the
// first stdout/stderr write reading from the channel.
initialTermSizeSet chan struct{}
// sendInitialTermSizeSetOnce is used to ensure that a value is sent to
// initialTermSizeSet channel only once, when the initial resize message
// is received.
sendinitialTermSizeSetOnce sync.Once
// initialCastHeaderSent is a channel to ensure that the cast
// header is the first thing that is streamed to the session recorder.
// Otherwise the stream will fail.
initialCastHeaderSent chan struct{}
zlibReqReader zlibReader
// writeBuf is used to store data written to the connection that has not
@@ -124,7 +141,7 @@ func (c *conn) Read(b []byte) (int, error) {
}
c.readBuf.Next(len(sf.Raw)) // advance buffer past the parsed frame
if !sf.Ctrl { // data frame
if !sf.Ctrl && c.hasTerm { // data frame
switch sf.StreamID {
case c.resizeStreamID.Load():
@@ -140,10 +157,19 @@ func (c *conn) Read(b []byte) (int, error) {
// subsequent resize message, we need to send asciinema
// resize message.
var isInitialResize bool
c.sendinitialTermSizeSetOnce.Do(func() {
c.writeCastHeaderOnce.Do(func() {
isInitialResize = true
close(c.initialTermSizeSet) // unblock sending of CastHeader
// If this is a session with a terminal attached,
// we must wait for the terminal width and
// height to be parsed from a resize message
// before sending CastHeader, else tsrecorder
// will not be able to play this recording.
err = c.rec.WriteCastHeader(c.ch)
close(c.initialCastHeaderSent)
})
if err != nil {
return 0, fmt.Errorf("error writing CastHeader: %w", err)
}
if !isInitialResize {
if err := c.rec.WriteResize(c.ch.Height, c.ch.Width); err != nil {
return 0, fmt.Errorf("error writing resize message: %w", err)
@@ -190,24 +216,14 @@ func (c *conn) Write(b []byte) (int, error) {
if !sf.Ctrl {
switch sf.StreamID {
case c.stdoutStreamID.Load(), c.stderrStreamID.Load():
var err error
c.writeCastHeaderOnce.Do(func() {
// If this is a session with a terminal attached,
// we must wait for the terminal width and
// height to be parsed from a resize message
// before sending CastHeader, else tsrecorder
// will not be able to play this recording.
if c.hasTerm {
c.log.Debugf("write: waiting for the initial terminal size to be set before proceeding with sending the first payload")
<-c.initialTermSizeSet
// we must wait for confirmation that the initial cast header was sent before proceeding with any more writes
select {
case <-c.ctx.Done():
return 0, c.ctx.Err()
case <-c.initialCastHeaderSent:
if err := c.rec.WriteOutput(sf.Payload); err != nil {
return 0, fmt.Errorf("error sending payload to session recorder: %w", err)
}
err = c.rec.WriteCastHeader(c.ch)
})
if err != nil {
return 0, fmt.Errorf("error writing CastHeader: %w", err)
}
if err := c.rec.WriteOutput(sf.Payload); err != nil {
return 0, fmt.Errorf("error sending payload to session recorder: %w", err)
}
}
}
k8s-operator/sessionrecording/spdy/conn_test.go
+55 -43
View File
@@ -6,10 +6,12 @@
package spdy
import (
"context"
"encoding/json"
"fmt"
"reflect"
"testing"
"time"
"go.uber.org/zap"
"tailscale.com/k8s-operator/sessionrecording/fakes"
@@ -29,15 +31,11 @@ func Test_Writes(t *testing.T) {
}
cl := tstest.NewClock(tstest.ClockOpts{})
tests := []struct {
name string
inputs [][]byte
wantForwarded []byte
wantRecorded []byte
firstWrite bool
width int
height int
sendInitialResize bool
hasTerm bool
name string
inputs [][]byte
wantForwarded []byte
wantRecorded []byte
hasTerm bool
}{
{
name: "single_write_control_frame_with_payload",
@@ -78,24 +76,17 @@ func Test_Writes(t *testing.T) {
wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
},
{
name: "single_first_write_stdout_data_frame_with_payload_sess_has_terminal",
inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...),
width: 10,
height: 20,
hasTerm: true,
firstWrite: true,
sendInitialResize: true,
name: "single_first_write_stdout_data_frame_with_payload_sess_has_terminal",
inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
hasTerm: true,
},
{
name: "single_first_write_stdout_data_frame_with_payload_sess_does_not_have_terminal",
inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...),
width: 10,
height: 20,
firstWrite: true,
wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
},
}
for _, tt := range tests {
@@ -104,29 +95,25 @@ func Test_Writes(t *testing.T) {
sr := &fakes.TestSessionRecorder{}
rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar())
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
c := &conn{
Conn: tc,
log: zl.Sugar(),
rec: rec,
ch: sessionrecording.CastHeader{
Width: tt.width,
Height: tt.height,
},
initialTermSizeSet: make(chan struct{}),
hasTerm: tt.hasTerm,
}
if !tt.firstWrite {
// this test case does not intend to test that cast header gets written once
c.writeCastHeaderOnce.Do(func() {})
}
if tt.sendInitialResize {
close(c.initialTermSizeSet)
ctx: ctx,
Conn: tc,
log: zl.Sugar(),
rec: rec,
ch: sessionrecording.CastHeader{},
initialCastHeaderSent: make(chan struct{}),
hasTerm: tt.hasTerm,
}
c.writeCastHeaderOnce.Do(func() {
close(c.initialCastHeaderSent)
})
c.stdoutStreamID.Store(stdoutStreamID)
c.stderrStreamID.Store(stderrStreamID)
for i, input := range tt.inputs {
c.hasTerm = tt.hasTerm
if _, err := c.Write(input); err != nil {
t.Errorf("[%d] spdyRemoteConnRecorder.Write() unexpected error %v", i, err)
}
@@ -171,11 +158,25 @@ func Test_Reads(t *testing.T) {
wantResizeStreamID uint32
wantWidth int
wantHeight int
wantRecorded []byte
resizeStreamIDBeforeRead uint32
}{
{
name: "resize_data_frame_single_read",
inputs: [][]byte{append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...)},
wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20),
resizeStreamIDBeforeRead: 1,
wantWidth: 10,
wantHeight: 20,
},
{
name: "resize_data_frame_many",
inputs: [][]byte{
append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...),
append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...),
},
wantRecorded: append(fakes.AsciinemaCastHeaderMsg(t, 10, 20), fakes.AsciinemaCastResizeMsg(t, 10, 20)...),
resizeStreamIDBeforeRead: 1,
wantWidth: 10,
wantHeight: 20,
@@ -183,6 +184,7 @@ func Test_Reads(t *testing.T) {
{
name: "resize_data_frame_two_reads",
inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg},
wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20),
resizeStreamIDBeforeRead: 1,
wantWidth: 10,
wantHeight: 20,
@@ -215,11 +217,15 @@ func Test_Reads(t *testing.T) {
tc := &fakes.TestConn{}
sr := &fakes.TestSessionRecorder{}
rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar())
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
c := &conn{
Conn: tc,
log: zl.Sugar(),
rec: rec,
initialTermSizeSet: make(chan struct{}),
ctx: ctx,
Conn: tc,
log: zl.Sugar(),
rec: rec,
initialCastHeaderSent: make(chan struct{}),
hasTerm: true,
}
c.resizeStreamID.Store(tt.resizeStreamIDBeforeRead)
@@ -251,6 +257,12 @@ func Test_Reads(t *testing.T) {
t.Errorf("want height: %v, got %v", tt.wantHeight, c.ch.Height)
}
}
// Assert that the expected bytes have been forwarded to the session recorder.
gotRecorded := sr.Bytes()
if !reflect.DeepEqual(gotRecorded, tt.wantRecorded) {
t.Errorf("expected bytes not recorded, wants\n%v\ngot\n%v", tt.wantRecorded, gotRecorded)
}
})
}
}