Reduce TestKube/Exec flakiness (#61260)

* Reduce TestKube/Exec flakiness

The test was not explicitly providing the target Kubernetes
cluster. As a result of this the test harness would try to
infer the Kubernetes cluster by examining the inventory. This
could potentially happen fast enough that no Kubernetes clusters
exist in the inventory yet. As a result the empty field in the
identity could lead to the `Kubernetes cluster "" not found`
error reported in https://github.com/gravitational/teleport/issues/59462.

Closes https://github.com/gravitational/teleport/issues/59462.

* Extract alpine-webserver:v1 building from GHA

The logic to construct an image used by Kubernetes integration
tests has been moved to standalone make targets in the fixtures
folder to facilitate easier local testing. The GHA workflows
have been updated to call the new targets directly.

.github/workflows/kube-integration-tests-non-root.yaml

@@ -92,33 +92,13 @@ jobs:
 
       - name: Build Alpine image with webserver
         run: |
-
-          export SHORT_VERSION=${ALPINE_VERSION%.*}
-
-          # download the alpine image
-          # store the files in the fixtures/alpine directory
+          # create the alpine image in the fixtures/alpine directory
           # to avoid passing all the repository files to the docker build context.
           cd ./fixtures/alpine
-
-          # download alpine minirootfs and signature
-          curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$SHORT_VERSION/releases/x86_64/alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz
-          curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$SHORT_VERSION/releases/x86_64/alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz.asc
-          curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$SHORT_VERSION/releases/x86_64/alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz.sha256
-
-          # verify the checksum
-          sha256sum -c alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz.sha256
-
-          # verify the signature
-          gpg --import  ./alpine-ncopa.at.alpinelinux.org.asc
-          gpg --verify ./alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz.asc ./alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz
-
-          # build the webserver
-          CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./webserver ./webserver.go
-
-          docker build -t alpine-webserver:v1 --build-arg=ALPINE_VERSION=$ALPINE_VERSION -f ./Dockerfile .
+          make SHORT_VERSION=${ALPINE_VERSION%.*} ALPINE_VERSION=${ALPINE_VERSION} build-image
 
           # load the image into the kind cluster
-          kind load docker-image alpine-webserver:v1
+          make load-image
 
           cd - 
 

fixtures/alpine/Makefile

@@ -0,0 +1,33 @@
+
+# build-image downloads and validates alpine artifacts, and then builds a
+# docker image to be consumed by pods in Kubernetes integration tests.
+.PHONY: build-image
+build-image: ALPINE_VERSION ?= 3.20.3
+build-image: SHORT_VERSION ?= 3.20
+build-image:
+	# download the alpine artifacts
+	curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$(SHORT_VERSION)/releases/x86_64/alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz
+	curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$(SHORT_VERSION)/releases/x86_64/alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz.asc
+	curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$(SHORT_VERSION)/releases/x86_64/alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz.sha256
+
+	# verify the checksum
+	sha256sum -c alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz.sha256
+
+	# verify the signature
+	gpg --import  ./alpine-ncopa.at.alpinelinux.org.asc
+	gpg --verify ./alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz.asc ./alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz
+
+	# build the web server
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./webserver ./webserver.go
+	docker build -t alpine-webserver:v1 --build-arg=ALPINE_VERSION=$(ALPINE_VERSION) -f ./Dockerfile .
+
+	rm webserver
+	rm alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz
+	rm alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz.asc
+	rm alpine-minirootfs-$(ALPINE_VERSION)-x86_64.tar.gz.sha256
+
+
+# load-image installs the custom build docker image from build-image into a KinD cluster.
+.PHONY: load-image
+load-image:
+	kind load docker-image alpine-webserver:v1

fixtures/alpine/README.md

@@ -1,50 +1,24 @@
-# `alpine-webserver:v1` Build Process
+# `alpine-webserver:v1`
 
-## Source
+## Why
 
-The `alpine-webserver:v1` image is based on `alpine` `minirootfs`, but instead of relying on Docker Hub's official Alpine image, we source the original files directly from Alpine's CDN. This approach mitigates issues with Docker Hub and GitHub Action network failures, which have been a common cause of integration test failures.
+We build a custom image for Kubernetes integration tests so that we don't have to 
+rely on external dependencies in CI. This approach mitigates issues with Docker Hub 
+and GitHub Actions network failures which have been a common cause of integration test failures
+in the past.
 
-The build process is specified in the `.github/workflows/kube-integration-tests-non-root.yaml` file.
+## How
 
-## Download
+The `make build-image` performs the following steps to produce the image.
 
-To download the new `alpine-minirootfs` image, follow the instructions:
+We download `alpine` `minirootfs` assets directly from the alpine CDN, compile the webserver,
+and build a minimal docker image.
 
-```bash
-$ export ALPINE_VERSION=3.20.3
-$ export SHORT_VERSION=${ALPINE_VERSION%.*}
+The build process validates both the SHA-256 checksum and the GPG signature. The signature verification 
+uses `alpine-ncopa.at.alpinelinux.org.asc`, which is the official public key used by Alpine Linux to 
+sign its assets. This public key is available on the [Alpine Linux Downloads page](https://www.alpinelinux.org/downloads/).
 
-# download alpine minirootfs and signature
-$ curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$SHORT_VERSION/releases/x86_64/alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz
-$ curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$SHORT_VERSION/releases/x86_64/alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz.asc
-$ curl -fSsLO https://dl-cdn.alpinelinux.org/alpine/v$SHORT_VERSION/releases/x86_64/alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz.sha256
-          
-```
+The docker image produced is tagged with `:v1` instead of `:latest` to prevent Kubernetes from 
+pulling the image from Docker Hub in an attempt to ensure the most recent image is used.
 
-## Source Validation
-
-The build process in `.github/workflows/kube-integration-tests-non-root.yaml` validates both the SHA-256 checksum and the GPG signature. The signature verification uses `alpine-ncopa.at.alpinelinux.org.asc`, which is the official public key used by Alpine Linux to sign its assets. This public key is available on the [Alpine Linux Downloads page](https://www.alpinelinux.org/downloads/).
-
-## Image Build Process
-
-The image is constructed from a scratch filesystem and incorporates only the necessary components to run the web server. Here’s the basic Dockerfile configuration:
-
-```Dockerfile
-FROM scratch
-
-ARG ALPINE_VERSION
-
-ADD alpine-minirootfs-$ALPINE_VERSION-x86_64.tar.gz /
-
-COPY webserver /webserver
-
-CMD [ "/webserver" ]
-```
-
-This minimalist configuration ensures the image remains lightweight, secure, and tailored to only the required functionalities.
-
-## Image distribution
-
-After sucessfull build, the image is loaded into our `kind` cluster with the tag `alpine-webserver:v1`.
-
-Note: `:latest` can't be used otherwise Kubernetes will try loading the image from dockerhub and fail.
+After a successful build, the image can be loaded into a `kind` cluster via `make load-image`.

integration/kube/fixtures.go

@@ -56,6 +56,14 @@ type ProxyConfig struct {
 // ProxyClient returns kubernetes client using local teleport proxy
 func ProxyClient(cfg ProxyConfig) (*kubernetes.Clientset, *rest.Config, error) {
 	ctx := context.Background()
+
+	// Require that the target Kubernetes cluster is specified rather than
+	// guessing it to avoid cache propagation delays which may falsely indicate that no
+	// clusters exist.
+	if cfg.KubeCluster == "" {
+		return nil, nil, trace.BadParameter("KubeCluster must be provided")
+	}
+
 	authServer := cfg.T.Process.GetAuthServer()
 	clusterName, err := authServer.GetClusterName(ctx)
 	if err != nil {
@@ -97,19 +105,13 @@ func ProxyClient(cfg ProxyConfig) (*kubernetes.Clientset, *rest.Config, error) {
 		return nil, nil, trace.Wrap(err)
 	}
 
-	kubeServers, _ := authServer.GetKubernetesServers(ctx)
-	kubeCluster := cfg.KubeCluster
-	if cfg.KubeCluster == "" && len(kubeServers) > 0 {
-		kubeCluster = kubeServers[0].GetCluster().GetName()
-	}
-
 	id := tlsca.Identity{
 		Username:          cfg.Username,
 		Groups:            user.GetRoles(),
 		KubernetesUsers:   cfg.KubeUsers,
 		KubernetesGroups:  cfg.KubeGroups,
 		RouteToCluster:    cfg.RouteToCluster,
-		KubernetesCluster: kubeCluster,
+		KubernetesCluster: cfg.KubeCluster,
 		PinnedIP:          cfg.PinnedIP,
 	}
 	subj, err := id.Subject()

integration/kube_integration_test.go

@@ -147,18 +147,18 @@ func newKubeSuite(t *testing.T) *KubeSuite {
 
 	// Create test namespace and pod to run k8s commands against.
 	ns := newNamespace(testNamespace)
-	_, err = suite.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
+	_, err = suite.CoreV1().Namespaces().Create(t.Context(), ns, metav1.CreateOptions{})
 	if err != nil {
 		require.True(t, kubeerrors.IsAlreadyExists(err), "Failed to create namespace: %v:", err)
 	}
 	p := newPod(testNamespace, testPod)
-	_, err = suite.CoreV1().Pods(testNamespace).Create(context.Background(), p, metav1.CreateOptions{})
+	_, err = suite.CoreV1().Pods(testNamespace).Create(t.Context(), p, metav1.CreateOptions{})
 	if err != nil {
 		require.True(t, kubeerrors.IsAlreadyExists(err), "Failed to create test pod: %v", err)
 	}
 	// Wait for pod to be running.
 	require.Eventually(t, func() bool {
-		rsp, err := suite.CoreV1().Pods(testNamespace).Get(context.Background(), testPod, metav1.GetOptions{})
+		rsp, err := suite.CoreV1().Pods(testNamespace).Get(t.Context(), testPod, metav1.GetOptions{})
 		if err != nil {
 			return false
 		}
@@ -251,24 +251,25 @@ func testExec(t *testing.T, suite *KubeSuite, pinnedIP string, clientError strin
 		PinnedIP:      pinnedIP,
 		KubeUsers:     kubeUsers,
 		KubeGroups:    kubeGroups,
+		KubeCluster:   teleport.Secrets.SiteName,
 		Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}},
 	})
 
 	require.NoError(t, err)
 
 	// try get request to fetch a pod
-	ctx := context.Background()
-	_, err = impersonatingProxyClient.CoreV1().Pods(testNamespace).Get(ctx, testPod, metav1.GetOptions{})
+	_, err = impersonatingProxyClient.CoreV1().Pods(testNamespace).Get(t.Context(), testPod, metav1.GetOptions{})
 	require.Error(t, err)
 
 	// scoped client requests will be allowed, as long as the impersonation headers
 	// are referencing users and groups allowed by existing roles
 	scopedProxyClient, scopedProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   username,
-		PinnedIP:   pinnedIP,
-		KubeUsers:  kubeUsers,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    username,
+		PinnedIP:    pinnedIP,
+		KubeUsers:   kubeUsers,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 		Impersonation: &rest.ImpersonationConfig{
 			UserName: role.GetKubeUsers(types.Allow)[0],
 			Groups:   role.GetKubeGroups(types.Allow),
@@ -276,7 +277,7 @@ func testExec(t *testing.T, suite *KubeSuite, pinnedIP string, clientError strin
 	})
 	require.NoError(t, err)
 
-	_, err = scopedProxyClient.CoreV1().Pods(testNamespace).Get(ctx, testPod, metav1.GetOptions{})
+	_, err = scopedProxyClient.CoreV1().Pods(testNamespace).Get(t.Context(), testPod, metav1.GetOptions{})
 	if clientError != "" {
 		require.ErrorContains(t, err, clientError)
 		return
@@ -284,20 +285,21 @@ func testExec(t *testing.T, suite *KubeSuite, pinnedIP string, clientError strin
 
 	// set up kube configuration using proxy
 	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   username,
-		KubeUsers:  kubeUsers,
-		PinnedIP:   pinnedIP,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    username,
+		KubeUsers:   kubeUsers,
+		PinnedIP:    pinnedIP,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
 	// try get request to fetch available pods
-	pod, err := proxyClient.CoreV1().Pods(testNamespace).Get(ctx, testPod, metav1.GetOptions{})
+	pod, err := proxyClient.CoreV1().Pods(testNamespace).Get(t.Context(), testPod, metav1.GetOptions{})
 	require.NoError(t, err)
 
 	out := &bytes.Buffer{}
-	err = kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -315,7 +317,7 @@ func testExec(t *testing.T, suite *KubeSuite, pinnedIP string, clientError strin
 	term.Type("\aecho hi\n\r\aexit\n\r\a")
 
 	out = &bytes.Buffer{}
-	err = kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -346,7 +348,7 @@ loop:
 	}
 
 	// read back the entire session and verify that it matches the stated output
-	capturedStream, _ := streamSession(ctx, t, teleport.Process.GetAuthServer(), sessionID)
+	capturedStream, _ := streamSession(t.Context(), t, teleport.Process.GetAuthServer(), sessionID)
 	require.Equal(t, sessionStream, capturedStream)
 
 	// impersonating kube exec should be denied
@@ -354,7 +356,7 @@ loop:
 	term = NewTerminal(250)
 	term.Type("\aecho hi\n\r\aexit\n\r\a")
 	out = &bytes.Buffer{}
-	err = kubeExec(impersonatingProxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), impersonatingProxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -371,7 +373,7 @@ loop:
 	term = NewTerminal(250)
 	term.Type("\aecho hi\n\r\aexit\n\r\a")
 	out = &bytes.Buffer{}
-	err = kubeExec(scopedProxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), scopedProxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -457,16 +459,16 @@ func testKubeDeny(t *testing.T, suite *KubeSuite) {
 
 	// set up kube configuration using proxy
 	proxyClient, _, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   username,
-		KubeUsers:  kubeUsers,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    username,
+		KubeUsers:   kubeUsers,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
 	// try get request to fetch available pods
-	ctx := context.Background()
-	_, err = proxyClient.CoreV1().Pods(testNamespace).Get(ctx, testPod, metav1.GetOptions{})
+	_, err = proxyClient.CoreV1().Pods(testNamespace).Get(t.Context(), testPod, metav1.GetOptions{})
 	require.Error(t, err)
 }
 
@@ -511,9 +513,10 @@ func testKubePortForward(t *testing.T, suite *KubeSuite) {
 
 	// set up kube configuration using proxy
 	_, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   username,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    username,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
@@ -522,6 +525,7 @@ func testKubePortForward(t *testing.T, suite *KubeSuite) {
 		T:             teleport,
 		Username:      username,
 		KubeGroups:    kubeGroups,
+		KubeCluster:   teleport.Secrets.SiteName,
 		Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}},
 	})
 	require.NoError(t, err)
@@ -637,9 +641,10 @@ func testKubePortForwardPodDisconnect(t *testing.T, suite *KubeSuite) {
 
 	// set up kube configuration using proxy
 	_, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   username,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    username,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
@@ -728,12 +733,12 @@ func testKubePortForwardPodDisconnect(t *testing.T, suite *KubeSuite) {
 				require.NoError(t, resp.Body.Close())
 
 				// Delete the pod.
-				err = suite.CoreV1().Pods(testNamespace).Delete(context.Background(), testPod, metav1.DeleteOptions{})
+				err = suite.CoreV1().Pods(testNamespace).Delete(t.Context(), testPod, metav1.DeleteOptions{})
 				require.NoError(t, err)
 
 				// Wait for pod deletion.
 				require.Eventually(t, func() bool {
-					if _, err := suite.CoreV1().Pods(testNamespace).Get(context.Background(), testPod, metav1.GetOptions{}); err != nil {
+					if _, err := suite.CoreV1().Pods(testNamespace).Get(t.Context(), testPod, metav1.GetOptions{}); err != nil {
 						return kubeerrors.IsNotFound(err)
 					}
 					return false
@@ -761,7 +766,7 @@ func testKubePortForwardPodDisconnect(t *testing.T, suite *KubeSuite) {
 // TestKubeTrustedClustersClientCert tests scenario with trusted clusters
 // using metadata encoded in the certificate
 func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) {
-	ctx := context.Background()
+	ctx := t.Context()
 	clusterMain := "cluster-main"
 	mainConf := suite.teleKubeConfig(Host)
 	// Main cluster doesn't need a kubeconfig to forward requests to auxiliary
@@ -918,7 +923,7 @@ func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) {
 	require.NoError(t, err)
 
 	out := &bytes.Buffer{}
-	err = kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -936,7 +941,7 @@ func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) {
 	term.Type("\aecho hi\n\r\aexit\n\r\a")
 
 	out = &bytes.Buffer{}
-	err = kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -975,7 +980,7 @@ loop:
 	term = NewTerminal(250)
 	term.Type("\aecho hi\n\r\aexit\n\r\a")
 	out = &bytes.Buffer{}
-	err = kubeExec(impersonatingProxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), impersonatingProxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -1036,7 +1041,7 @@ loop:
 // using SNI-forwarding
 // DELETE IN(4.3.0)
 func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) {
-	ctx := context.Background()
+	ctx := t.Context()
 
 	clusterMain := "cluster-main"
 	mainConf := suite.teleKubeConfig(Host)
@@ -1170,6 +1175,7 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) {
 		T:             main,
 		Username:      username,
 		KubeGroups:    mainKubeGroups,
+		KubeCluster:   main.Secrets.SiteName,
 		Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}},
 	})
 	require.NoError(t, err)
@@ -1180,9 +1186,10 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) {
 
 	// set up kube configuration using main proxy
 	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          main,
-		Username:   username,
-		KubeGroups: mainKubeGroups,
+		T:           main,
+		Username:    username,
+		KubeGroups:  mainKubeGroups,
+		KubeCluster: main.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
@@ -1191,7 +1198,7 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) {
 	require.NoError(t, err)
 
 	out := &bytes.Buffer{}
-	err = kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -1209,7 +1216,7 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) {
 	term.Type("\aecho hi\n\r\aexit\n\r\a")
 
 	out = &bytes.Buffer{}
-	err = kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -1248,7 +1255,7 @@ loop:
 	term = NewTerminal(250)
 	term.Type("\aecho hi\n\r\aexit\n\r\a")
 	out = &bytes.Buffer{}
-	err = kubeExec(impersonatingProxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), impersonatingProxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -1381,19 +1388,20 @@ func runKubeDisconnectTest(t *testing.T, suite *KubeSuite, tc disconnectTestCase
 
 	// set up kube configuration using proxy
 	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   username,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    username,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
 	// try get request to fetch available pods
-	ctx := context.Background()
+	ctx := t.Context()
 	pod, err := proxyClient.CoreV1().Pods(testNamespace).Get(ctx, testPod, metav1.GetOptions{})
 	require.NoError(t, err)
 
 	out := &bytes.Buffer{}
-	err = kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 		podName:      pod.Name,
 		podNamespace: pod.Namespace,
 		container:    pod.Spec.Containers[0].Name,
@@ -1410,7 +1418,7 @@ func runKubeDisconnectTest(t *testing.T, suite *KubeSuite, tc disconnectTestCase
 	sessionCtx, sessionCancel := context.WithCancel(ctx)
 	go func() {
 		defer sessionCancel()
-		err := kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+		err := kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 			podName:      pod.Name,
 			podNamespace: pod.Namespace,
 			container:    pod.Spec.Containers[0].Name,
@@ -1480,13 +1488,14 @@ func testKubeTransportProtocol(t *testing.T, suite *KubeSuite) {
 
 	// set up kube configuration using proxy
 	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   username,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    username,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
-	ctx := context.Background()
+	ctx := t.Context()
 	pod, err := proxyClient.CoreV1().Pods(testNamespace).Get(ctx, testPod, metav1.GetOptions{})
 	require.NoError(t, err)
 
@@ -1529,12 +1538,12 @@ func testKubeTransportProtocol(t *testing.T, suite *KubeSuite) {
 		command:      []string{"ls"},
 	}
 
-	err = kubeExec(proxyClientConfig, execInContainer, command)
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, command)
 	require.NoError(t, err)
 
 	// stream fails with an h2 transport
 	proxyClientConfig.TLSClientConfig.NextProtos = []string{"h2"}
-	err = kubeExec(proxyClientConfig, execInContainer, command)
+	err = kubeExec(t.Context(), proxyClientConfig, execInContainer, command)
 	require.Error(t, err)
 }
 
@@ -1626,15 +1635,16 @@ func testKubeEphemeralContainers(t *testing.T, suite *KubeSuite) {
 
 	// set up kube configuration using proxy
 	proxyClient, kubeConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   username,
-		KubeUsers:  kubeUsers,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    username,
+		KubeUsers:   kubeUsers,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
 	// try get request to fetch available pods
-	ctx := context.Background()
+	ctx := t.Context()
 	podsClient := proxyClient.CoreV1().Pods(testNamespace)
 	pod, err := podsClient.Get(ctx, testPod, metav1.GetOptions{})
 	require.NoError(t, err)
@@ -1671,7 +1681,7 @@ func testKubeEphemeralContainers(t *testing.T, suite *KubeSuite) {
 			return trace.Wrap(err)
 		}
 
-		err = kubeExec(kubeConfig, attachToContainer, kubeExecArgs{
+		err = kubeExec(t.Context(), kubeConfig, attachToContainer, kubeExecArgs{
 			podName:      pod.Name,
 			podNamespace: testNamespace,
 			container:    contName,
@@ -1695,7 +1705,7 @@ func testKubeEphemeralContainers(t *testing.T, suite *KubeSuite) {
 		// We need to wait for the session to be created here. We can't use the
 		// session manager's WaitUntilExists method because it doesn't work for
 		// kubernetes sessions.
-		sessions, err := teleport.Process.GetAuthServer().GetActiveSessionTrackers(context.Background())
+		sessions, err := teleport.Process.GetAuthServer().GetActiveSessionTrackers(ctx)
 		require.NoError(t, err)
 		require.NotEmpty(t, sessions, "no active sessions found")
 		session = sessions[0]
@@ -1705,10 +1715,11 @@ func testKubeEphemeralContainers(t *testing.T, suite *KubeSuite) {
 	group.Go(func() error {
 		// verify that the ephemeral container hasn't actually been created yet
 		proxyClient, _, err := kube.ProxyClient(kube.ProxyConfig{
-			T:          teleport,
-			Username:   moderatorUser,
-			KubeUsers:  kubeUsers,
-			KubeGroups: kubeGroups,
+			T:           teleport,
+			Username:    moderatorUser,
+			KubeUsers:   kubeUsers,
+			KubeGroups:  kubeGroups,
+			KubeCluster: teleport.Secrets.SiteName,
 		})
 		require.NoError(t, err)
 
@@ -1733,10 +1744,11 @@ func testKubeEphemeralContainers(t *testing.T, suite *KubeSuite) {
 		stream, err := kubeJoin(
 			ctx,
 			kube.ProxyConfig{
-				T:          teleport,
-				Username:   moderatorUser,
-				KubeUsers:  kubeUsers,
-				KubeGroups: kubeGroups,
+				T:           teleport,
+				Username:    moderatorUser,
+				KubeUsers:   kubeUsers,
+				KubeGroups:  kubeGroups,
+				KubeCluster: teleport.Secrets.SiteName,
 			},
 			tc,
 			session,
@@ -1953,7 +1965,7 @@ func testKubeExecWeb(t *testing.T, suite *KubeSuite) {
 
 		ws := openWebsocketAndReadSession(t, endpoint, req)
 
-		wsStream := terminal.NewWStream(context.Background(), ws, logtest.NewLogger(), nil)
+		wsStream := terminal.NewWStream(t.Context(), ws, logtest.NewLogger(), nil)
 
 		// Check for the expected string in the output.
 		findTextInReader(t, wsStream, testNamespace, time.Second*2)
@@ -1974,7 +1986,7 @@ func testKubeExecWeb(t *testing.T, suite *KubeSuite) {
 
 		ws := openWebsocketAndReadSession(t, endpoint, req)
 
-		wsStream := terminal.NewWStream(context.Background(), ws, logtest.NewLogger(), nil)
+		wsStream := terminal.NewWStream(t.Context(), ws, logtest.NewLogger(), nil)
 
 		// Read first prompt from the server.
 		readData := make([]byte, 255)
@@ -2224,7 +2236,7 @@ const (
 )
 
 // kubeExec executes command against kubernetes API server
-func kubeExec(kubeConfig *rest.Config, mode execMode, args kubeExecArgs) error {
+func kubeExec(ctx context.Context, kubeConfig *rest.Config, mode execMode, args kubeExecArgs) error {
 	query := make(url.Values)
 	if mode == execInContainer {
 		for _, arg := range args.command {
@@ -2270,7 +2282,7 @@ func kubeExec(kubeConfig *rest.Config, mode execMode, args kubeExecArgs) error {
 		Stderr: args.stderr,
 		Tty:    args.tty,
 	}
-	return executor.StreamWithContext(context.Background(), opts)
+	return executor.StreamWithContext(ctx, opts)
 }
 
 func kubeJoin(ctx context.Context, kubeConfig kube.ProxyConfig, tc *client.TeleportClient, meta types.SessionTracker, mode types.SessionParticipantMode) (*client.KubeSession, error) {
@@ -2381,15 +2393,16 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) {
 	require.NoError(t, err)
 	defer teleport.StopAll()
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
 
 	// set up kube configuration using proxy
 	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   hostUsername,
-		KubeUsers:  kubeUsers,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    hostUsername,
+		KubeUsers:   kubeUsers,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
@@ -2406,7 +2419,7 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) {
 
 	// Start the main session.
 	group.Go(func() error {
-		err := kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+		err := kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 			podName:      pod.Name,
 			podNamespace: pod.Namespace,
 			container:    pod.Spec.Containers[0].Name,
@@ -2461,10 +2474,11 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) {
 		stream, err := kubeJoin(
 			ctx,
 			kube.ProxyConfig{
-				T:          teleport,
-				Username:   peerUsername,
-				KubeUsers:  kubeUsers,
-				KubeGroups: kubeGroups,
+				T:           teleport,
+				Username:    peerUsername,
+				KubeUsers:   kubeUsers,
+				KubeGroups:  kubeGroups,
+				KubeCluster: teleport.Secrets.SiteName,
 			},
 			tc,
 			session,
@@ -2609,15 +2623,16 @@ func testKubeJoinWeb(t *testing.T, suite *KubeSuite) {
 	require.NoError(t, err)
 	defer teleport.StopAll()
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
 
 	// set up kube configuration using proxy
 	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
-		T:          teleport,
-		Username:   hostUsername,
-		KubeUsers:  kubeUsers,
-		KubeGroups: kubeGroups,
+		T:           teleport,
+		Username:    hostUsername,
+		KubeUsers:   kubeUsers,
+		KubeGroups:  kubeGroups,
+		KubeCluster: teleport.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
@@ -2633,7 +2648,7 @@ func testKubeJoinWeb(t *testing.T, suite *KubeSuite) {
 
 	// Start the main session.
 	group.Go(func() error {
-		err := kubeExec(proxyClientConfig, execInContainer, kubeExecArgs{
+		err := kubeExec(t.Context(), proxyClientConfig, execInContainer, kubeExecArgs{
 			podName:      pod.Name,
 			podNamespace: pod.Namespace,
 			container:    pod.Spec.Containers[0].Name,
@@ -2828,6 +2843,7 @@ func kubeJoinObserverWithSNISet(ctx context.Context, t *testing.T, tc *client.Te
 			Username:            tc.Username,
 			KubeUsers:           kubeUsers,
 			KubeGroups:          kubeGroups,
+			KubeCluster:         teleport.Secrets.SiteName,
 			CustomTLSServerName: constants.KubeTeleportProxyALPNPrefix + Host,
 		},
 		tc,
@@ -2934,11 +2950,12 @@ func testExecNoAuth(t *testing.T, suite *KubeSuite) {
 			KubeUsers:     kubeUsers,
 			KubeGroups:    kubeGroups,
 			TargetAddress: *kubeAddr,
+			KubeCluster:   teleport.Secrets.SiteName,
 		})
 		if err != nil {
 			return false
 		}
-		ctx := context.Background()
+		ctx := t.Context()
 		// try get request to fetch available pods
 		_, err = proxyClient.CoreV1().Pods(testNamespace).Get(ctx, testPod, metav1.GetOptions{})
 		return err == nil
@@ -2949,6 +2966,7 @@ func testExecNoAuth(t *testing.T, suite *KubeSuite) {
 		Username:      adminUsername,
 		KubeUsers:     kubeUsers,
 		KubeGroups:    kubeGroups,
+		KubeCluster:   teleport.Secrets.SiteName,
 		TargetAddress: *kubeAddr,
 	})
 	require.NoError(t, err)
@@ -2958,6 +2976,7 @@ func testExecNoAuth(t *testing.T, suite *KubeSuite) {
 		Username:      userUsername,
 		KubeUsers:     kubeUsers,
 		KubeGroups:    kubeGroups,
+		KubeCluster:   teleport.Secrets.SiteName,
 		TargetAddress: *kubeAddr,
 	})
 	require.NoError(t, err)
@@ -2993,7 +3012,7 @@ func testExecNoAuth(t *testing.T, suite *KubeSuite) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-			ctx := context.Background()
+			ctx := t.Context()
 			// try get request to fetch available pods
 			pod, err := tt.proxyClient.CoreV1().Pods(testNamespace).Get(ctx, testPod, metav1.GetOptions{})
 			require.NoError(t, err)
@@ -3003,7 +3022,7 @@ func testExecNoAuth(t *testing.T, suite *KubeSuite) {
 			term := NewTerminal(250)
 			// lets type "echo hi" followed by "enter" and then "exit" + "enter":
 			term.Type("\aecho hi\n\r\aexit\n\r\a")
-			err = kubeExec(tt.clientConfig, execInContainer, kubeExecArgs{
+			err = kubeExec(t.Context(), tt.clientConfig, execInContainer, kubeExecArgs{
 				podName:      pod.Name,
 				podNamespace: pod.Namespace,
 				container:    pod.Spec.Containers[0].Name,