e2e runner changes - BPF, remote docker host (SSH), restrict tests to certain browsers (#69447)

* Support ssh:// DOCKER_HOST

* Allow specs to pin themselves to browsers

* Add ssh-node-bpf fixture

* Run a separate node per requested ssh-node fixture

* Run every Teleport config pass even if an earlier one fails

* Fix e2e/runner linting
This commit is contained in:
Ryan Clark
2026-08-11 13:31:39 +00:00
committed by GitHub
parent 863e35d4dd
commit a8a6ad6f81
20 changed files with 1208 additions and 97 deletions
+62 -4
View File
@@ -60,14 +60,60 @@ automatically starts the required infrastructure.
Available fixtures:
| Fixture | Description |
|--------------|--------------------------------------------------------------------------------------|
| `ssh-node` | Start and connect a Teleport SSH node (runs in Docker) |
| `connect` | Build Teleport Connect. Auto-detected from Connect test helpers. |
| Fixture | Description |
|----------------|------------------------------------------------------------------------------------|
| `ssh-node` | Start and connect a Teleport SSH node (runs in Docker) |
| `ssh-node-bpf` | A second node, `docker-node-bpf`, with Enhanced Session Recording enabled. |
| `connect` | Build Teleport Connect. Auto-detected from Connect test helpers. |
Fixtures can also be enabled manually with `--with-<name>` flags (e.g. `--with-ssh-node`, `--with-connect`),
which is useful for modes like `--codegen` or `--browse` where auto-detection does not run.
#### `ssh-node-bpf` needs a Linux docker host
**This fixture almost certainly will not work against Docker on macOS.** A container does not have its own kernel, and
BPF programs are loaded into the docker host's. On macOS that host is a minimal VM kernel, and both OrbStack and
Docker Desktop build theirs without `CONFIG_AUDIT`, so `task_struct` has no `sessionid` field for the command hooks to
read. It is absent from the kernel's BTF, the CO-RE relocation cannot be resolved, and the programs fail to load. No
container setting can add a struct field to a kernel that was not built with it.
Point `DOCKER_HOST` at a Linux machine running docker instead. The kernel needs BTF and `CONFIG_AUDIT`; a stock Ubuntu
or Debian kernel has both. The runner's own probe checks for both, in a privileged container, and you can run the same
check by hand:
```bash
docker run --rm --privileged debian:bookworm-slim \
sh -c 'test -e /sys/kernel/btf/vmlinux && test -e /proc/self/sessionid'
```
`ssh://` works, including a `Host` alias from `~/.ssh/config`, so the browsers can keep running locally:
```bash
DOCKER_HOST=ssh://user@linux-box ./e2e/run.sh --with-ssh-node-bpf e2e/tests/web/authenticated/ssh.spec.ts
```
The node is built for the daemon's architecture rather than pinned to amd64, because the `bpf()` syscall is not proxied
through architecture emulation. On macOS that means a cross-compiler matching the daemon, which the runner picks for the
architecture it detected (`x86_64-unknown-linux-gnu-gcc` for an amd64 host, `aarch64-unknown-linux-gnu-gcc` for arm64).
`CC` overrides it.
Only the nodes a run actually asked for are started. A selection that declares just `ssh-node` gets
`docker-node`, just `ssh-node-bpf` gets `docker-node-bpf`, and a selection containing both gets both
containers, so a spec can compare a node with Enhanced Session Recording against one without.
The runner probes the daemon's kernel before starting the node. Outside CI an unsupported kernel downgrades to a plain
node and exports `E2E_SKIP_ENHANCED_RECORDING=1`, so running the whole suite locally stays green. A spec that needs BPF
opts into that skip:
```ts
import { skipEnhancedRecording } from '@gravitational/e2e/helpers/env';
test.skip(skipEnhancedRecording, "docker daemon's kernel cannot run enhanced session recording");
```
In CI the unsupported kernel is a hard error instead, so the coverage cannot quietly disappear. If the node fails for
some other reason it exits rather than degrading, so check `e2e/docker-node-<browser>.log`.
### Users and Roles
By default, the runner creates a single user with the `access` and `editor` roles. Tests that need custom
@@ -196,6 +242,18 @@ test.describe('custom license', () => {
Note that the e2e runner will only restart Teleport with the different config and not re-initialize it from fresh, so the data directory
will remains the same for all tests which means the cluster's identity like cluster name, CA, bootstrapped users, and roles will carry over unchanged.
### Restricting a spec to certain browsers
Web specs run against every browser by default. A test whose subject is not the browser can opt out:
```ts
test.use({ browsers: ['chromium'] });
```
This is worth doing for a test that also declares a Teleport config, since the runner otherwise repeats that config's
Teleport restart once per browser. The restriction applies to the whole file rather than to an enclosing describe, and
the runner enforces it when choosing which specs each browser runs.
### Session Recordings
The runner automatically seeds session recordings into Teleport's data directory at startup so the Web UI's
+4
View File
@@ -44,3 +44,7 @@ export const teleportConfig = required('E2E_TELEPORT_CONFIG');
export const startUrl = required('START_URL');
export const connectTshBin = required('E2E_CONNECT_TSH_BIN');
export const connectAppDir = required('E2E_CONNECT_APP_DIR');
// Set by the runner when the docker daemon's kernel cannot load the enhanced recording programs.
export const skipEnhancedRecording =
process.env.E2E_SKIP_ENHANCED_RECORDING === '1';
+1
View File
@@ -22,6 +22,7 @@ import { test as base } from '@playwright/test';
export type Fixture =
| 'ssh-node'
| 'ssh-node-bpf'
| 'connect';
export const test = base.extend<{ fixtures: Fixture[] }>({
+2 -2
View File
@@ -35,9 +35,9 @@ export class UnifiedResourcesPage {
async connect(serverName: string, login: string) {
const row = this.page
.locator('div')
.filter({ hasText: serverName })
.filter({ has: this.page.getByText(serverName, { exact: true }) })
.filter({ has: this.page.getByRole('button', { name: 'Connect' }) })
.first();
.last();
await row.getByRole('button', { name: 'Connect' }).click();
const popupPromise = this.page.waitForEvent('popup');
+4
View File
@@ -78,11 +78,14 @@ const tryLoadRecordingMapping = cachedJSONLoader<
const defaultUser: UserDefinition = { roles: ['access', 'editor'] };
type Browser = 'chromium' | 'firefox' | 'webkit';
interface E2EFixtures {
recordings: string[];
user: UserDefinition;
users: UserDefinition[];
teleport: TeleportOption;
browsers: Browser[];
username: string;
loginAs: (index: number) => Promise<LoginAsResult>;
recordingIds: Record<string, string>;
@@ -104,6 +107,7 @@ export const test = base.extend<E2EFixtures>({
user: [undefined as unknown as UserDefinition, { option: true }],
users: [[], { option: true }],
teleport: [undefined as unknown as TeleportOption, { option: true }],
browsers: [[], { option: true }],
username: async ({ user, users }, use, testInfo) => {
const mapping = tryLoadUserMapping() ?? {};
+6 -2
View File
@@ -1,7 +1,7 @@
{{- /*gotype: github.com/gravitational/teleport/e2e/runner.TeleportNodeConfig */ -}}
version: v3
teleport:
nodename: docker-node
nodename: {{ .NodeName }}
auth_token: foo
data_dir: /var/lib/teleport
log:
@@ -17,9 +17,13 @@ auth_service:
ssh_service:
enabled: "yes"
listen_addr: 0.0.0.0:{{ .SSHServerPort }}
public_addr: localhost:{{ .SSHServerPort }}
public_addr: {{ .SSHPublicHost }}:{{ .SSHServerPort }}
labels:
env: example
{{- if .EnhancedRecording }}
enhanced_recording:
enabled: "yes"
{{- end }}
proxy_service:
enabled: "no"
+85
View File
@@ -0,0 +1,85 @@
/**
* Teleport
* Copyright (C) 2026 Gravitational, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"context"
"fmt"
"strings"
"github.com/moby/moby/client"
"github.com/gravitational/teleport/e2e/runner/fixtures"
)
type nodeArch string
const (
amd64 nodeArch = "amd64"
arm64 nodeArch = "arm64"
)
// crossCompilers maps a GOARCH to the C cross-compiler used to build the docker node binary
var crossCompilers = map[nodeArch]string{
amd64: "x86_64-unknown-linux-gnu-gcc",
arm64: "aarch64-unknown-linux-gnu-gcc",
}
// resolveNodeArch returns the architecture to build and run the docker node for
func resolveNodeArch(ctx context.Context) (nodeArch, error) {
if !fixtures.SSHNodeBPF.Enabled {
return amd64, nil
}
arch, err := daemonArch(ctx)
if err != nil {
return "", fmt.Errorf("determining node architecture for enhanced recording: %w", err)
}
return arch, nil
}
// unameToGOARCH maps the machine names the docker daemon reports to their GOARCH equivalents.
var unameToGOARCH = map[string]nodeArch{
"x86_64": amd64,
"amd64": amd64,
"aarch64": arm64,
"arm64": arm64,
}
// daemonArch returns the GOARCH of the machine running the docker daemon, which is not necessarily
// the machine running the tests.
func daemonArch(ctx context.Context) (nodeArch, error) {
api, err := dockerAPI()
if err != nil {
return "", err
}
result, err := api.Info(ctx, client.InfoOptions{})
if err != nil {
return "", fmt.Errorf("querying docker daemon info: %w", err)
}
arch, ok := unameToGOARCH[strings.ToLower(result.Info.Architecture)]
if !ok {
return "", fmt.Errorf("unsupported docker daemon architecture %q", result.Info.Architecture)
}
return arch, nil
}
+25 -9
View File
@@ -46,7 +46,7 @@ func build(ctx context.Context, config *e2eConfig) error {
}
var buildNode bool
if fixtures.SSHNode.Enabled && runtime.GOOS != "linux" {
if sshNodeEnabled() && needsNodeBuild(config) {
buildNode = shouldBuild(filepath.Join(nodeBuildDir, "build", "teleport-node"), config.noBuild)
}
@@ -95,18 +95,27 @@ func build(ctx context.Context, config *e2eConfig) error {
if buildNode {
g.Go(func() error {
slog.InfoContext(ctx, "cross-compiling teleport for linux (docker node)", "dir", nodeBuildDir)
arch := config.nodeArch
slog.InfoContext(ctx, "building teleport for linux (docker node)", "dir", nodeBuildDir, "arch", arch)
output := filepath.Join(nodeBuildDir, "build", "teleport-node")
cmd := exec.CommandContext(ctx, "go", "build",
"-o", output,
"-buildvcs=false",
"./tool/teleport",
)
args := []string{"build", "-o", output, "-buildvcs=false"}
if fixtures.SSHNodeBPF.Enabled {
args = append(args, "-tags", "bpf")
}
args = append(args, "./tool/teleport")
cmd := exec.CommandContext(ctx, "go", args...)
cmd.Dir = nodeBuildDir
env := append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=1")
env := append(os.Environ(), "GOOS=linux", "GOARCH="+string(arch), "CGO_ENABLED=1")
if os.Getenv("CC") == "" {
env = append(env, "CC=x86_64-unknown-linux-gnu-gcc")
cc, ok := crossCompilers[arch]
if !ok {
return fmt.Errorf("no cross-compiler known for GOARCH %s, set CC to override", arch)
}
env = append(env, "CC="+cc)
}
cmd.Env = env
cmd.Stdout = os.Stdout
@@ -169,6 +178,13 @@ func build(ctx context.Context, config *e2eConfig) error {
return g.Wait()
}
// needsNodeBuild reports whether the docker node needs a binary of its own rather than the host's,
// which is whenever the host build does not target the platform the container runs. A native linux
// build already carries the bpf tag (see BPF_TAG in common.mk), so it needs nothing extra.
func needsNodeBuild(config *e2eConfig) bool {
return runtime.GOOS != "linux" || config.nodeArch != nodeArch(runtime.GOARCH)
}
func runMake(ctx context.Context, dir string, targets ...string) error {
if err := runInDir(ctx, dir, "make", targets...); err != nil {
return fmt.Errorf("make %v: %w", targets, err)
+302
View File
@@ -0,0 +1,302 @@
/**
* Teleport
* Copyright (C) 2026 Gravitational, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"context"
"fmt"
"io"
"net"
"net/url"
"os"
"os/exec"
"strings"
"sync"
"time"
sdkclient "github.com/docker/go-sdk/client"
dockercontext "github.com/docker/go-sdk/context"
apicontainer "github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
// dockerHostURL is the parsed DOCKER_HOST, or nil when talking to a local daemon.
var dockerHostURL = sync.OnceValues(func() (*url.URL, error) {
host := os.Getenv("DOCKER_HOST")
if host == "" {
return nil, nil
}
u, err := url.Parse(host)
if err != nil {
return nil, fmt.Errorf("parsing DOCKER_HOST %q: %w", host, err)
}
return u, nil
})
// dockerAPI is the moby client for the daemon named by DOCKER_HOST. The ssh:// scheme is handled
// here because neither the moby client nor go-sdk implements it.
var dockerAPI = sync.OnceValues(func() (*client.Client, error) {
u, err := dockerHostURL()
if err != nil {
return nil, err
}
if u == nil || u.Scheme != "ssh" {
// CurrentDockerHost prefers a rootless socket over DOCKER_HOST, so it is only consulted when
// DOCKER_HOST is unset, to keep docker's own precedence.
host := ""
if u != nil {
host = u.String()
} else {
resolved, err := dockercontext.CurrentDockerHost()
if err != nil {
return nil, fmt.Errorf("resolving docker host: %w", err)
}
host = resolved
}
cli, err := client.New(client.FromEnv, client.WithHost(host))
if err != nil {
return nil, fmt.Errorf("creating docker client for %q: %w", host, err)
}
return cli, nil
}
cli, err := client.New(
client.WithHost("http://docker.example.com"),
client.WithDialContext(sshDialContext(u)),
)
if err != nil {
return nil, fmt.Errorf("creating docker client over ssh: %w", err)
}
return cli, nil
})
// dockerSDK wraps the moby client so go-sdk's container helpers use the same transport.
var dockerSDK = sync.OnceValues(func() (sdkclient.SDKClient, error) {
api, err := dockerAPI()
if err != nil {
return nil, err
}
cli, err := sdkclient.New(context.Background(), sdkclient.WithDockerAPI(api))
if err != nil {
return nil, fmt.Errorf("creating docker sdk client: %w", err)
}
return cli, nil
})
// sshDialContext returns a dialer that reaches the remote daemon by running the docker CLI there,
// which is how the docker CLI itself implements ssh:// transport.
func sshDialContext(u *url.URL) func(ctx context.Context, network, addr string) (net.Conn, error) {
// -T because the stream carries the docker API, and an ssh config forcing a PTY would corrupt it.
args := []string{"-T", "-o", "ConnectTimeout=30"}
if u.User != nil {
args = append(args, "-l", u.User.Username())
}
if port := u.Port(); port != "" {
args = append(args, "-p", port)
}
// The terminator goes before the destination, as the docker CLI does it, so that it does not depend
// on ssh consuming one that follows the destination.
args = append(args, "--", u.Hostname(), "docker", "system", "dial-stdio")
return func(ctx context.Context, _, _ string) (net.Conn, error) {
cmd := exec.CommandContext(ctx, "ssh", args...)
cmd.Stderr = os.Stderr
stdin, err := cmd.StdinPipe()
if err != nil {
return nil, fmt.Errorf("ssh stdin: %w", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("ssh stdout: %w", err)
}
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("starting ssh: %w", err)
}
return &commandConn{cmd: cmd, reader: stdout, writer: stdin}, nil
}
}
type commandConn struct {
cmd *exec.Cmd
reader io.ReadCloser
writer io.WriteCloser
}
func (c *commandConn) Read(b []byte) (int, error) { return c.reader.Read(b) }
func (c *commandConn) Write(b []byte) (int, error) { return c.writer.Write(b) }
func (c *commandConn) Close() error {
_ = c.writer.Close()
_ = c.reader.Close()
if c.cmd.Process != nil {
_ = c.cmd.Process.Kill()
}
_ = c.cmd.Wait()
return nil
}
func (c *commandConn) LocalAddr() net.Addr { return commandAddr{} }
func (c *commandConn) RemoteAddr() net.Addr { return commandAddr{} }
func (c *commandConn) SetDeadline(_ time.Time) error { return nil }
func (c *commandConn) SetReadDeadline(_ time.Time) error { return nil }
func (c *commandConn) SetWriteDeadline(_ time.Time) error { return nil }
type commandAddr struct{}
func (commandAddr) Network() string { return "ssh" }
func (commandAddr) String() string { return "ssh" }
// bpfPreconditions is run in a throwaway privileged container to decide whether the daemon's kernel
// can support Enhanced Session Recordings.
const bpfPreconditions = "test -e /sys/kernel/btf/vmlinux && test -e /proc/self/sessionid"
// daemonSupportsBPF reports whether the docker daemon's kernel can load the enhanced recording programs.
func daemonSupportsBPF(ctx context.Context, image string, arch nodeArch) (bool, error) {
api, err := dockerAPI()
if err != nil {
return false, err
}
created, err := api.ContainerCreate(ctx, client.ContainerCreateOptions{
Config: &apicontainer.Config{
Image: image,
Entrypoint: []string{"sh", "-c", bpfPreconditions},
},
HostConfig: &apicontainer.HostConfig{Privileged: true},
Platform: &ocispec.Platform{OS: "linux", Architecture: string(arch)},
})
if err != nil {
return false, fmt.Errorf("creating bpf probe container: %w", err)
}
defer func() {
_, _ = api.ContainerRemove(context.WithoutCancel(ctx), created.ID, client.ContainerRemoveOptions{Force: true})
}()
// Wait before starting so a container that exits immediately cannot be missed.
wait := api.ContainerWait(ctx, created.ID, client.ContainerWaitOptions{
Condition: apicontainer.WaitConditionNextExit,
})
if _, err := api.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {
return false, fmt.Errorf("starting bpf probe container: %w", err)
}
select {
case err := <-wait.Error:
return false, fmt.Errorf("waiting for bpf probe container: %w", err)
case result := <-wait.Result:
return result.StatusCode == 0, nil
case <-ctx.Done():
return false, ctx.Err()
}
}
// nodePublicHost is the address the node advertises for its SSH port. The container publishes that port
// on whichever machine runs the daemon, so with a remote DOCKER_HOST it is not the machine running the
// tests, and the proxy would otherwise dial itself.
func nodePublicHost() (string, error) {
u, err := dockerHostURL()
if err != nil {
return "", err
}
return publicHostFor(u)
}
func publicHostFor(u *url.URL) (string, error) {
if u == nil {
return "localhost", nil
}
switch u.Scheme {
case "ssh":
target, err := sshRouteTarget(u)
if err != nil {
return "", err
}
host, _, err := net.SplitHostPort(target)
if err != nil {
return "", fmt.Errorf("splitting ssh route target %q: %w", target, err)
}
return host, nil
case "tcp", "http", "https":
return u.Hostname(), nil
default:
return "localhost", nil
}
}
// sshRouteTarget resolves the real host and port behind a DOCKER_HOST ssh URL by asking ssh to apply its own config
func sshRouteTarget(u *url.URL) (string, error) {
out, err := exec.Command("ssh", "-G", u.Hostname()).Output()
if err != nil {
return "", fmt.Errorf("resolving ssh host %q: %w", u.Hostname(), err)
}
hostname, port := parseSSHConfig(out)
if hostname == "" {
return "", fmt.Errorf("ssh did not report a hostname for %q", u.Hostname())
}
// An explicit port in DOCKER_HOST wins over the one ssh config supplies.
if p := u.Port(); p != "" {
port = p
}
return net.JoinHostPort(hostname, port), nil
}
func parseSSHConfig(out []byte) (hostname, port string) {
port = "22"
for line := range strings.Lines(string(out)) {
key, value, ok := strings.Cut(strings.TrimSpace(line), " ")
if !ok {
continue
}
switch key {
case "hostname":
hostname = value
case "port":
port = value
}
}
return hostname, port
}
+91
View File
@@ -0,0 +1,91 @@
/**
* Teleport
* Copyright (C) 2026 Gravitational, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"net/url"
"testing"
"github.com/stretchr/testify/require"
)
func TestPublicHostFor(t *testing.T) {
tests := []struct {
name string
host string
want string
}{
{name: "no DOCKER_HOST", host: "", want: "localhost"},
{name: "local socket", host: "unix:///var/run/docker.sock", want: "localhost"},
{name: "remote tcp", host: "tcp://linux-box:2375", want: "linux-box"},
{name: "remote tcp without port", host: "tcp://10.0.1.42", want: "10.0.1.42"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var u *url.URL
if test.host != "" {
parsed, err := url.Parse(test.host)
require.NoError(t, err)
u = parsed
}
got, err := publicHostFor(u)
require.NoError(t, err)
require.Equal(t, test.want, got)
})
}
}
func TestParseSSHConfig(t *testing.T) {
// Trimmed `ssh -G linux-box` output for a Host alias that rewrites both hostname and port.
const withAlias = `user ryan
hostname 10.0.1.42
port 2222
addressfamily any
identityfile ~/.ssh/id_ed25519
requesttty auto
`
const noPort = `user ryan
hostname linux-box.internal
addressfamily any
`
tests := []struct {
name string
out string
wantHostname string
wantPort string
}{
{name: "alias rewrites hostname and port", out: withAlias, wantHostname: "10.0.1.42", wantPort: "2222"},
{name: "port defaults to 22", out: noPort, wantHostname: "linux-box.internal", wantPort: "22"},
{name: "no hostname reported", out: "user ryan\n", wantHostname: "", wantPort: "22"},
{name: "empty output", out: "", wantHostname: "", wantPort: "22"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
hostname, port := parseSSHConfig([]byte(test.out))
require.Equal(t, test.wantHostname, hostname)
require.Equal(t, test.wantPort, port)
})
}
}
+3 -1
View File
@@ -22,7 +22,9 @@ import "flag"
var (
SSHNode = register("ssh-node")
Connect = register("connect")
// SSHNodeBPF runs a second node, docker-node-bpf, with Enhanced Session Recording enabled.
SSHNodeBPF = register("ssh-node-bpf")
Connect = register("connect")
)
type Fixture struct {
+3 -3
View File
@@ -4,7 +4,9 @@ go 1.26.5
require (
github.com/DataDog/datadog-agent/pkg/template v0.77.2
github.com/docker/go-sdk/client v0.1.0-alpha012
github.com/docker/go-sdk/container v0.1.0-alpha013
github.com/docker/go-sdk/context v0.1.0-alpha012
github.com/google/go-github/v84 v84.0.0
github.com/google/uuid v1.6.0
github.com/gravitational/teleport v0.0.0-00010101000000-000000000000
@@ -12,6 +14,7 @@ require (
github.com/lmittmann/tint v1.1.3
github.com/moby/moby/api v1.54.2
github.com/moby/moby/client v0.4.1
github.com/opencontainers/image-spec v1.1.1
github.com/stretchr/testify v1.11.1
golang.org/x/crypto v0.54.0
golang.org/x/oauth2 v0.36.0
@@ -91,9 +94,7 @@ require (
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/docker/go-connections v0.7.0 // indirect
github.com/docker/go-sdk/client v0.1.0-alpha012 // indirect
github.com/docker/go-sdk/config v0.1.0-alpha012 // indirect
github.com/docker/go-sdk/context v0.1.0-alpha012 // indirect
github.com/docker/go-sdk/image v0.1.0-alpha013 // indirect
github.com/docker/go-sdk/network v0.1.0-alpha012 // indirect
github.com/docker/go-units v0.5.0 // indirect
@@ -208,7 +209,6 @@ require (
github.com/okta/okta-sdk-golang/v2 v2.20.0 // indirect
github.com/openai/openai-go/v3 v3.8.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pkg/errors v0.9.1 // indirect
+7 -7
View File
@@ -30,14 +30,14 @@ type testInstance struct {
log *slog.Logger
proxyPort int
authPort int
sshPort int
sshPorts []int
e2eDir string
dataDir string
tctlBin string
noResourceSetup bool
teleportConfigPath string
teleport *teleportInstance
node *dockerNode
nodes []*dockerNode
}
// start starts the Teleport instance and SSH node for this test instance.
@@ -68,11 +68,11 @@ func (inst *testInstance) start(ctx context.Context) error {
}
}
if inst.node != nil {
if err = inst.node.start(ctx); err != nil {
for _, node := range inst.nodes {
if err = node.start(ctx); err != nil {
return fmt.Errorf("failed to start docker node for %s: %w", inst.browser, err)
}
if err = inst.node.waitJoined(ctx, 30*time.Second); err != nil {
if err = node.waitJoined(ctx, 30*time.Second); err != nil {
return fmt.Errorf("docker node for %s failed to join cluster: %w", inst.browser, err)
}
}
@@ -81,8 +81,8 @@ func (inst *testInstance) start(ctx context.Context) error {
}
func (inst *testInstance) stop() {
if inst.node != nil {
inst.node.stop(context.Background())
for _, node := range inst.nodes {
node.stop(context.Background())
}
if inst.teleport != nil {
+104 -30
View File
@@ -29,7 +29,6 @@ import (
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"sort"
"syscall"
"time"
@@ -143,6 +142,16 @@ type e2eConfig struct {
// Empty when the teleport binary is overridden and no build is needed.
teleportBuildDir string
// nodeArch is the GOARCH the docker node is built and run for.
nodeArch nodeArch
// skipEnhancedRecording is set when the docker daemon's kernel cannot run Enhanced Session
// Recording, so the tests that need it skip themselves instead of failing.
skipEnhancedRecording bool
// browserRestrictions maps a spec path to the browsers it declared it should run against.
browserRestrictions map[string][]string
connectAppDir string
connectTshBinPath string
@@ -239,8 +248,11 @@ func run(flags *e2eFlags, mode runMode, e2eDir string, isCI bool) error {
var portTargets []*int
for _, inst := range config.instances {
portTargets = append(portTargets, &inst.proxyPort, &inst.authPort)
if fixtures.SSHNode.Enabled {
portTargets = append(portTargets, &inst.sshPort)
if variants := nodeVariants(); len(variants) > 0 {
inst.sshPorts = make([]int, len(variants))
for i := range inst.sshPorts {
portTargets = append(portTargets, &inst.sshPorts[i])
}
}
}
if ci := config.connectInstance; ci != nil {
@@ -252,12 +264,22 @@ func run(flags *e2eFlags, mode runMode, e2eDir string, isCI bool) error {
}
for _, inst := range config.instances {
inst.log.DebugContext(ctx, "allocated ports", "proxy", inst.proxyPort, "auth", inst.authPort, "ssh", inst.sshPort)
inst.log.DebugContext(ctx, "allocated ports", "proxy", inst.proxyPort, "auth", inst.authPort, "ssh", inst.sshPorts)
}
if ci := config.connectInstance; ci != nil {
ci.log.DebugContext(ctx, "allocated ports", "proxy", ci.proxyPort, "auth", ci.authPort)
}
if sshNodeEnabled() {
nodeArch, err := resolveNodeArch(ctx)
if err != nil {
return err
}
config.nodeArch = nodeArch
slog.DebugContext(ctx, "resolved docker node architecture", "arch", config.nodeArch)
}
if err := build(ctx, config); err != nil {
return fmt.Errorf("failed to build binaries: %w", err)
}
@@ -296,6 +318,12 @@ func run(flags *e2eFlags, mode runMode, e2eDir string, isCI bool) error {
}
}
// Left alone when nothing resolved, since an empty list would mean "run everything" instead of
// letting Playwright reject the selection.
if expanded := expandedTestFiles(targets); len(config.testFiles) > 0 && len(expanded) > 0 {
config.testFiles = expanded
}
scannedUsers, err := scanUsersFromTargets(targets)
if err != nil {
return fmt.Errorf("failed to scan users: %w", err)
@@ -307,6 +335,14 @@ func run(flags *e2eFlags, mode runMode, e2eDir string, isCI bool) error {
return fmt.Errorf("failed to scan teleport configs: %w", err)
}
config.browserRestrictions, err = scanBrowserRestrictions(targets)
if err != nil {
return fmt.Errorf("failed to scan browser restrictions: %w", err)
}
if len(config.browserRestrictions) > 0 {
slog.DebugContext(ctx, "discovered browser restrictions", "specs", config.browserRestrictions)
}
bootstrap, err := buildBootstrapState(e2eDir, scannedUsers)
if err != nil {
return fmt.Errorf("failed to build bootstrap state: %w", err)
@@ -376,11 +412,11 @@ func run(flags *e2eFlags, mode runMode, e2eDir string, isCI bool) error {
inst.teleport = teleport
}
if fixtures.SSHNode.Enabled {
slog.InfoContext(ctx, "running with SSH node fixture enabled")
if sshNodeEnabled() {
slog.InfoContext(ctx, "running with SSH node fixture enabled", "nodes", nodeVariantNames())
nodeBin := config.teleportBin
if runtime.GOOS != "linux" {
if needsNodeBuild(config) {
buildDir := config.teleportBuildDir
if buildDir == "" {
buildDir = config.repoRoot
@@ -393,33 +429,71 @@ func run(flags *e2eFlags, mode runMode, e2eDir string, isCI bool) error {
return fmt.Errorf("resolving docker host: %w", err)
}
for _, inst := range config.instances {
outPath := filepath.Join(e2eDir, "node", inst.browser+"-node.yaml")
nodeConfigPath, err := generateTeleportNodeConfig(config.nodeConfigTemplate, outPath, &TeleportNodeConfig{
AuthServerHost: dockerHost,
AuthServerPort: inst.authPort,
SSHServerPort: inst.sshPort,
})
if err != nil {
return fmt.Errorf("failed to generate node config for %s: %w", inst.browser, err)
}
inst.log.DebugContext(ctx, "generated Teleport node config", "path", nodeConfigPath)
sshPublicHost, err := nodePublicHost()
if err != nil {
return fmt.Errorf("resolving node public host: %w", err)
}
inst.node = &dockerNode{
log: inst.log,
sshPort: inst.sshPort,
tctlBin: config.tctlBin,
teleportConfigPath: inst.teleportConfigPath,
logFilePath: filepath.Join(config.e2eDir, "docker-node-"+inst.browser+".log"),
imageName: nodeImage,
containerName: "teleport-e2e-node-" + inst.browser,
configPath: nodeConfigPath,
teleportBin: nodeBin,
if err := pullImage(ctx, nodeImage, config.nodeArch); err != nil {
return fmt.Errorf("pulling docker image: %w", err)
}
enhancedRecording := fixtures.SSHNodeBPF.Enabled
if enhancedRecording {
supported, probeErr := daemonSupportsBPF(ctx, nodeImage, config.nodeArch)
if probeErr != nil {
return fmt.Errorf("checking enhanced recording support: %w", probeErr)
}
if !supported {
if config.isCI {
return fmt.Errorf("the docker daemon's kernel cannot run enhanced session recording: " +
"it needs BTF at /sys/kernel/btf/vmlinux and CONFIG_AUDIT for task_struct.sessionid")
}
slog.WarnContext(ctx, "docker daemon's kernel cannot run enhanced session recording, skipping those tests",
"needs", "BTF and CONFIG_AUDIT")
enhancedRecording = false
config.skipEnhancedRecording = true
}
}
if err := pullImage(ctx, nodeImage); err != nil {
return fmt.Errorf("pulling docker image: %w", err)
for _, inst := range config.instances {
for i, variant := range nodeVariants() {
// Only the BPF variant carries enhanced recording, and it loses it when the daemon's
// kernel cannot run the programs.
nodeEnhancedRecording := variant.enhancedRecording && enhancedRecording
outPath := filepath.Join(e2eDir, "node", inst.browser+"-"+variant.name+".yaml")
nodeConfigPath, err := generateTeleportNodeConfig(config.nodeConfigTemplate, outPath, &TeleportNodeConfig{
NodeName: variant.name,
AuthServerHost: dockerHost,
AuthServerPort: inst.authPort,
SSHServerPort: inst.sshPorts[i],
SSHPublicHost: sshPublicHost,
EnhancedRecording: nodeEnhancedRecording,
})
if err != nil {
return fmt.Errorf("failed to generate node config for %s: %w", inst.browser, err)
}
inst.log.DebugContext(ctx, "generated Teleport node config", "path", nodeConfigPath)
inst.nodes = append(inst.nodes, &dockerNode{
log: inst.log,
nodeName: variant.name,
sshPort: inst.sshPorts[i],
tctlBin: config.tctlBin,
teleportConfigPath: inst.teleportConfigPath,
logFilePath: filepath.Join(config.e2eDir, variant.name+"-"+inst.browser+".log"),
imageName: nodeImage,
containerName: "teleport-e2e-" + variant.name + "-" + inst.browser,
configPath: nodeConfigPath,
teleportBin: nodeBin,
arch: config.nodeArch,
enhancedRecording: nodeEnhancedRecording,
})
}
}
}
}
+101 -18
View File
@@ -32,12 +32,57 @@ import (
apicontainer "github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
"github.com/moby/moby/client"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/gravitational/teleport/e2e/runner/fixtures"
)
const nodeImage = "debian:bookworm-slim"
// nodeVariant is one docker node the run needs. A spec asks for a variant by declaring its fixture, and
// only the variants that were asked for are started, so a run that wants both gets two nodes to target.
type nodeVariant struct {
name string
enhancedRecording bool
}
func sshNodeEnabled() bool {
return fixtures.SSHNode.Enabled || fixtures.SSHNodeBPF.Enabled
}
func nodeVariants() []nodeVariant {
var variants []nodeVariant
if fixtures.SSHNode.Enabled {
variants = append(variants, nodeVariant{name: "docker-node"})
}
if fixtures.SSHNodeBPF.Enabled {
variants = append(variants, nodeVariant{name: "docker-node-bpf", enhancedRecording: true})
}
return variants
}
func nodeVariantNames() []string {
var names []string
for _, v := range nodeVariants() {
names = append(names, v.name)
}
return names
}
// nodeStartCommand is how the node is launched inside the container.
const nodeStartCommand = "teleport start --insecure -c /etc/teleport/node.yaml"
// mountTracefs mounts tracefs for the tracepoint and kprobe programs to attach through, since Docker
// does not mount it even for a privileged container. Either mount point will do, so failures are left
// for the node to report.
const mountTracefs = "mount -t tracefs nodev /sys/kernel/tracing 2>/dev/null || " +
"mount -t debugfs nodev /sys/kernel/debug 2>/dev/null || true"
type dockerNode struct {
log *slog.Logger
nodeName string
sshPort int
tctlBin string
teleportConfigPath string
@@ -48,6 +93,9 @@ type dockerNode struct {
configPath string
teleportBin string
arch nodeArch
enhancedRecording bool
ctr *container.Container
}
@@ -55,30 +103,43 @@ func (d *dockerNode) start(ctx context.Context) error {
return d.runContainer(ctx)
}
func (d *dockerNode) entrypoint() []string {
if !d.enhancedRecording {
return strings.Fields(nodeStartCommand)
}
return []string{"sh", "-c", mountTracefs + "; exec " + nodeStartCommand}
}
func (d *dockerNode) removeStale(ctx context.Context) {
cli, err := client.New()
cli, err := dockerAPI()
if err != nil {
return
}
defer cli.Close()
_, _ = cli.ContainerRemove(ctx, d.containerName, client.ContainerRemoveOptions{Force: true})
}
func (d *dockerNode) runContainer(ctx context.Context) error {
d.log.InfoContext(ctx, "starting docker SSH node")
d.log.InfoContext(ctx, "starting docker SSH node", "node", d.nodeName)
d.removeStale(ctx)
sdk, err := dockerSDK()
if err != nil {
return err
}
ctr, err := container.Run(ctx,
container.WithClient(sdk),
container.WithImage(d.imageName),
container.WithImagePlatform("linux/amd64"),
container.WithImagePlatform("linux/"+string(d.arch)),
container.WithPullHandler(func(r io.ReadCloser) error {
_, err := io.Copy(io.Discard, r)
return err
}),
container.WithName(d.containerName),
container.WithEntrypoint("teleport", "start", "--insecure", "-c", "/etc/teleport/node.yaml"),
container.WithEntrypoint(d.entrypoint()...),
container.WithExposedPorts(fmt.Sprintf("%d/tcp", d.sshPort)),
container.WithFiles(
container.File{
@@ -96,6 +157,11 @@ func (d *dockerNode) runContainer(ctx context.Context) error {
if os.Getenv("DOCKER_HOST") == "" {
hc.ExtraHosts = []string{"host.docker.internal:host-gateway"}
}
if d.enhancedRecording {
hc.Privileged = true
}
hc.PortBindings = network.PortMap{
network.MustParsePort(fmt.Sprintf("%d/tcp", d.sshPort)): []network.PortBinding{
{HostPort: fmt.Sprintf("%d", d.sshPort)},
@@ -114,7 +180,7 @@ func (d *dockerNode) runContainer(ctx context.Context) error {
}
func (d *dockerNode) waitJoined(ctx context.Context, timeout time.Duration) error {
d.log.DebugContext(ctx, "waiting for docker node to join cluster")
d.log.DebugContext(ctx, "waiting for docker node to join cluster", "node", d.nodeName)
probe := func(ctx context.Context) (bool, error) {
cmd := exec.CommandContext(ctx, d.tctlBin, "nodes", "ls",
@@ -124,14 +190,21 @@ func (d *dockerNode) waitJoined(ctx context.Context, timeout time.Duration) erro
return false, nil
}
return strings.Contains(string(out), "docker-node"), nil
// Exact field match, since one node's name can be a prefix of another's.
for line := range strings.Lines(string(out)) {
if fields := strings.Fields(line); len(fields) > 0 && fields[0] == d.nodeName {
return true, nil
}
}
return false, nil
}
if err := pollUntil(ctx, timeout, 1*time.Second, probe); err != nil {
return fmt.Errorf("docker node failed to join cluster: %w", err)
return fmt.Errorf("docker node %s failed to join cluster: %w", d.nodeName, err)
}
d.log.InfoContext(ctx, "docker SSH node is ready")
d.log.InfoContext(ctx, "docker SSH node is ready", "node", d.nodeName)
return nil
}
@@ -170,27 +243,37 @@ func (d *dockerNode) stop(ctx context.Context) {
return
}
d.log.InfoContext(ctx, "stopping docker SSH node")
d.log.InfoContext(ctx, "stopping docker SSH node", "node", d.nodeName)
d.saveLogs(ctx)
_ = d.ctr.Terminate(ctx, container.TerminateTimeout(10*time.Second))
}
func pullImage(ctx context.Context, image string) error {
slog.InfoContext(ctx, "pulling docker image", "image", image)
func pullImage(ctx context.Context, image string, arch nodeArch) error {
slog.InfoContext(ctx, "pulling docker image", "image", image, "arch", arch)
cli, err := client.New()
cli, err := dockerAPI()
if err != nil {
return fmt.Errorf("creating docker client: %w", err)
return err
}
defer cli.Close()
rc, err := cli.ImagePull(ctx, image, client.ImagePullOptions{})
rc, err := cli.ImagePull(ctx, image, client.ImagePullOptions{
Platforms: []ocispec.Platform{{OS: "linux", Architecture: string(arch)}},
})
if err != nil {
return fmt.Errorf("pulling image: %w", err)
}
defer rc.Close()
_, err = io.Copy(io.Discard, rc)
return err
// Wait surfaces failures reported inside the progress stream, which a plain copy would discard.
// A cached image is still usable when the registry refuses us, so that is only a warning.
if err := rc.Wait(ctx); err != nil {
if _, inspectErr := cli.ImageInspect(ctx, image); inspectErr != nil {
return fmt.Errorf("pulling image: %w", err)
}
slog.WarnContext(ctx, "could not refresh docker image, using the cached one", "image", image, "error", err)
}
return nil
}
+74
View File
@@ -0,0 +1,74 @@
/**
* Teleport
* Copyright (C) 2026 Gravitational, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/gravitational/teleport/e2e/runner/fixtures"
)
func TestNodeVariants(t *testing.T) {
tests := []struct {
name string
plain bool
bpf bool
want []string
wantBPF []bool
anyNodes bool
}{
{name: "no node fixtures", want: nil, anyNodes: false},
{name: "plain only", plain: true, want: []string{"docker-node"}, wantBPF: []bool{false}, anyNodes: true},
{name: "bpf only", bpf: true, want: []string{"docker-node-bpf"}, wantBPF: []bool{true}, anyNodes: true},
{
name: "both requested runs two nodes", plain: true, bpf: true,
want: []string{"docker-node", "docker-node-bpf"}, wantBPF: []bool{false, true}, anyNodes: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
plain, bpf := fixtures.SSHNode.Enabled, fixtures.SSHNodeBPF.Enabled
t.Cleanup(func() { fixtures.SSHNode.Enabled, fixtures.SSHNodeBPF.Enabled = plain, bpf })
fixtures.SSHNode.Enabled, fixtures.SSHNodeBPF.Enabled = test.plain, test.bpf
require.Equal(t, test.want, nodeVariantNames())
require.Equal(t, test.anyNodes, sshNodeEnabled())
for i, v := range nodeVariants() {
require.Equal(t, test.wantBPF[i], v.enhancedRecording, "variant %s", v.name)
}
})
}
}
func TestDockerNodeEntrypoint(t *testing.T) {
plain := (&dockerNode{}).entrypoint()
require.Equal(t, []string{"teleport", "start", "--insecure", "-c", "/etc/teleport/node.yaml"}, plain)
bpf := (&dockerNode{enhancedRecording: true}).entrypoint()
require.Len(t, bpf, 3)
require.Equal(t, []string{"sh", "-c"}, bpf[:2])
require.Contains(t, bpf[2], "mount -t tracefs")
// exec keeps teleport as PID 1 so it still receives the stop signal.
require.Contains(t, bpf[2], "; exec "+nodeStartCommand)
}
+67 -17
View File
@@ -28,6 +28,7 @@ import (
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"time"
@@ -101,6 +102,16 @@ func (p *playwrightRunner) test(ctx context.Context, debug bool) error {
testErr := g.Wait()
// Nothing ran, so there is no report to merge and the previous run's results are still on disk.
// Reporting success here would read as green.
if blobs, err := filepath.Glob(filepath.Join(blobBaseDir, "*.zip")); err == nil && len(blobs) == 0 {
if testErr != nil {
return testErr
}
return fmt.Errorf("no specs ran, every selected spec was restricted to other browsers")
}
slog.InfoContext(ctx, "merging blob reports")
mergeArgs := []string{"exec", "playwright", "merge-reports", p.configFlag(), blobBaseDir}
mergeEnv := os.Environ()
@@ -125,49 +136,84 @@ func (p *playwrightRunner) test(ctx context.Context, debug bool) error {
}
func (p *playwrightRunner) runInstance(ctx context.Context, inst *testInstance, blobBaseDir string, debug bool, extraArgs []string) error {
hasConfigs := len(p.config.teleportConfigs) > 0
selected := p.config.testFiles
if hasConfigs {
selected = p.config.defaultTestFiles
}
defaultFiles := p.filesForProject(inst, selected)
// An empty list tells Playwright to run everything, so a selection that filtered down to nothing
// has to skip the pass rather than widen it.
runDefault := len(defaultFiles) > 0 || (!hasConfigs && len(selected) == 0)
configFiles := make([][]string, len(p.config.teleportConfigs))
anyConfigFiles := false
for i, cfg := range p.config.teleportConfigs {
configFiles[i] = p.filesForProject(inst, cfg.files)
anyConfigFiles = anyConfigFiles || len(configFiles[i]) > 0
}
if !runDefault && !anyConfigFiles {
inst.log.InfoContext(ctx, "no selected specs run against this browser, skipping")
return nil
}
if err := inst.start(ctx); err != nil {
return err
}
defer inst.stop()
hasConfigs := len(p.config.teleportConfigs) > 0
defaultFiles := filesForProject(inst, p.config.testFiles)
if hasConfigs {
defaultFiles = filesForProject(inst, p.config.defaultTestFiles)
}
// Every pass runs even after an earlier one fails. Each re-initializes Teleport from the base
// config, so they do not depend on each other, and stopping early would report a failing spec as
// the only result while silently dropping the coverage of the passes behind it.
var errs []error
// Skip the default pass if this instance has nothing to run against the base config
if !hasConfigs || len(defaultFiles) > 0 {
if runDefault {
blobPath := filepath.Join(blobBaseDir, inst.browser+".zip")
if err := p.runInstanceTests(ctx, inst, defaultFiles, blobPath, debug, extraArgs); err != nil {
return err
errs = append(errs, err)
}
}
baseConfigPath := inst.teleportConfigPath
for i, cfg := range p.config.teleportConfigs {
files := filesForProject(inst, cfg.files)
files := configFiles[i]
if len(files) == 0 {
continue // no tests for this instance's project
}
// An interrupted run is the one case worth abandoning, since the remaining passes can only
// fail on the dead context.
if ctx.Err() != nil {
break
}
if err := p.runTeleportConfig(ctx, inst, baseConfigPath, cfg, files, i, blobBaseDir, debug, extraArgs); err != nil {
return err
errs = append(errs, err)
}
}
return nil
return errors.Join(errs...)
}
// filesForProject picks the connect specs or browser specs for a testInstance
func filesForProject(inst *testInstance, files []string) []string {
// filesForProject picks the connect specs or browser specs for a testInstance, dropping any spec
// that restricted itself to other browsers.
func (p *playwrightRunner) filesForProject(inst *testInstance, files []string) []string {
if len(files) == 0 {
return files
}
var out []string
for _, f := range files {
isConnect := strings.HasPrefix(filepath.ToSlash(f), "tests/connect/")
if isConnect == (inst.browser == "connect") {
out = append(out, f)
if isConnect != (inst.browser == "connect") {
continue
}
spec := lineNumberSuffixRe.ReplaceAllString(filepath.ToSlash(f), "")
if allowed, ok := p.config.browserRestrictions[spec]; ok && !slices.Contains(allowed, inst.browser) {
continue
}
out = append(out, f)
}
return out
}
@@ -230,8 +276,8 @@ func (p *playwrightRunner) runTeleportConfig(ctx context.Context, inst *testInst
return fmt.Errorf("teleport for %s not ready after config change: %w", inst.browser, err)
}
if inst.node != nil {
if err := inst.node.waitJoined(ctx, 30*time.Second); err != nil {
for _, node := range inst.nodes {
if err := node.waitJoined(ctx, 30*time.Second); err != nil {
return fmt.Errorf("node for %s failed to rejoin: %w", inst.browser, err)
}
}
@@ -349,6 +395,10 @@ func (p *playwrightRunner) startEnv(inst *testInstance) ([]string, error) {
env = append(env, "E2E_CONNECT_TSH_BIN="+p.config.connectTshBinPath)
env = append(env, "E2E_CONNECT_APP_DIR="+p.config.connectAppDir)
if p.config.skipEnhancedRecording {
env = append(env, "E2E_SKIP_ENHANCED_RECORDING=1")
}
return env, nil
}
+76
View File
@@ -37,6 +37,7 @@ import (
var (
fixtureArrayRe = regexp.MustCompile(`fixtures:\s*\[+([^]]*)]`)
browsersArrayRe = regexp.MustCompile(`browsers:\s*\[([^]]*)]`)
lineNumberSuffixRe = regexp.MustCompile(`:\d+$`)
fixtureRefRe = regexp.MustCompile(`['"]([^'"]+)['"]`)
helperImportRe = regexp.MustCompile(`from\s+['"]@gravitational/e2e/helpers/(\w+)['"]`)
@@ -238,6 +239,26 @@ func resolveFilesToScan(e2eDir string, testFiles []string) ([]scanTarget, error)
return targets, nil
}
// expandedTestFiles turns resolved targets back into Playwright selectors. Directory and substring
// arguments are handed on as the specs they matched, so that per-spec settings can be keyed by path.
func expandedTestFiles(targets []scanTarget) []string {
var files []string
for _, t := range targets {
if t.sourceFile == "" {
continue
}
file := filepath.ToSlash(t.sourceFile)
if t.line > 0 {
file = fmt.Sprintf("%s:%d", file, t.line)
}
files = append(files, file)
}
return files
}
func walkSpecFiles(root string) ([]string, error) {
var files []string
@@ -1077,6 +1098,61 @@ func sortRoles(roles []scannedRole) {
})
}
// scanBrowserRestrictions returns the browsers each spec file limited itself to, keyed by its path
// relative to e2e/.
//
// The restriction applies to the whole file rather than to an enclosing describe, since the runner
// picks instances per file.
func scanBrowserRestrictions(targets []scanTarget) (map[string][]string, error) {
restrictions := make(map[string][]string)
for _, t := range targets {
// Skip helper modules, they cannot restrict themselves to an individual browser.
if t.sourceFile == "" {
continue
}
cleaned, ok := readCleaned(t.path)
if !ok {
continue
}
content := strings.Join(cleaned, "\n")
var browsers []string
for _, call := range findTestUseCalls(content) {
body := content[call.start:call.end]
for _, m := range browsersArrayRe.FindAllStringSubmatch(body, -1) {
for _, ref := range fixtureRefRe.FindAllStringSubmatch(m[1], -1) {
browsers = append(browsers, ref[1])
}
}
}
if len(browsers) == 0 {
continue
}
spec := filepath.ToSlash(t.sourceFile)
// A restriction that no instance can satisfy would silently run the spec nowhere, so both an
// unknown browser and a Connect spec naming one are rejected rather than dropped.
if strings.HasPrefix(spec, "tests/connect/") {
return nil, fmt.Errorf("%s is a Connect spec, which runs in Electron and cannot restrict browsers", spec)
}
for _, b := range browsers {
if !slices.Contains(validBrowsers, b) {
return nil, fmt.Errorf("%s declares invalid browser %q, must be one of: %s",
spec, b, strings.Join(validBrowsers, ", "))
}
}
restrictions[spec] = browsers
}
return restrictions, nil
}
// uniqueTeleportConfig is a distinct custom Teleport config declared by one or more
// test files, and the test files that declare it.
type uniqueTeleportConfig struct {
+175
View File
@@ -1351,3 +1351,178 @@ func TestExtractTeleportConfigsRejectsNonInlineOption(t *testing.T) {
require.Error(t, err)
require.Contains(t, err.Error(), "must be an inline object")
}
func TestScanBrowserRestrictions(t *testing.T) {
dir := t.TempDir()
writeFile(t, dir, "pinned.spec.ts", `import { test } from '@gravitational/e2e/helpers/test';
test.describe('d', () => {
test.use({ fixtures: ['ssh-node'], browsers: ['chromium'] });
test('t', async () => {});
});`)
writeFile(t, dir, "multi.spec.ts", `import { test } from '@gravitational/e2e/helpers/test';
test.use({ browsers: ["chromium", 'firefox'] });
test('t', async () => {});`)
writeFile(t, dir, "unpinned.spec.ts", `import { test } from '@gravitational/e2e/helpers/test';
test.use({ fixtures: ['ssh-node'] });
test('t', async () => {});`)
got, err := scanBrowserRestrictions([]scanTarget{
{path: filepath.Join(dir, "pinned.spec.ts"), sourceFile: "pinned.spec.ts"},
{path: filepath.Join(dir, "multi.spec.ts"), sourceFile: "multi.spec.ts"},
{path: filepath.Join(dir, "unpinned.spec.ts"), sourceFile: "unpinned.spec.ts"},
})
require.NoError(t, err)
require.Equal(t, map[string][]string{
"pinned.spec.ts": {"chromium"},
"multi.spec.ts": {"chromium", "firefox"},
}, got)
}
func TestScanBrowserRestrictionsIgnoresHelpers(t *testing.T) {
dir := t.TempDir()
writeFile(t, dir, "helper.ts", `test.use({ browsers: ['chromium'] });`)
// A helper has no spec to attach the restriction to, so it must not silently pin anything.
got, err := scanBrowserRestrictions([]scanTarget{{path: filepath.Join(dir, "helper.ts")}})
require.NoError(t, err)
require.Empty(t, got)
}
func TestScanBrowserRestrictionsRejectsUnknownBrowser(t *testing.T) {
dir := t.TempDir()
writeFile(t, dir, "typo.spec.ts", `import { test } from '@gravitational/e2e/helpers/test';
test.use({ browsers: ['chrome'] });
test('t', async () => {});`)
_, err := scanBrowserRestrictions([]scanTarget{
{path: filepath.Join(dir, "typo.spec.ts"), sourceFile: "typo.spec.ts"},
})
require.Error(t, err)
require.Contains(t, err.Error(), `invalid browser "chrome"`)
}
func TestScanBrowserRestrictionsRejectsConnectSpec(t *testing.T) {
dir := t.TempDir()
writeFile(t, dir, "shell.spec.ts", `import { test } from '@gravitational/e2e/helpers/test';
test.use({ browsers: ['chromium'] });
test('t', async () => {});`)
// A Connect spec is routed to the Electron instance, so any browser pin leaves it running nowhere.
_, err := scanBrowserRestrictions([]scanTarget{
{path: filepath.Join(dir, "shell.spec.ts"), sourceFile: "tests/connect/shell.spec.ts"},
})
require.Error(t, err)
require.Contains(t, err.Error(), "cannot restrict browsers")
}
func TestExpandedTestFiles(t *testing.T) {
got := expandedTestFiles([]scanTarget{
{path: "/repo/e2e/tests/web/authenticated/ssh.spec.ts", sourceFile: "tests/web/authenticated/ssh.spec.ts"},
{path: "/repo/e2e/tests/web/authenticated/rbac.spec.ts", sourceFile: "tests/web/authenticated/rbac.spec.ts", line: 24},
// Helper modules are not selectors.
{path: "/repo/e2e/helpers/test.ts"},
})
require.Equal(t, []string{
"tests/web/authenticated/ssh.spec.ts",
"tests/web/authenticated/rbac.spec.ts:24",
}, got)
}
func TestExpandedTestFilesFromResolvedArgs(t *testing.T) {
e2eDir := t.TempDir()
specDir := filepath.Join(e2eDir, "tests", "web")
require.NoError(t, os.MkdirAll(specDir, 0o755))
require.NoError(t, os.MkdirAll(filepath.Join(e2eDir, "tests", "empty"), 0o755))
for _, name := range []string{"ssh.spec.ts", "rbac.spec.ts"} {
writeFile(t, specDir, name, `import { test } from '@gravitational/e2e/helpers/test';
test('t', async () => {});`)
}
tests := []struct {
name string
args []string
want []string
wantErr bool
}{
{
name: "file",
args: []string{"tests/web/ssh.spec.ts"},
want: []string{"tests/web/ssh.spec.ts"},
},
{
name: "file with line",
args: []string{"tests/web/ssh.spec.ts:42"},
want: []string{"tests/web/ssh.spec.ts:42"},
},
{
name: "directory expands to its specs",
args: []string{"tests/web"},
want: []string{"tests/web/rbac.spec.ts", "tests/web/ssh.spec.ts"},
},
{
name: "substring filter expands to matches",
args: []string{"ssh"},
want: []string{"tests/web/ssh.spec.ts"},
},
// A directory holding no specs resolves to nothing, and the caller must not treat that as
// "run everything".
{
name: "directory with no specs",
args: []string{"tests/empty"},
want: nil,
},
{
name: "substring matching nothing",
args: []string{"nosuchspec"},
wantErr: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
targets, err := resolveTargetsWithHelpers(e2eDir, test.args)
if test.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.Equal(t, test.want, expandedTestFiles(targets))
})
}
}
func TestFilesForProjectHonoursBrowserRestrictions(t *testing.T) {
p := &playwrightRunner{config: &e2eConfig{
browserRestrictions: map[string][]string{
"tests/web/authenticated/pinned.spec.ts": {"chromium"},
},
}}
files := []string{
"tests/web/authenticated/pinned.spec.ts:24",
"tests/web/authenticated/other.spec.ts",
"tests/connect/shell.spec.ts",
}
require.Equal(t, []string{
"tests/web/authenticated/pinned.spec.ts:24",
"tests/web/authenticated/other.spec.ts",
}, p.filesForProject(&testInstance{browser: "chromium"}, files))
// The pinned spec drops out for other browsers, so its config never triggers a re-init there.
require.Equal(t, []string{
"tests/web/authenticated/other.spec.ts",
}, p.filesForProject(&testInstance{browser: "firefox"}, files))
// Connect specs are still routed by project, not by the restriction map.
require.Equal(t, []string{
"tests/connect/shell.spec.ts",
}, p.filesForProject(&testInstance{browser: "connect"}, files))
}
+16 -4
View File
@@ -168,9 +168,12 @@ func generateTeleportConfig(templatePath, outPath string, data *TeleportConfig)
}
type TeleportNodeConfig struct {
AuthServerHost string
AuthServerPort int
SSHServerPort int
NodeName string
AuthServerHost string
AuthServerPort int
SSHServerPort int
SSHPublicHost string
EnhancedRecording bool
}
func generateTeleportNodeConfig(templatePath, outPath string, data *TeleportNodeConfig) (string, error) {
@@ -188,7 +191,16 @@ func resolveDockerHost() (string, error) {
return "", fmt.Errorf("parsing DOCKER_HOST: %w", err)
}
conn, err := net.Dial("udp", u.Host)
// This dial carries no traffic, it only picks a route so we can report the local address the daemon's host can reach us back on.
target := u.Host
if u.Scheme == "ssh" {
target, err = sshRouteTarget(u)
if err != nil {
return "", err
}
}
conn, err := net.Dial("udp", target)
if err != nil {
return "", fmt.Errorf("dialing docker host: %w", err)
}