MWI: Implement RFD 216 - tbot resiliency improvements (#55609)

* Replace `lib/auth/authclient` dependencies with `api/client`

* Add new `lib/tbot/client` package

* Handle identity renewal failure on-startup

* Add a `Ready` channel you can use to block until bot identity service is ready

* Allow embedders of `tbot` (e.g. operator) to use old behavior

* Fix racy test

* Fix duplicate import

* Keep retrying failed identity renewal on-startup

* Fix truncated comment

* Remove warning in v19

* Simplify loop function

* Always wait for first loop run

* Fix racy test

integrations/lib/embeddedtbot/bot_test.go

@@ -123,7 +123,8 @@ func TestBotJoinAuth(t *testing.T) {
 			TokenValue: tokenName,
 			JoinMethod: types.JoinMethodToken,
 		},
-		AuthServer: authAddr.Addr,
+		AuthServer:            authAddr.Addr,
+		AuthServerAddressMode: config.AllowProxyAsAuthServer,
 		CredentialLifetime: config.CredentialLifetime{
 			TTL:             defaultCertificateTTL,
 			RenewalInterval: defaultRenewalInterval,

integrations/operator/main.go

@@ -40,6 +40,7 @@ import (
 	"github.com/gravitational/teleport/integrations/lib/embeddedtbot"
 	"github.com/gravitational/teleport/integrations/operator/controllers"
 	"github.com/gravitational/teleport/integrations/operator/controllers/resources"
+	tbotconfig "github.com/gravitational/teleport/lib/tbot/config"
 	logutils "github.com/gravitational/teleport/lib/utils/log"
 )
 
@@ -70,6 +71,7 @@ func main() {
 	config := &operatorConfig{}
 	config.BindFlags(flag.CommandLine)
 	botConfig := &embeddedtbot.BotConfig{}
+	botConfig.AuthServerAddressMode = tbotconfig.AllowProxyAsAuthServer
 	botConfig.BindFlags(flag.CommandLine)
 	flag.Parse()
 

lib/tbot/client/client.go

@@ -0,0 +1,278 @@
+/*
+ * Teleport
+ * Copyright (C) 2025  Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// Package client wraps the api/client package to implement tbot's connection
+// logic.
+//
+// tbot supports two methods of dialing a gRPC connection to an auth server:
+//
+//  1. Via a direct TLS connection to the auth server itself
+//  2. Via an SSH tunnel through a proxy server
+//
+// Both methods support client-side HTTP and SOCKS5 proxies, and method #2
+// supports "TLS Routing" (via ALPN) and our "websocket upgrade" trick for
+// traversing L7 load balancers that break ALPN.
+//
+// This is a subset of the methods offered by the api/client package. In the
+// future, we may wish to add support for more of these methods, but many are
+// unnecessarily complex for tbot's use-cases.
+//
+// If you start tbot with the `proxy_server` config option or `--proxy-server`
+// flag, client.New will return a gRPC client *without* testing the connection.
+//
+// In other words: client.New will not return an error if the proxy or auth
+// server is down, which is desirable because it allows us to start tbot in a
+// "degraded" mode and retry the connection in the background.
+//
+// In previous versions of tbot, there was no way to explicitly provide the
+// address of a proxy server. Instead, you could either put a proxy or auth
+// server address in the `auth_server` option or `--auth-server` flag and tbot
+// would try both connection methods.
+//
+// For backward compatibility, if you provide `auth_server` or `--auth-server`
+// in major versions earlier than v19, client.New will test the connection and
+// return an error if the proxy or auth server is unavailable. So in order for
+// tbot to run in degraded mode, you must use `proxy_server` or `--proxy-server`.
+//
+// As tbot is also embedded in our Kubernetes operator and `tctl terraform env`
+// which both support providing an auth server or proxy address using the same
+// field, this package allows you to opt-in to the previous behavior by setting
+// Config.AuthServerAddressMode to AllowProxyAsAuthServer.
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"log/slog"
+
+	"github.com/gravitational/trace"
+	grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
+	"github.com/jonboulle/clockwork"
+	"golang.org/x/crypto/ssh"
+	"google.golang.org/grpc"
+
+	"github.com/gravitational/teleport"
+	"github.com/gravitational/teleport/api/breaker"
+	"github.com/gravitational/teleport/api/client"
+	"github.com/gravitational/teleport/api/metadata"
+	"github.com/gravitational/teleport/lib/reversetunnelclient"
+	"github.com/gravitational/teleport/lib/tbot/config"
+)
+
+// Address contains an address string tagged with whether it belongs to a proxy
+// or auth server.
+type Address struct {
+	// Addr contains the address string.
+	Addr string
+
+	// Kind of address (i.e. auth server or proxy).
+	Kind config.AddressKind
+}
+
+// String implements fmt.Stringer.
+func (a Address) String() string {
+	return fmt.Sprintf("%s: %s", a.Kind, a.Addr)
+}
+
+// Validate the address string and kind.
+func (a Address) Validate() error {
+	if a.Addr == "" {
+		return trace.BadParameter("address is required")
+	}
+
+	switch a.Kind {
+	case config.AddressKindProxy, config.AddressKindAuth:
+		return nil
+	default:
+		return trace.BadParameter("unsupported address type: %s", a.Kind)
+	}
+}
+
+// Identity provides the TLS and SSH credentials required to dial a connection.
+type Identity interface {
+	// TLSConfig returns the bot's TLS configuration.
+	TLSConfig() (*tls.Config, error)
+
+	// SSHClientConfig returns the bot's SSH client configuration.
+	SSHClientConfig() (*ssh.ClientConfig, error)
+}
+
+// Config contains options used to create the API client.
+type Config struct {
+	// Address that will be dialed to create the client connection.
+	Address Address
+
+	// AuthServerAddressMode controls the behavior when a proxy address is
+	// given as an auth server address.
+	AuthServerAddressMode config.AuthServerAddressMode
+
+	// Identity that will provide the TLS and SSH credentials.
+	Identity Identity
+
+	// Resolver that will be used to find the address of a proxy server.
+	Resolver reversetunnelclient.Resolver
+
+	// Logger to which log messages will be written.
+	Logger *slog.Logger
+
+	// Insecure controls whether we will skip TLS host verification.
+	Insecure bool
+
+	// Metrics will record gRPC client metrics.
+	Metrics *grpcprom.ClientMetrics
+}
+
+// CheckAndSetDefaults checks whether required config options have been provided
+// and sets defaults.
+func (c *Config) CheckAndSetDefaults() error {
+	if err := c.Address.Validate(); err != nil {
+		return err
+	}
+	if c.Identity == nil {
+		return trace.BadParameter("identity is required")
+	}
+	if c.Resolver == nil {
+		return trace.BadParameter("resolver is required")
+	}
+	if c.Logger == nil {
+		c.Logger = slog.Default()
+	}
+	return nil
+}
+
+// New creates an API client. See the package documentation for more information.
+func New(ctx context.Context, cfg Config) (*client.Client, error) {
+	if err := cfg.CheckAndSetDefaults(); err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	// If the address is known to be a proxy, dial without testing the
+	// connection.
+	if cfg.Address.Kind == config.AddressKindProxy {
+		return dialViaProxy(ctx, cfg)
+	}
+
+	// If it is known to be an auth server (i.e. we do not support falling back
+	// to treating it as a proxy), dial without testing the connection.
+	if cfg.AuthServerAddressMode == config.AuthServerMustBeAuthServer {
+		return dialDirectly(ctx, cfg)
+	}
+
+	// If the address is thought to be an auth server, try to dial it directly.
+	clt, directErr := dialDirectly(ctx, cfg)
+	if directErr == nil {
+		// Send a ping to test the connection.
+		if _, directErr = clt.Ping(ctx); directErr == nil {
+			return clt, nil
+		} else {
+			_ = clt.Close()
+		}
+	}
+
+	// If the direct connection failed, try to connect to it as a proxy.
+	clt, proxyErr := dialViaProxy(ctx, cfg)
+	if proxyErr == nil {
+		// Send a ping to test the connection.
+		if _, proxyErr = clt.Ping(ctx); proxyErr == nil {
+			if cfg.AuthServerAddressMode == config.WarnIfAuthServerIsProxy {
+				cfg.Logger.WarnContext(ctx,
+					"Support for providing a proxy address via the 'auth_server' configuration option or '--auth-server' flag is deprecated and will be removed in v19. Use 'proxy_server' or '--proxy-server' instead.",
+				)
+			}
+			return clt, nil
+		} else {
+			_ = clt.Close()
+		}
+	}
+
+	return nil, trace.NewAggregate(
+		trace.Wrap(directErr, "failed direct dial to auth server"),
+		trace.Wrap(proxyErr, "failed dial to auth server through reverse tunnel"),
+	)
+}
+
+func dialViaProxy(ctx context.Context, cfg Config) (*client.Client, error) {
+	tlsConfig, err := cfg.Identity.TLSConfig()
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	sshConfig, err := cfg.Identity.SSHClientConfig()
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	dialer, err := reversetunnelclient.NewTunnelAuthDialer(reversetunnelclient.TunnelAuthDialerConfig{
+		Resolver:              cfg.Resolver,
+		ClientConfig:          sshConfig,
+		Log:                   cfg.Logger,
+		InsecureSkipTLSVerify: cfg.Insecure,
+		GetClusterCAs:         client.ClusterCAsFromCertPool(tlsConfig.RootCAs),
+	})
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	return client.New(ctx, client.Config{
+		DialInBackground: true,
+		Credentials: []client.Credentials{
+			client.LoadTLS(tlsConfig),
+		},
+		Dialer:                   dialer,
+		DialOpts:                 dialOpts(cfg),
+		CircuitBreakerConfig:     circuitBreakerConfig(),
+		InsecureAddressDiscovery: cfg.Insecure,
+	})
+}
+
+func dialDirectly(ctx context.Context, cfg Config) (*client.Client, error) {
+	tlsConfig, err := cfg.Identity.TLSConfig()
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	return client.New(ctx, client.Config{
+		DialInBackground: true,
+		Addrs:            []string{cfg.Address.Addr},
+		Credentials: []client.Credentials{
+			client.LoadTLS(tlsConfig),
+		},
+		DialOpts:                 dialOpts(cfg),
+		CircuitBreakerConfig:     circuitBreakerConfig(),
+		InsecureAddressDiscovery: cfg.Insecure,
+	})
+}
+
+func dialOpts(cfg Config) []grpc.DialOption {
+	opts := []grpc.DialOption{
+		metadata.WithUserAgentFromTeleportComponent(teleport.ComponentTBot),
+	}
+	if cfg.Metrics != nil {
+		opts = append(opts,
+			grpc.WithChainUnaryInterceptor(cfg.Metrics.UnaryClientInterceptor()),
+			grpc.WithChainStreamInterceptor(cfg.Metrics.StreamClientInterceptor()),
+		)
+	}
+	return opts
+}
+
+func circuitBreakerConfig() breaker.Config {
+	cfg := breaker.DefaultBreakerConfig(clockwork.NewRealClock())
+	cfg.TrippedErrorMessage = "Unable to communicate with the Teleport Auth Service"
+	return cfg
+}

lib/tbot/config/config.go

@@ -175,12 +175,17 @@ type BotConfig struct {
 	Outputs  ServiceConfigs `yaml:"outputs,omitempty"`
 	Services ServiceConfigs `yaml:"services,omitempty"`
 
-	Debug      bool   `yaml:"debug"`
-	AuthServer string `yaml:"auth_server,omitempty"`
-	// ProxyServer is the teleport proxy address. Unlike `AuthServer` this must
-	// explicitly point to a Teleport proxy.
-	// Example: "example.teleport.sh:443"
-	ProxyServer        string             `yaml:"proxy_server,omitempty"`
+	Debug       bool   `yaml:"debug"`
+	AuthServer  string `yaml:"auth_server,omitempty"`
+	ProxyServer string `yaml:"proxy_server,omitempty"`
+
+	// AuthServerAddressMode controls whether it's permissible to provide a
+	// proxy server address as an auth server address. This is unsupported in
+	// the tbot binary as of v19, but we maintain support for cases where tbot
+	// is embedded in a binary which does not differentiate between address types
+	// such as tctl or the Kubernetes operator.
+	AuthServerAddressMode AuthServerAddressMode `yaml:"-"`
+
 	CredentialLifetime CredentialLifetime `yaml:",inline"`
 	Oneshot            bool               `yaml:"oneshot"`
 	// FIPS instructs `tbot` to run in a mode designed to comply with FIPS
@@ -227,6 +232,25 @@ func (conf *BotConfig) Address() (string, AddressKind) {
 	}
 }
 
+// AuthServerAddressMode controls the behavior when a proxy address is given
+// as an auth server address.
+type AuthServerAddressMode int
+
+const (
+	// AuthServerMustBeAuthServer means that only an actual auth server address
+	// may be given.
+	AuthServerMustBeAuthServer AuthServerAddressMode = iota
+
+	// WarnIfAuthServerIsProxy means that a proxy address will be accepted as an
+	// auth server address, but we will log a warning that this is going away in
+	// v19.
+	WarnIfAuthServerIsProxy
+
+	// AllowProxyAsAuthServer means that a proxy address will be accepted as an
+	// auth server address.
+	AllowProxyAsAuthServer
+)
+
 func (conf *BotConfig) CipherSuites() []uint16 {
 	if conf.FIPS {
 		return defaults.FIPSCipherSuites

lib/tbot/database_access.go

@@ -28,11 +28,10 @@ import (
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/defaults"
 	"github.com/gravitational/teleport/api/types"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	libdefaults "github.com/gravitational/teleport/lib/defaults"
 )
 
-func getDatabase(ctx context.Context, clt *authclient.Client, name string) (types.Database, error) {
+func getDatabase(ctx context.Context, clt *apiclient.Client, name string) (types.Database, error) {
 	ctx, span := tracer.Start(ctx, "getDatabase")
 	defer span.End()
 
@@ -59,7 +58,7 @@ func getDatabase(ctx context.Context, clt *authclient.Client, name string) (type
 func getRouteToDatabase(
 	ctx context.Context,
 	log *slog.Logger,
-	client *authclient.Client,
+	client *apiclient.Client,
 	service string,
 	username string,
 	database string,

lib/tbot/output_utils.go

@@ -32,6 +32,7 @@ import (
 	"github.com/gravitational/trace"
 	"golang.org/x/crypto/ssh"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/constants"
 	"github.com/gravitational/teleport/api/types"
@@ -288,7 +289,7 @@ type identityConfigurator = func(req *proto.UserCertsRequest)
 // certs.
 func generateIdentity(
 	ctx context.Context,
-	client *authclient.Client,
+	client *apiclient.Client,
 	currentIdentity *identity.Identity,
 	roles []string,
 	ttl time.Duration,

lib/tbot/service_application_output.go

@@ -30,7 +30,6 @@ import (
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/defaults"
 	"github.com/gravitational/teleport/api/types"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/tbot/config"
 	"github.com/gravitational/teleport/lib/tbot/identity"
@@ -39,7 +38,7 @@ import (
 // ApplicationOutputService generates the artifacts necessary to connect to a
 // HTTP or TCP application using Teleport.
 type ApplicationOutputService struct {
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.ApplicationOutput
 	getBotIdentity    getBotIdentityFn
@@ -210,7 +209,7 @@ func (s *ApplicationOutputService) render(
 func getRouteToApp(
 	ctx context.Context,
 	botIdentity *identity.Identity,
-	client *authclient.Client,
+	client *apiclient.Client,
 	appName string,
 ) (proto.RouteToApp, types.Application, error) {
 	ctx, span := tracer.Start(ctx, "getRouteToApp")
@@ -233,7 +232,7 @@ func getRouteToApp(
 	return routeToApp, app, nil
 }
 
-func getApp(ctx context.Context, clt *authclient.Client, appName string) (types.Application, error) {
+func getApp(ctx context.Context, clt *apiclient.Client, appName string) (types.Application, error) {
 	ctx, span := tracer.Start(ctx, "getApp")
 	defer span.End()
 

lib/tbot/service_application_tunnel.go

@@ -27,10 +27,9 @@ import (
 
 	"github.com/gravitational/trace"
 
-	"github.com/gravitational/teleport/api/client"
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/types"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/srv/alpnproxy"
 	"github.com/gravitational/teleport/lib/srv/alpnproxy/common"
@@ -49,7 +48,7 @@ type ApplicationTunnelService struct {
 	proxyPingCache *proxyPingCache
 	log            *slog.Logger
 	resolver       reversetunnelclient.Resolver
-	botClient      *authclient.Client
+	botClient      *apiclient.Client
 	getBotIdentity getBotIdentityFn
 }
 
@@ -172,7 +171,7 @@ func (s *ApplicationTunnelService) buildLocalProxyConfig(ctx context.Context) (l
 		Cert:               *appCert,
 		InsecureSkipVerify: s.botCfg.Insecure,
 	}
-	if client.IsALPNConnUpgradeRequired(
+	if apiclient.IsALPNConnUpgradeRequired(
 		ctx,
 		proxyAddr,
 		s.botCfg.Insecure,

lib/tbot/service_bot_identity.go

@@ -30,10 +30,11 @@ import (
 	"github.com/gravitational/trace"
 	"golang.org/x/crypto/ssh"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/utils/keys"
-	"github.com/gravitational/teleport/lib/auth/authclient"
+	"github.com/gravitational/teleport/api/utils/retryutils"
 	"github.com/gravitational/teleport/lib/auth/join"
 	"github.com/gravitational/teleport/lib/auth/join/boundkeypair"
 	"github.com/gravitational/teleport/lib/auth/state"
@@ -53,18 +54,17 @@ const botIdentityRenewalRetryLimit = 7
 // identityService is a [bot.Service] that handles renewing the bot's identity.
 // It renews the bot's identity periodically and when receiving a broadcasted
 // reload signal.
-//
-// It does not offer a [bot.OneShotService] implementation as the Bot's identity
-// is renewed automatically during initialization.
 type identityService struct {
 	log               *slog.Logger
 	reloadBroadcaster *channelBroadcaster
 	cfg               *config.BotConfig
 	resolver          reversetunnelclient.Resolver
 
-	mu     sync.Mutex
-	client *authclient.Client
-	facade *identity.Facade
+	mu              sync.Mutex
+	client          *apiclient.Client
+	facade          *identity.Facade
+	initialized     chan struct{}
+	initializedOnce sync.Once
 }
 
 // GetIdentity returns the current Bot identity.
@@ -76,12 +76,36 @@ func (s *identityService) GetIdentity() *identity.Identity {
 
 // GetClient returns the facaded client for the Bot identity for use by other
 // components of `tbot`. Consumers should not call `Close` on the client.
-func (s *identityService) GetClient() *authclient.Client {
+func (s *identityService) GetClient() *apiclient.Client {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	return s.client
 }
 
+// Ready returns a channel that will be closed when the initial identity renewal
+// process has completed. It provides a way to "block" startup of services that
+// cannot gracefully handle the API client being unavailable.
+func (s *identityService) Ready() <-chan struct{} {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if s.initialized == nil {
+		s.initialized = make(chan struct{})
+	}
+
+	return s.initialized
+}
+
+// IsReady returns whether the initial identity renewal process has completed.
+func (s *identityService) IsReady() bool {
+	select {
+	case <-s.Ready():
+		return true
+	default:
+		return false
+	}
+}
+
 // String returns a human-readable name of the service.
 func (s *identityService) String() string {
 	return "identity"
@@ -96,11 +120,11 @@ func hasTokenChanged(configTokenBytes, identityBytes []byte) bool {
 }
 
 // loadIdentityFromStore attempts to load a persisted identity from a store.
-// It then checks:
-// - This identity against the configured onboarding profile.
-// - This identity is not expired
-// If any checks fail, it will not return the loaded identity.
-func (s *identityService) loadIdentityFromStore(ctx context.Context, store bot.Destination) *identity.Identity {
+//
+// If the persisted identity does not match the onboarding profile/join token,
+// a nil identity will be returned. If the identity certificate has expired, the
+// bool return value will be false.
+func (s *identityService) loadIdentityFromStore(ctx context.Context, store bot.Destination) (*identity.Identity, bool) {
 	ctx, span := tracer.Start(ctx, "identityService/loadIdentityFromStore")
 	defer span.End()
 	s.log.InfoContext(ctx, "Loading existing bot identity from store", "store", store)
@@ -109,14 +133,14 @@ func (s *identityService) loadIdentityFromStore(ctx context.Context, store bot.D
 	if err != nil {
 		if trace.IsNotFound(err) {
 			s.log.InfoContext(ctx, "No existing bot identity found in store")
-			return nil
+			return nil, false
 		} else {
 			s.log.WarnContext(
 				ctx,
 				"Failed to load existing bot identity from store",
 				"error", err,
 			)
-			return nil
+			return nil, false
 		}
 	}
 
@@ -130,7 +154,7 @@ func (s *identityService) loadIdentityFromStore(ctx context.Context, store bot.D
 				s.log.InfoContext(ctx, "Bot identity loaded from store does not match configured token")
 				// If the token has changed, do not return the loaded
 				// identity.
-				return nil
+				return nil, false
 			}
 		} else {
 			// we failed to get the newly configured token to compare to,
@@ -151,25 +175,26 @@ func (s *identityService) loadIdentityFromStore(ctx context.Context, store bot.D
 	)
 
 	now := time.Now().UTC()
+	valid := true
 	if now.After(loadedIdent.X509Cert.NotAfter) {
+		valid = false
 		s.log.WarnContext(
 			ctx,
 			"Identity loaded from store is expired, it will not be used",
 			"not_after", loadedIdent.X509Cert.NotAfter.Format(time.RFC3339),
 			"current_time", now.Format(time.RFC3339),
 		)
-		return nil
 	} else if now.Before(loadedIdent.X509Cert.NotBefore) {
+		valid = false
 		s.log.WarnContext(
 			ctx,
 			"Identity loaded from store is not yet valid, it will not be used. Confirm that the system time is correct",
 			"not_before", loadedIdent.X509Cert.NotBefore.Format(time.RFC3339),
 			"current_time", now.Format(time.RFC3339),
 		)
-		return nil
 	}
 
-	return loadedIdent
+	return loadedIdent, valid
 }
 
 // Initialize sets up the bot identity at startup. This process has a few
@@ -188,13 +213,11 @@ func (s *identityService) Initialize(ctx context.Context) error {
 	defer span.End()
 
 	s.log.InfoContext(ctx, "Initializing bot identity")
-	// nil will be returned if no identity can be found in store or
-	// the identity in the store is no longer relevant or valid.
-	loadedIdent := s.loadIdentityFromStore(ctx, s.cfg.Storage.Destination)
-	if loadedIdent == nil {
+	loadedIdent, valid := s.loadIdentityFromStore(ctx, s.cfg.Storage.Destination)
+	if !valid {
 		if !s.cfg.Onboarding.HasToken() {
-			// There's no loaded identity to work with, and they've not
-			// configured  a token to use to request an identity :(
+			// If there's no pre-existing identity (or it has expired) and the
+			// configuration contains no join token, we cannot do anything.
 			return trace.BadParameter(
 				"no existing identity found on disk or join token configured",
 			)
@@ -205,29 +228,62 @@ func (s *identityService) Initialize(ctx context.Context) error {
 		)
 	}
 
-	var err error
-	var newIdentity *identity.Identity
-	if loadedIdent != nil {
-		newIdentity, err = renewIdentity(ctx, s.log, s.cfg, s.resolver, loadedIdent)
-		if err != nil {
-			return trace.Wrap(err, "renewing identity using loaded identity")
+	var (
+		newIdentity *identity.Identity
+		err         error
+	)
+	if loadedIdent == nil {
+		// If there was no identity already on-disk, or it did not match the
+		// onboarding profile / join token, try to join from scratch.
+		//
+		// If this fails, tbot will exit because we cannot proceed with no
+		// identity at all.
+		if newIdentity, err = botIdentityFromToken(ctx, s.log, s.cfg, nil); err != nil {
+			return trace.Wrap(err, "joining with token")
 		}
 	} else {
-		// TODO(noah): If the above renewal fails, do we want to try joining
-		// instead? Is there a sane amount of times to try renewing before
-		// giving up and rejoining?
-		newIdentity, err = botIdentityFromToken(ctx, s.log, s.cfg, nil)
+		if valid {
+			// If the identity is valid (not expired), try to renew it.
+			newIdentity, err = renewIdentity(ctx, s.log, s.cfg, s.resolver, loadedIdent)
+		} else {
+			// If the identity has expired, try to join again from scratch.
+			newIdentity, err = botIdentityFromToken(ctx, s.log, s.cfg, nil)
+		}
+
+		// If there was an identity on-disk from a previous run, but renewing it
+		// or re-joining fails, tbot will continue running using the (possibly
+		// expired) existing identity.
+		//
+		// In long-running mode, the Run method will retry the renewal process
+		// in case connectivity to the auth server has been restored etc. In the
+		// meantime, some services may be able to continue operating with cached
+		// data.
+		//
+		// In one-shot mode, the OneShot method will make a ping RPC to test the
+		// connection and exit immediately if the connection is unavailable.
 		if err != nil {
-			return trace.Wrap(err, "joining with token")
+			facade := identity.NewFacade(s.cfg.FIPS, s.cfg.Insecure, loadedIdent)
+			client, clientErr := clientForFacade(ctx, s.log, s.cfg, facade, s.resolver)
+			if clientErr != nil {
+				return trace.Wrap(clientErr)
+			}
+
+			s.mu.Lock()
+			s.facade = facade
+			s.client = client
+			s.mu.Unlock()
+
+			s.log.ErrorContext(ctx, "Failed to renew bot identity. Will attempt to proceed with the old identity, API calls may fail", "error", err)
+			return nil
 		}
 	}
 
+	// We successfully renewed the bot identity!
 	s.log.InfoContext(ctx, "Fetched new bot identity", "identity", describeTLSIdentity(ctx, s.log, newIdentity))
 	if err := identity.SaveIdentity(ctx, newIdentity, s.cfg.Storage.Destination, identity.BotKinds()...); err != nil {
 		return trace.Wrap(err)
 	}
 
-	// Create the facaded client we can share with other components of tbot.
 	facade := identity.NewFacade(s.cfg.FIPS, s.cfg.Insecure, newIdentity)
 	c, err := clientForFacade(ctx, s.log, s.cfg, facade, s.resolver)
 	if err != nil {
@@ -238,6 +294,8 @@ func (s *identityService) Initialize(ctx context.Context) error {
 	s.facade = facade
 	s.mu.Unlock()
 
+	s.unblockWaiters()
+
 	s.log.InfoContext(ctx, "Identity initialized successfully")
 	return nil
 }
@@ -250,6 +308,13 @@ func (s *identityService) Close() error {
 	return trace.Wrap(c.Close())
 }
 
+func (s *identityService) OneShot(ctx context.Context) error {
+	if _, err := s.GetClient().Ping(ctx); err != nil {
+		return trace.Wrap(err, "testing auth service connection")
+	}
+	return nil
+}
+
 func (s *identityService) Run(ctx context.Context) error {
 	ctx, span := tracer.Start(ctx, "identityService/Run")
 	defer span.End()
@@ -260,6 +325,35 @@ func (s *identityService) Run(ctx context.Context) error {
 	// etc)
 	storageDestination := s.cfg.Storage.Destination
 
+	// Keep retrying renewal if it failed on startup.
+	if !s.IsReady() {
+		retry, err := retryutils.NewRetryV2(retryutils.RetryV2Config{
+			Driver: retryutils.NewExponentialDriver(1 * time.Second),
+			Max:    1 * time.Minute,
+			Jitter: retryutils.HalfJitter,
+		})
+		if err != nil {
+			return trace.Wrap(err, "creating retry")
+		}
+
+		for {
+			retry.Inc()
+
+			s.log.InfoContext(ctx, "Unable to renew bot identity on startup. Waiting to retry", "wait", retry.Duration())
+
+			select {
+			case <-retry.After():
+			case <-ctx.Done():
+				return nil
+			}
+
+			if err := s.renew(ctx, storageDestination); err == nil {
+				s.unblockWaiters()
+				break
+			}
+		}
+	}
+
 	s.log.InfoContext(
 		ctx,
 		"Beginning bot identity renewal loop",
@@ -273,12 +367,11 @@ func (s *identityService) Run(ctx context.Context) error {
 		f: func(ctx context.Context) error {
 			return s.renew(ctx, storageDestination)
 		},
-		interval:             s.cfg.CredentialLifetime.RenewalInterval,
-		exitOnRetryExhausted: true,
-		retryLimit:           botIdentityRenewalRetryLimit,
-		log:                  s.log,
-		reloadCh:             reloadCh,
-		waitBeforeFirstRun:   true,
+		interval:           s.cfg.CredentialLifetime.RenewalInterval,
+		retryLimit:         botIdentityRenewalRetryLimit,
+		log:                s.log,
+		reloadCh:           reloadCh,
+		waitBeforeFirstRun: true,
 	})
 	return trace.Wrap(err)
 }
@@ -312,6 +405,17 @@ func (s *identityService) renew(
 	return nil
 }
 
+func (s *identityService) unblockWaiters() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if s.initialized == nil {
+		s.initialized = make(chan struct{})
+	}
+
+	s.initializedOnce.Do(func() { close(s.initialized) })
+}
+
 func renewIdentity(
 	ctx context.Context,
 	log *slog.Logger,
@@ -356,7 +460,7 @@ func botIdentityFromAuth(
 	ctx context.Context,
 	log *slog.Logger,
 	ident *identity.Identity,
-	client *authclient.Client,
+	client *apiclient.Client,
 	ttl time.Duration,
 ) (*identity.Identity, error) {
 	ctx, span := tracer.Start(ctx, "botIdentityFromAuth")
@@ -425,7 +529,7 @@ func botIdentityFromToken(
 	ctx context.Context,
 	log *slog.Logger,
 	cfg *config.BotConfig,
-	authClient *authclient.Client,
+	authClient *apiclient.Client,
 ) (*identity.Identity, error) {
 	_, span := tracer.Start(ctx, "botIdentityFromToken")
 	defer span.End()

lib/tbot/service_ca_rotation.go

@@ -31,9 +31,9 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/utils/retryutils"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 )
 
 // debouncer accepts a duration, and a function. When `attempt` is called on
@@ -129,7 +129,7 @@ const caRotationRetryBackoff = time.Second * 2
 type caRotationService struct {
 	log               *slog.Logger
 	reloadBroadcaster *channelBroadcaster
-	botClient         *authclient.Client
+	botClient         *apiclient.Client
 	getBotIdentity    getBotIdentityFn
 }
 

lib/tbot/service_client_credential.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/gravitational/trace"
 
-	"github.com/gravitational/teleport/lib/auth/authclient"
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/lib/tbot/config"
 )
 
@@ -34,7 +34,7 @@ type ClientCredentialOutputService struct {
 	// botAuthClient should be an auth client using the bots internal identity.
 	// This will not have any roles impersonated and should only be used to
 	// fetch CAs.
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.UnstableClientCredentialOutput
 	getBotIdentity    getBotIdentityFn

lib/tbot/service_database_output.go

@@ -26,9 +26,9 @@ import (
 
 	"github.com/gravitational/trace"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/types"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/client/identityfile"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/tbot/bot"
@@ -39,7 +39,7 @@ import (
 // DatabaseOutputService generates the artifacts necessary to connect to a
 // database using Teleport.
 type DatabaseOutputService struct {
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.DatabaseOutput
 	getBotIdentity    getBotIdentityFn

lib/tbot/service_database_tunnel.go

@@ -27,9 +27,8 @@ import (
 
 	"github.com/gravitational/trace"
 
-	"github.com/gravitational/teleport/api/client"
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/client/proto"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/srv/alpnproxy"
 	"github.com/gravitational/teleport/lib/srv/alpnproxy/common"
@@ -69,7 +68,7 @@ type DatabaseTunnelService struct {
 	proxyPingCache *proxyPingCache
 	log            *slog.Logger
 	resolver       reversetunnelclient.Resolver
-	botClient      *authclient.Client
+	botClient      *apiclient.Client
 	getBotIdentity getBotIdentityFn
 }
 
@@ -162,7 +161,7 @@ func (s *DatabaseTunnelService) buildLocalProxyConfig(ctx context.Context) (lpCf
 		Cert:               *dbCert,
 		InsecureSkipVerify: s.botCfg.Insecure,
 	}
-	if client.IsALPNConnUpgradeRequired(
+	if apiclient.IsALPNConnUpgradeRequired(
 		ctx,
 		proxyAddr,
 		s.botCfg.Insecure,

lib/tbot/service_identity_output.go

@@ -29,9 +29,9 @@ import (
 
 	"github.com/gravitational/trace"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/types"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	autoupdate "github.com/gravitational/teleport/lib/autoupdate/agent"
 	"github.com/gravitational/teleport/lib/config/openssh"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
@@ -48,7 +48,7 @@ type IdentityOutputService struct {
 	// botAuthClient should be an auth client using the bots internal identity.
 	// This will not have any roles impersonated and should only be used to
 	// fetch CAs.
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.IdentityOutput
 	getBotIdentity    getBotIdentityFn
@@ -412,7 +412,7 @@ func renderSSHConfig(
 }
 
 func getClusterNames(
-	ctx context.Context, client *authclient.Client, connectedClusterName string,
+	ctx context.Context, client *apiclient.Client, connectedClusterName string,
 ) ([]string, error) {
 	allClusterNames := []string{connectedClusterName}
 

lib/tbot/service_kubernetes_output.go

@@ -36,7 +36,6 @@ import (
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/defaults"
 	"github.com/gravitational/teleport/api/types"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	autoupdate "github.com/gravitational/teleport/lib/autoupdate/agent"
 	"github.com/gravitational/teleport/lib/client"
 	"github.com/gravitational/teleport/lib/kube/kubeconfig"
@@ -54,7 +53,7 @@ type KubernetesOutputService struct {
 	// botAuthClient should be an auth client using the bots internal identity.
 	// This will not have any roles impersonated and should only be used to
 	// fetch CAs.
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.KubernetesOutput
 	getBotIdentity    getBotIdentityFn

lib/tbot/service_kubernetes_v2_output.go

@@ -37,7 +37,6 @@ import (
 	"github.com/gravitational/teleport/api/defaults"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/utils"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	autoupdate "github.com/gravitational/teleport/lib/autoupdate/agent"
 	"github.com/gravitational/teleport/lib/client"
 	"github.com/gravitational/teleport/lib/kube/kubeconfig"
@@ -53,7 +52,7 @@ type KubernetesV2OutputService struct {
 	// botAuthClient should be an auth client using the bots internal identity.
 	// This will not have any roles impersonated and should only be used to
 	// fetch CAs.
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.KubernetesV2Output
 	getBotIdentity    getBotIdentityFn

lib/tbot/service_spiffe_svid_output.go

@@ -32,9 +32,9 @@ import (
 	"github.com/gravitational/trace"
 	"google.golang.org/protobuf/types/known/durationpb"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1"
 	"github.com/gravitational/teleport/api/utils/retryutils"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/cryptosuites"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/tbot/config"
@@ -53,7 +53,7 @@ const (
 // SVIDs to a destination. It produces an output compatible with the
 // `spiffe-helper` tool.
 type SPIFFESVIDOutputService struct {
-	botAuthClient  *authclient.Client
+	botAuthClient  *apiclient.Client
 	botCfg         *config.BotConfig
 	cfg            *config.SPIFFESVIDOutput
 	getBotIdentity getBotIdentityFn
@@ -311,7 +311,7 @@ func (s *SPIFFESVIDOutputService) render(
 
 func generateJWTSVIDs(
 	ctx context.Context,
-	clt *authclient.Client,
+	clt *apiclient.Client,
 	svid config.SVIDRequest,
 	reqs []config.JWTSVID,
 	ttl time.Duration,
@@ -363,7 +363,7 @@ func generateJWTSVIDs(
 // call.
 func generateSVID(
 	ctx context.Context,
-	clt *authclient.Client,
+	clt *apiclient.Client,
 	reqs []config.SVIDRequest,
 	ttl time.Duration,
 ) (*machineidv1pb.SignX509SVIDsResponse, crypto.Signer, error) {

lib/tbot/service_spiffe_workload_api.go

@@ -51,8 +51,8 @@ import (
 	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/gravitational/teleport"
+	apiclient "github.com/gravitational/teleport/api/client"
 	machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/observability/metrics"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
@@ -83,7 +83,7 @@ type SPIFFEWorkloadAPIService struct {
 	trustBundleCache *workloadidentity.TrustBundleCache
 
 	// client holds the impersonated client for the service
-	client           *authclient.Client
+	client           *apiclient.Client
 	attestor         *workloadattest.Attestor
 	localTrustDomain spiffeid.TrustDomain
 }

lib/tbot/service_ssh_host_output.go

@@ -28,10 +28,10 @@ import (
 	"github.com/gravitational/trace"
 	"google.golang.org/protobuf/types/known/durationpb"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	trustpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/trust/v1"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/utils/keys"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/client"
 	"github.com/gravitational/teleport/lib/client/identityfile"
 	"github.com/gravitational/teleport/lib/cryptosuites"
@@ -42,7 +42,7 @@ import (
 )
 
 type SSHHostOutputService struct {
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.SSHHostOutput
 	getBotIdentity    getBotIdentityFn

lib/tbot/service_ssh_multiplexer.go

@@ -44,10 +44,10 @@ import (
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	proxyclient "github.com/gravitational/teleport/api/client/proxy"
 	"github.com/gravitational/teleport/api/observability/tracing"
 	"github.com/gravitational/teleport/api/utils/grpc/interceptors"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	autoupdate "github.com/gravitational/teleport/lib/autoupdate/agent"
 	libclient "github.com/gravitational/teleport/lib/client"
 	"github.com/gravitational/teleport/lib/config/openssh"
@@ -96,7 +96,7 @@ type SSHMultiplexerService struct {
 	// botAuthClient should be an auth client using the bots internal identity.
 	// This will not have any roles impersonated and should only be used to
 	// fetch CAs.
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.SSHMultiplexerService
 	getBotIdentity    getBotIdentityFn
@@ -150,7 +150,7 @@ func writeIfChanged(ctx context.Context, dest bot.Destination, log *slog.Logger,
 func (s *SSHMultiplexerService) writeArtifacts(
 	ctx context.Context,
 	proxyHost string,
-	authClient *authclient.Client,
+	authClient *apiclient.Client,
 ) error {
 	dest := s.cfg.Destination.(*config.DestinationDirectory)
 
@@ -218,7 +218,7 @@ func (s *SSHMultiplexerService) writeArtifacts(
 }
 
 func (s *SSHMultiplexerService) setup(ctx context.Context) (
-	_ *authclient.Client,
+	_ *apiclient.Client,
 	_ *cyclingHostDialClient,
 	proxyHost string,
 	_ *libclient.TSHConfig,
@@ -389,7 +389,7 @@ func (s *SSHMultiplexerService) generateIdentity(ctx context.Context) (*identity
 }
 
 func (s *SSHMultiplexerService) identityRenewalLoop(
-	ctx context.Context, proxyHost string, authClient *authclient.Client,
+	ctx context.Context, proxyHost string, authClient *apiclient.Client,
 ) error {
 	reloadCh, unsubscribe := s.reloadBroadcaster.subscribe()
 	defer unsubscribe()
@@ -556,7 +556,7 @@ func (s *SSHMultiplexerService) Run(ctx context.Context) (err error) {
 func (s *SSHMultiplexerService) handleConn(
 	ctx context.Context,
 	tshConfig *libclient.TSHConfig,
-	authClient *authclient.Client,
+	authClient *apiclient.Client,
 	hostDialer *cyclingHostDialClient,
 	proxyHost string,
 	downstream net.Conn,
@@ -675,7 +675,7 @@ func (s *SSHMultiplexerService) handleConn(
 		host = cleanTargetHost(host, proxyHost, clusterName)
 		target = net.JoinHostPort(host, port)
 	} else {
-		node, err := resolveTargetHostWithClient(ctx, authClient.APIClient, expanded.Search, expanded.Query)
+		node, err := resolveTargetHostWithClient(ctx, authClient, expanded.Search, expanded.Query)
 		if err != nil {
 			return trace.Wrap(err, "resolving target host")
 		}

lib/tbot/service_workload_identity_api.go

@@ -43,7 +43,7 @@ import (
 	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/gravitational/teleport"
-	"github.com/gravitational/teleport/lib/auth/authclient"
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/observability/metrics"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
@@ -75,7 +75,7 @@ type WorkloadIdentityAPIService struct {
 	crlCache         *workloadidentity.CRLCache
 
 	// client holds the impersonated client for the service
-	client           *authclient.Client
+	client           *apiclient.Client
 	attestor         *workloadattest.Attestor
 	localTrustDomain spiffeid.TrustDomain
 }

lib/tbot/service_workload_identity_aws_ra.go

@@ -32,8 +32,8 @@ import (
 	"github.com/spiffe/go-spiffe/v2/svid/x509svid"
 	"gopkg.in/ini.v1"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	workloadidentityv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/workloadidentity/v1"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/tbot/bot"
 	"github.com/gravitational/teleport/lib/tbot/config"
@@ -44,7 +44,7 @@ import (
 // WorkloadIdentityAWSRAService is a service that retrieves X.509 certificates
 // and exchanges them for AWS credentials using the AWS Roles Anywhere service.
 type WorkloadIdentityAWSRAService struct {
-	botAuthClient     *authclient.Client
+	botAuthClient     *apiclient.Client
 	botCfg            *config.BotConfig
 	cfg               *config.WorkloadIdentityAWSRAService
 	getBotIdentity    getBotIdentityFn

lib/tbot/service_workload_identity_jwt.go

@@ -26,9 +26,9 @@ import (
 
 	"github.com/gravitational/trace"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	workloadidentityv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/workloadidentity/v1"
 	"github.com/gravitational/teleport/api/utils/retryutils"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/tbot/config"
 	"github.com/gravitational/teleport/lib/tbot/identity"
@@ -38,7 +38,7 @@ import (
 // WorkloadIdentityJWTService is a service that retrieves JWT workload identity
 // credentials for WorkloadIdentity resources.
 type WorkloadIdentityJWTService struct {
-	botAuthClient  *authclient.Client
+	botAuthClient  *apiclient.Client
 	botCfg         *config.BotConfig
 	cfg            *config.WorkloadIdentityJWTService
 	getBotIdentity getBotIdentityFn

lib/tbot/service_workload_identity_x509.go

@@ -30,9 +30,9 @@ import (
 
 	"github.com/gravitational/trace"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	workloadidentityv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/workloadidentity/v1"
 	"github.com/gravitational/teleport/api/utils/retryutils"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
 	"github.com/gravitational/teleport/lib/tbot/config"
 	"github.com/gravitational/teleport/lib/tbot/identity"
@@ -42,7 +42,7 @@ import (
 // WorkloadIdentityX509Service is a service that retrieves X.509 certificates
 // for WorkloadIdentity resources.
 type WorkloadIdentityX509Service struct {
-	botAuthClient  *authclient.Client
+	botAuthClient  *apiclient.Client
 	botCfg         *config.BotConfig
 	cfg            *config.WorkloadIdentityX509Service
 	getBotIdentity getBotIdentityFn

lib/tbot/tbot.go

@@ -33,21 +33,19 @@ import (
 	"go.opentelemetry.io/otel"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/singleflight"
-	"google.golang.org/grpc"
 
 	"github.com/gravitational/teleport"
-	"github.com/gravitational/teleport/api/client"
+	apiclient "github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/client/webclient"
 	machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1"
-	"github.com/gravitational/teleport/api/metadata"
 	apitracing "github.com/gravitational/teleport/api/observability/tracing"
 	"github.com/gravitational/teleport/api/types"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	autoupdate "github.com/gravitational/teleport/lib/autoupdate/agent"
 	"github.com/gravitational/teleport/lib/modules"
 	"github.com/gravitational/teleport/lib/observability/metrics"
 	"github.com/gravitational/teleport/lib/reversetunnelclient"
+	"github.com/gravitational/teleport/lib/tbot/client"
 	"github.com/gravitational/teleport/lib/tbot/config"
 	"github.com/gravitational/teleport/lib/tbot/identity"
 	"github.com/gravitational/teleport/lib/tbot/workloadidentity"
@@ -125,9 +123,21 @@ type getBotIdentityFn func() *identity.Identity
 // BotIdentity returns the bot's own identity. This will return nil if the bot
 // has not been started.
 func (b *Bot) BotIdentity() *identity.Identity {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
 	return b.botIdentitySvc.GetIdentity()
 }
 
+// Client returns the bot's API client. This will return nil if the bot has not
+// been started.
+func (b *Bot) Client() *apiclient.Client {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	return b.botIdentitySvc.GetClient()
+}
+
 func (b *Bot) Run(ctx context.Context) (err error) {
 	ctx, span := tracer.Start(ctx, "Bot/Run")
 	defer func() { apitracing.EndSpan(span, err) }()
@@ -216,6 +226,7 @@ func (b *Bot) Run(ctx context.Context) (err error) {
 		}()
 	}
 
+	b.mu.Lock()
 	b.botIdentitySvc = &identityService{
 		cfg:               b.cfg,
 		reloadBroadcaster: reloadBroadcaster,
@@ -224,6 +235,8 @@ func (b *Bot) Run(ctx context.Context) (err error) {
 			teleport.ComponentKey, teleport.Component(componentTBot, "identity"),
 		),
 	}
+	b.mu.Unlock()
+
 	// Initialize bot's own identity. This will load from disk, or fetch a new
 	// identity, and perform an initial renewal if necessary.
 	if err := b.botIdentitySvc.Initialize(ctx); err != nil {
@@ -692,9 +705,6 @@ func (b *Bot) preRunChecks(ctx context.Context) (_ func() error, err error) {
 		return nil, trace.BadParameter(
 			"either a proxy or auth address must be set using --proxy-server, --auth-server or configuration",
 		)
-	case config.AddressKindAuth:
-		// TODO(noah): DELETE IN V17.0.0
-		b.log.WarnContext(ctx, "We recently introduced the ability to explicitly configure the address of the Teleport Proxy using --proxy-server. We recommend switching to this if you currently provide the address of the Proxy to --auth-server.")
 	}
 
 	// Ensure they have provided a join method.
@@ -779,58 +789,27 @@ func clientForFacade(
 	log *slog.Logger,
 	cfg *config.BotConfig,
 	facade *identity.Facade,
-	resolver reversetunnelclient.Resolver) (_ *authclient.Client, err error) {
+	resolver reversetunnelclient.Resolver) (_ *apiclient.Client, err error) {
 	ctx, span := tracer.Start(ctx, "clientForFacade")
 	defer func() { apitracing.EndSpan(span, err) }()
 
-	tlsConfig, err := facade.TLSConfig()
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-	sshConfig, err := facade.SSHClientConfig()
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-
-	addr, _ := cfg.Address()
-	parsedAddr, err := utils.ParseAddr(addr)
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-
-	dialer, err := reversetunnelclient.NewTunnelAuthDialer(reversetunnelclient.TunnelAuthDialerConfig{
-		Resolver:              resolver,
-		ClientConfig:          sshConfig,
-		Log:                   log,
-		InsecureSkipTLSVerify: cfg.Insecure,
-		GetClusterCAs:         client.ClusterCAsFromCertPool(tlsConfig.RootCAs),
-	})
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-
-	authClientConfig := &authclient.Config{
-		TLS: tlsConfig,
-		SSH: sshConfig,
-		// TODO(noah): It'd be ideal to distinguish the proxy addr and auth addr
-		// here to avoid pointlessly hitting the address as an auth server.
-		AuthServers: []utils.NetAddr{*parsedAddr},
-		Log:         log,
-		Insecure:    cfg.Insecure,
-		ProxyDialer: dialer,
-		DialOpts: []grpc.DialOption{
-			metadata.WithUserAgentFromTeleportComponent(teleport.ComponentTBot),
-			grpc.WithChainUnaryInterceptor(clientMetrics.UnaryClientInterceptor()),
-			grpc.WithChainStreamInterceptor(clientMetrics.StreamClientInterceptor()),
+	addr, kind := cfg.Address()
+	return client.New(ctx, client.Config{
+		Address: client.Address{
+			Addr: addr,
+			Kind: kind,
 		},
-	}
-
-	c, err := authclient.Connect(ctx, authClientConfig)
-	return c, trace.Wrap(err)
+		AuthServerAddressMode: cfg.AuthServerAddressMode,
+		Identity:              facade,
+		Resolver:              resolver,
+		Logger:                log,
+		Insecure:              cfg.Insecure,
+		Metrics:               clientMetrics,
+	})
 }
 
 type authPingCache struct {
-	client *authclient.Client
+	client *apiclient.Client
 	log    *slog.Logger
 
 	mu          sync.RWMutex
@@ -1001,7 +980,7 @@ func (a *alpnProxyConnUpgradeRequiredCache) isUpgradeRequired(ctx context.Contex
 		// Ok, now we know for sure that the work hasn't already been done or
 		// isn't in flight, we can complete it.
 		a.log.DebugContext(ctx, "Testing ALPN upgrade necessary", "addr", addr, "insecure", insecure)
-		v = client.IsALPNConnUpgradeRequired(ctx, addr, insecure)
+		v = apiclient.IsALPNConnUpgradeRequired(ctx, addr, insecure)
 		a.log.DebugContext(ctx, "Tested ALPN upgrade necessary", "addr", addr, "insecure", insecure, "result", v)
 		if err := ctx.Err(); err != nil {
 			// Check for case where false is returned because client canceled ctx.

lib/tbot/tbot_test.go

@@ -24,7 +24,9 @@ import (
 	"crypto/ecdsa"
 	"crypto/rand"
 	"crypto/rsa"
+	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net"
 	"os"
@@ -32,6 +34,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -42,6 +45,7 @@ import (
 	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/crypto/ssh/knownhosts"
 
+	"github.com/gravitational/teleport/api/client"
 	headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1"
 	machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1"
 	"github.com/gravitational/teleport/api/types"
@@ -153,8 +157,9 @@ func defaultBotConfig(
 	}
 
 	cfg := &config.BotConfig{
-		AuthServer: authServer,
-		Onboarding: *onboarding,
+		AuthServer:            authServer,
+		AuthServerAddressMode: config.WarnIfAuthServerIsProxy,
+		Onboarding:            *onboarding,
 		Storage: &config.StorageConfig{
 			Destination: &config.DestinationMemory{},
 		},
@@ -592,6 +597,234 @@ func TestBot_ResumeFromStorage(t *testing.T) {
 	require.NoError(t, thirdBot.Run(ctx))
 }
 
+func TestBot_IdentityRenewalFails(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	log := utils.NewSlogLoggerForTests()
+
+	// This test asserts that we can continue running (and recover) even when
+	// identity renewal fails on-startup.
+	//
+	// How it works:
+	//
+	// We run a TCP proxy in front of the real Teleport proxy that can be set to
+	// immediately close connections.
+	//
+	// We publish that address in TunnelPublicAddrs but put the *real* address
+	// in tbot's config so that we allow requests to the `/webapi/find` endpoint
+	// through as-is, without needing to sniff the ALPN protocols or something.
+	//
+	// This is necessary because tbot's resolver caches errors, so if we blocked
+	// the connection to that HTTP endpoint, we'd need to wait for the cache to
+	// expire.
+	proxy := newFailureProxy(t)
+
+	// Make a new auth server.
+	process := testenv.MakeTestServer(t,
+		func(o *testenv.TestServersOpts) {
+			defaultTestServerOpts(t, log)(o)
+
+			testenv.WithConfig(func(cfg *servicecfg.Config) {
+				cfg.Proxy.TunnelPublicAddrs = []utils.NetAddr{*utils.MustParseAddr(proxy.addr())}
+			})(o)
+		},
+	)
+	rootClient := testenv.MakeDefaultAuthClient(t, process)
+
+	// Create bot user and join token
+	botParams, _ := makeBot(t, rootClient, "test", "access")
+	botConfig := defaultBotConfig(t,
+		process,
+		botParams,
+		config.ServiceConfigs{},
+		defaultBotConfigOpts{insecure: true},
+	)
+
+	dest := &config.DestinationDirectory{
+		Path:     t.TempDir(),
+		Symlinks: botfs.SymlinksInsecure,
+		ACLs:     botfs.ACLOff,
+	}
+	botConfig.Storage.Destination = dest
+
+	// Configure our failure proxy to send traffic to the real proxy
+	tunnelAddr, err := process.ProxyTunnelAddr()
+	require.NoError(t, err)
+	proxy.dst = tunnelAddr.String()
+	go proxy.run(t)
+
+	botConfig.ProxyServer = tunnelAddr.String()
+	botConfig.AuthServer = ""
+
+	// Run the bot a first time
+	firstBot := New(botConfig, log)
+	require.NoError(t, firstBot.Run(ctx))
+
+	// Block connections. Running the bot should now fail.
+	proxy.setFailing(true)
+	secondBot := New(botConfig, log)
+	require.ErrorContains(t, secondBot.Run(ctx), "Error while dialing")
+
+	// Drain the notification channel so we can listen for new connections.
+	select {
+	case <-proxy.notif:
+	default:
+	}
+
+	// Run it again in long-running mode, and it should eventually succeed once
+	// the network partition has healed.
+	botConfig.Oneshot = false
+	outputDest := newWriteNotifier(&config.DestinationMemory{})
+	require.NoError(t, outputDest.CheckAndSetDefaults())
+	botConfig.Services = append(botConfig.Services, &config.IdentityOutput{
+		Destination: outputDest,
+	})
+	thirdBot := New(botConfig, log)
+
+	ctx, cancel := context.WithCancel(ctx)
+	t.Cleanup(cancel)
+	go thirdBot.Run(ctx)
+
+	// Wait for at least one failed connection, then heal the network partition.
+	select {
+	case <-proxy.notif:
+	case <-time.After(5 * time.Second):
+		t.Fatal("timeout waiting for connection")
+	}
+
+	// Wait for the client to be available.
+	var client *client.Client
+	require.Eventually(t, func() bool {
+		client = thirdBot.Client()
+		return client != nil
+	}, 5*time.Second, 100*time.Millisecond, "timeout waiting for client to become available")
+
+	t.Log("Healing network partition")
+	proxy.setFailing(false)
+
+	// We must reset gRPC's connection backoff, otherwise it'll keep returning
+	// the cached error.
+	client.GetConnection().ResetConnectBackoff()
+
+	// Wait for the destination to be written to.
+	select {
+	case <-outputDest.ch:
+	case <-time.After(5 * time.Second):
+		t.Fatal("timeout waiting for output to be written")
+	}
+}
+
+func newWriteNotifier(dst bot.Destination) *writeNotifier {
+	return &writeNotifier{
+		Destination: dst,
+		ch:          make(chan struct{}, 1),
+	}
+}
+
+type writeNotifier struct {
+	bot.Destination
+
+	ch chan struct{}
+}
+
+func (w writeNotifier) Write(ctx context.Context, name string, data []byte) error {
+	if name != identity.WriteTestKey {
+		defer func() {
+			select {
+			case w.ch <- struct{}{}:
+			default:
+			}
+		}()
+	}
+	return w.Destination.Write(ctx, name, data)
+}
+
+type failureProxy struct {
+	dst     string
+	lis     net.Listener
+	failing atomic.Bool
+	notif   chan struct{}
+}
+
+func newFailureProxy(t *testing.T) *failureProxy {
+	t.Helper()
+
+	lis, err := net.Listen("tcp", ":0")
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		if err := lis.Close(); err != nil {
+			t.Logf("failed to close listener: %v", err)
+		}
+	})
+
+	return &failureProxy{
+		lis:   lis,
+		notif: make(chan struct{}, 1),
+	}
+}
+
+func (f *failureProxy) run(t *testing.T) {
+	t.Helper()
+
+	for {
+		conn, err := f.lis.Accept()
+		if errors.Is(err, net.ErrClosed) {
+			return
+		}
+		if err != nil {
+			t.Logf("accept failed: %v", err)
+			continue
+		}
+
+		select {
+		case f.notif <- struct{}{}:
+		default:
+		}
+
+		go f.handleConn(t, conn)
+	}
+}
+
+func (f *failureProxy) handleConn(t *testing.T, conn net.Conn) {
+	defer func() {
+		if err := conn.Close(); err != nil {
+			t.Logf("failed to close connection: %v", err)
+		}
+	}()
+
+	// If we're failing, just close the connection immediately.
+	if f.failing.Load() {
+		return
+	}
+
+	upstream, err := net.Dial("tcp", f.dst)
+	if err != nil {
+		t.Logf("failed to dial upstream: %v", err)
+		return
+	}
+
+	done := make(chan struct{}, 1)
+	pipe := func(dst io.Writer, src io.Reader) {
+		_, _ = io.Copy(dst, src)
+		done <- struct{}{}
+	}
+
+	go pipe(upstream, conn)
+	go pipe(conn, upstream)
+
+	<-done
+}
+
+func (f *failureProxy) addr() string {
+	return f.lis.Addr().String()
+}
+
+func (f *failureProxy) setFailing(failing bool) {
+	f.failing.Store(failing)
+}
+
 func TestBot_InsecureViaProxy(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()

lib/tbot/workloadidentity/issue.go

@@ -26,8 +26,8 @@ import (
 	"github.com/gravitational/trace"
 	"google.golang.org/protobuf/types/known/durationpb"
 
+	apiclient "github.com/gravitational/teleport/api/client"
 	workloadidentityv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/workloadidentity/v1"
-	"github.com/gravitational/teleport/lib/auth/authclient"
 	"github.com/gravitational/teleport/lib/cryptosuites"
 	"github.com/gravitational/teleport/lib/tbot/config"
 	"github.com/gravitational/teleport/lib/tbot/workloadidentity/attrs"
@@ -182,7 +182,7 @@ func labelsToSelectors(in map[string][]string) []*workloadidentityv1pb.LabelSele
 func IssueJWTWorkloadIdentity(
 	ctx context.Context,
 	log *slog.Logger,
-	clt *authclient.Client,
+	clt *apiclient.Client,
 	workloadIdentity config.WorkloadIdentitySelector,
 	audiences []string,
 	ttl time.Duration,

tool/tctl/common/terraform_command.go

@@ -314,9 +314,10 @@ func (c *TerraformCommand) useBotToObtainIdentity(ctx context.Context, addr util
 		Insecure: clt.Config().InsecureSkipVerify,
 	}
 
-	// When invoked only with auth address, tbot will try both joining as an auth and as a proxy.
+	// When setting AuthServerAddressMode to ProxyAllowed, tbot will try both joining as an auth and as a proxy.
 	// This allows us to not care about how the user connects to Teleport (auth vs proxy joining).
 	cfg.AuthServer = addr.String()
+	cfg.AuthServerAddressMode = config.AllowProxyAsAuthServer
 
 	// Insecure joining is not compatible with CA pinning
 	if !cfg.Insecure {