Allow ignoring errors in tctl workload-identity x509-issuer-overrides sign-csrs (#57600)

docs/pages/reference/workload-identity/issuer-override.mdx

@@ -58,6 +58,8 @@ SERIALNUMBER=234567890123456789012345678901234567890,CN=clustername,O=clusternam
 
 Use of this command requires `create` permissions for the `workload_identity_x509_issuer_override_csr` resource kind in one of the roles associated with the identity running the command.
 
+In clusters that make use of Hardware Security Modules (HSMs) it's possible that no single Teleport Auth Service instance is capable of generating signatures for all the keys that make up the SPIFFE certificate authority at once. In such situations, it's possible to use the `--force` option with the `sign-csrs` command on each machine running the Auth Service, to gather CSRs for keys managed by the different HSMs.
+
 ## Using `tctl` to create a `workload_identity_x509_issuer_override` from certificate chain PEM files
 
 The `tctl workload-identity x509-issuer-overrides create` command can be used to build a `workload_identity_x509_issuer_override` resource out of one or more PEM files containing a certificate chain each, and to create or forcibly overwrite an existing resource in the cluster. The command will check that the first certificates in the specified chains have different public keys, and that they match 1:1 with the trusted X.509 certificates in the SPIFFE certificate authority in the Teleport cluster.

tool/tctl/common/workload_identity_command.go

@@ -68,8 +68,9 @@ type WorkloadIdentityCommand struct {
 	revocationReason string
 	revocationExpiry string
 
-	overridesSignCmd  *kingpin.CmdClause
-	overridesSignMode workloadidentityv1pb.CSRCreationMode
+	overridesSignCmd   *kingpin.CmdClause
+	overridesSignMode  workloadidentityv1pb.CSRCreationMode
+	overridesSignForce bool
 
 	overridesCreateCmd        *kingpin.CmdClause
 	overridesCreateName       string
@@ -175,6 +176,10 @@ func (c *WorkloadIdentityCommand) Initialize(
 		)).
 		StringVar(&overridesSignMode)
 	c.overridesSignMode = workloadidentityv1pb.CSRCreationMode_CSR_CREATION_MODE_SAME
+	c.overridesSignCmd.
+		Flag("force", "Attempt to sign as many CSRs as possible even in the presence of errors.").
+		Short('f').
+		BoolVar(&c.overridesSignForce)
 
 	c.overridesCreateCmd = overridesCmd.Command("create", "Create an issuer override from the given certificate chains.")
 	c.overridesCreateCmd.
@@ -654,31 +659,55 @@ func (c *WorkloadIdentityCommand) runOverridesSignCSRs(ctx context.Context, clie
 	}
 
 	keypairs := ca.GetTrustedTLSKeyPairs()
-	csrs := make([]*x509.CertificateRequest, 0, len(keypairs))
+	type result struct {
+		issuer *x509.Certificate
+		csr    *x509.CertificateRequest
+		err    error
+	}
+	results := make([]result, 0, len(keypairs))
 	for _, kp := range keypairs {
-		block, _ := pem.Decode(kp.Cert)
-		if block == nil {
-			return trace.BadParameter("failed to decode PEM block in SPIFFE CA")
+		issuer, err := tlsca.ParseCertificatePEM(kp.Cert)
+		if err != nil {
+			return trace.Wrap(err)
 		}
 		resp, err := oclt.SignX509IssuerCSR(ctx, &workloadidentityv1pb.SignX509IssuerCSRRequest{
-			Issuer:          block.Bytes,
+			Issuer:          issuer.Raw,
 			CsrCreationMode: c.overridesSignMode,
 		})
 		if err != nil {
-			return trace.Wrap(err)
+			if !c.overridesSignForce {
+				return trace.Wrap(err)
+			}
+			results = append(results, result{
+				issuer: issuer,
+				csr:    nil,
+				err:    err,
+			})
+			continue
 		}
 		csr, err := x509.ParseCertificateRequest(resp.GetCsr())
 		if err != nil {
 			return trace.Wrap(err)
 		}
-		csrs = append(csrs, csr)
-	}
-	for _, csr := range csrs {
-		fmt.Fprintln(c.stdout, csr.Subject)
-		_ = pem.Encode(c.stdout, &pem.Block{
-			Type:  "CERTIFICATE REQUEST",
-			Bytes: csr.Raw,
+		results = append(results, result{
+			issuer: issuer,
+			csr:    csr,
+			err:    nil,
 		})
 	}
-	return nil
+
+	var errs []error
+	for _, r := range results {
+		fmt.Fprintln(c.stdout, r.issuer.Subject)
+		if r.err != nil {
+			errs = append(errs, r.err)
+			fmt.Fprintln(c.stdout, r.err.Error())
+			continue
+		}
+		_ = pem.Encode(c.stdout, &pem.Block{
+			Type:  "CERTIFICATE REQUEST",
+			Bytes: r.csr.Raw,
+		})
+	}
+	return trace.Wrap(trace.NewAggregate(errs...), "some or all signature requests failed")
 }

tool/tctl/common/workload_identity_command_overrides_test.go

@@ -215,7 +215,7 @@ func TestOverrideSign(t *testing.T) {
 
 	l, err := out.ReadString('\n')
 	require.NoError(t, err)
-	require.Equal(t, "CN=csr1\n", l)
+	require.Equal(t, "CN=c1\n", l)
 	b, rest := pem.Decode(out.Bytes())
 	require.NotNil(t, b)
 	require.Equal(t, "CERTIFICATE REQUEST", b.Type)
@@ -224,7 +224,7 @@ func TestOverrideSign(t *testing.T) {
 
 	l, err = out.ReadString('\n')
 	require.NoError(t, err)
-	require.Equal(t, "CN=csr2\n", l)
+	require.Equal(t, "CN=c2\n", l)
 	b, rest = pem.Decode(out.Bytes())
 	require.NotNil(t, b)
 	require.Equal(t, "CERTIFICATE REQUEST", b.Type)
@@ -232,6 +232,62 @@ func TestOverrideSign(t *testing.T) {
 	out.Next(out.Len() - len(rest))
 
 	require.Zero(t, out.Len())
+
+	clt = runFakeAPIServer(t, func(s *grpc.Server) {
+		proto.RegisterAuthServiceServer(s, &domainNameServer{
+			domainName: "clustername",
+		})
+		trustv1.RegisterTrustServiceServer(s, &caServer{
+			cas: map[types.CertAuthID]*types.CertAuthorityV2{
+				{Type: types.SPIFFECA, DomainName: "clustername"}: {
+					Spec: types.CertAuthoritySpecV2{
+						ActiveKeys: types.CAKeySet{
+							TLS: []*types.TLSKeyPair{
+								{Cert: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c1.Leaf.Raw})},
+								{Cert: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c2.Leaf.Raw})},
+							},
+						},
+					},
+				},
+			},
+		})
+		workloadidentityv1.RegisterX509OverridesServiceServer(s, &csrServer{
+			resp: map[string][]byte{
+				string(c1.Leaf.Raw): csr1,
+			},
+		})
+	})
+
+	out.Reset()
+
+	err = runCommand(t, clt, &WorkloadIdentityCommand{stdout: out}, []string{
+		"workload-identity", "x509-issuer-overrides", "sign-csrs",
+	})
+	require.ErrorAs(t, err, new(*trace.NotFoundError))
+	require.Zero(t, out.Len())
+
+	err = runCommand(t, clt, &WorkloadIdentityCommand{stdout: out}, []string{
+		"workload-identity", "x509-issuer-overrides", "sign-csrs", "--force",
+	})
+	require.ErrorAs(t, err, new(*trace.NotFoundError))
+
+	l, err = out.ReadString('\n')
+	require.NoError(t, err)
+	require.Equal(t, "CN=c1\n", l)
+	b, rest = pem.Decode(out.Bytes())
+	require.NotNil(t, b)
+	require.Equal(t, "CERTIFICATE REQUEST", b.Type)
+	require.Equal(t, csr1, b.Bytes)
+	out.Next(out.Len() - len(rest))
+
+	l, err = out.ReadString('\n')
+	require.NoError(t, err)
+	require.Equal(t, "CN=c2\n", l)
+	l, err = out.ReadString('\n')
+	require.NoError(t, err)
+	require.Equal(t, signKeyNotFoundMessage+"\n", l)
+
+	require.Zero(t, out.Len())
 }
 
 type domainNameServer struct {
@@ -292,13 +348,15 @@ type csrServer struct {
 	resp map[string][]byte
 }
 
+const signKeyNotFoundMessage = "key not found or something idk"
+
 func (s *csrServer) SignX509IssuerCSR(ctx context.Context, req *workloadidentityv1.SignX509IssuerCSRRequest) (*workloadidentityv1.SignX509IssuerCSRResponse, error) {
 	if req.GetCsrCreationMode() == 0 {
 		return nil, status.Errorf(codes.InvalidArgument, "")
 	}
 	csr, ok := s.resp[string(req.GetIssuer())]
 	if !ok {
-		return nil, status.Errorf(codes.NotFound, "")
+		return nil, status.Errorf(codes.NotFound, signKeyNotFoundMessage)
 	}
 	return &workloadidentityv1.SignX509IssuerCSRResponse{Csr: csr}, nil
 }