Added parallel OS package publishing and rolling release channel (#28659)

* Added parallel OS package publishing and rolling release channel

* Bump `e` for test

* Fixed parallelism issue

* Re-ran make dronegen

* Addressed PR comments

* reverted accidental e ref change

* Linter fix

* Bumped `e` to `gravitational/teleport` master ref

build.assets/tooling/cmd/gh-trigger-workflow/args.go

@@ -67,16 +67,17 @@ func (k *key) Set(s string) error {
 
 // args holds the parsed command-line arguments for the command.
 type args struct {
-	appID          int64
-	appKey         key
-	owner          string
-	repo           string
-	workflow       string
-	workflowRef    string
-	useWorkflowTag bool
-	seriesRun      bool
-	timeout        time.Duration
-	inputs         inputMap
+	appID           int64
+	appKey          key
+	owner           string
+	repo            string
+	workflow        string
+	workflowRef     string
+	useWorkflowTag  bool
+	seriesRun       bool
+	seriesRunFilter string
+	timeout         time.Duration
+	inputs          inputMap
 }
 
 func parseCommandLine() (args, error) {
@@ -95,6 +96,7 @@ func parseCommandLine() (args, error) {
 	flag.StringVar(&cliArgs.workflowRef, "workflow-ref", cliArgs.workflowRef, "Revision reference")
 	flag.BoolVar(&cliArgs.useWorkflowTag, "tag-workflow", false, "Use a workflow input to tag and ID workflows spawned by the event")
 	flag.BoolVar(&cliArgs.seriesRun, "series-run", false, "Attempts to wait for any workflows scheduled but not completed before starting this one")
+	flag.StringVar(&cliArgs.seriesRunFilter, "series-run-filter", ".*", "Regexp filter to apply when determining what workflow runs should should be waited on, defaulting to \".*\"")
 	flag.DurationVar(&cliArgs.timeout, "timeout", time.Duration(0), "Timeout. If not specified, waits forever.")
 	flag.Var(cliArgs.inputs, "input", "Input to target workflow")
 

build.assets/tooling/cmd/gh-trigger-workflow/main.go

@@ -50,6 +50,7 @@ import (
 	"flag"
 	"log"
 	"net/http"
+	"regexp"
 	"strings"
 	"time"
 
@@ -188,12 +189,21 @@ func getIncompleteWorkflowRunID(ctx context.Context, gh *ghapi.Client, args args
 		return nil, trace.Wrap(err, "failed to get a list of current workflow runs")
 	}
 
+	regex, err := regexp.Compile(args.seriesRunFilter)
+	if err != nil {
+		return nil, trace.Wrap(err, "failed to compile series run regex %q", args.seriesRunFilter)
+	}
+
 	for _, recentRun := range recentRuns {
 		runStatus := recentRun.GetStatus()
 		if runStatus == "" {
 			return nil, trace.Errorf("failed to get status for run ID %q", recentRun.GetID())
 		}
 
+		if !regex.MatchString(*recentRun.Name) {
+			continue
+		}
+
 		if runStatus != "completed" {
 			return recentRun, nil
 		}

dronegen/gha.go

@@ -33,16 +33,18 @@ type ghaWorkflow struct {
 	slackOnError      bool
 	shouldTagWorkflow bool
 	seriesRun         bool
+	seriesRunFilter   string
 	inputs            map[string]string
 }
 
 type ghaBuildType struct {
 	buildType
 	trigger
-	pipelineName string
-	checkoutPath string
-	dependsOn    []string
-	workflows    []ghaWorkflow
+	pipelineName               string
+	checkoutPath               string
+	dependsOn                  []string
+	workflows                  []ghaWorkflow
+	enableParallelWorkflowRuns bool
 }
 
 func ghaBuildPipeline(ghaBuild ghaBuildType) pipeline {
@@ -73,19 +75,67 @@ func ghaMultiBuildPipeline(setupSteps []step, ghaBuild ghaBuildType) pipeline {
 		},
 	}
 
+	setupStepNames := getStepNames(p.Steps)
+	if ghaBuild.enableParallelWorkflowRuns && len(setupSteps) >= 1 {
+		for index := range setupSteps {
+			setupSteps[index].DependsOn = append(setupSteps[index].DependsOn, setupStepNames...)
+		}
+		setupStepNames = getStepNames(setupSteps)
+	}
+
 	p.Steps = append(p.Steps, setupSteps...)
 
-	for _, workflow := range ghaBuild.workflows {
-		p.Steps = append(p.Steps, buildGHAWorkflowCallStep(workflow, checkoutPath))
-
-		if workflow.slackOnError {
-			p.Steps = append(p.Steps, sendErrorToSlackStep())
-		}
+	for counter, workflow := range ghaBuild.workflows {
+		// This wait time is to avoid a GHA pending/queuing issue where multiple workflows fired off at the same time
+		// (or somewhat close to the same time) will cause only one workflow to be queued, with the others being canceled.
+		// For details, see
+		// https://docs.github.com/en/actions/using-jobs/using-concurrency#example-only-cancel-in-progress-jobs-or-runs-for-the-current-workflow:~:text=When%20a%20concurrent,progress%3A%20true.
+		sleepTime := time.Duration(counter*10) * time.Second // 10 seconds for each workflow
+		p.Steps = append(p.Steps, buildWorkflowSteps(workflow, checkoutPath, ghaBuild.enableParallelWorkflowRuns, sleepTime, setupStepNames)...)
 	}
 
 	return p
 }
 
+func buildWorkflowSteps(workflow ghaWorkflow, checkoutPath string, enableParallelWorkflowRuns bool, sleepTime time.Duration, setupStepNames []string) []step {
+	var steps []step
+	workflowStep := buildGHAWorkflowCallStep(workflow, checkoutPath)
+
+	if enableParallelWorkflowRuns {
+		if sleepTime > 0 {
+			sleepStep := sleepStep(sleepTime, setupStepNames, workflow.stepName)
+			steps = append(steps, sleepStep)
+			workflowStep.DependsOn = append(workflowStep.DependsOn, sleepStep.Name)
+		} else {
+			workflowStep.DependsOn = append(workflowStep.DependsOn, setupStepNames...)
+		}
+	}
+
+	steps = append(steps, workflowStep)
+
+	if workflow.slackOnError {
+		slackStep := sendErrorToSlackStep()
+		if enableParallelWorkflowRuns {
+			slackStep.DependsOn = append(slackStep.DependsOn, workflowStep.Name)
+		}
+
+		steps = append(steps, slackStep)
+	}
+
+	return steps
+}
+
+func sleepStep(sleepTime time.Duration, setupStepNames []string, stepNameSuffix string) step {
+	return step{
+		Name:  fmt.Sprintf("Wait - %s", stepNameSuffix),
+		Image: "alpine:latest",
+		Commands: []string{
+			fmt.Sprintf("sleep %v", sleepTime.Round(time.Second).Seconds()),
+		},
+		DependsOn: setupStepNames,
+	}
+}
+
 func buildGHAWorkflowCallStep(workflow ghaWorkflow, checkoutPath string) step {
 	var cmd strings.Builder
 	cmd.WriteString(`go run ./cmd/gh-trigger-workflow `)
@@ -98,6 +148,10 @@ func buildGHAWorkflowCallStep(workflow ghaWorkflow, checkoutPath string) step {
 
 	if workflow.seriesRun {
 		cmd.WriteString(`-series-run `)
+
+		if workflow.seriesRunFilter != "" {
+			fmt.Fprintf(&cmd, `-series-run-filter %s `, workflow.seriesRunFilter)
+		}
 	}
 
 	fmt.Fprintf(&cmd, `-timeout %s `, workflow.timeout.String())

dronegen/os_repos.go

@@ -42,6 +42,13 @@ func promoteBuildOsRepoPipeline() pipeline {
 			packageNameFilter: `teleport-ent-updater*`,
 			displayName:       "teleport-ent-updater",
 		},
+		// Rolling release pipelines
+		{
+			versionChannel:    "rolling",
+			packageNameFilter: `$($DRONE_REPO_PRIVATE && echo "*ent*" || echo "")`,
+			packageToTest:     "teleport-ent",
+			displayName:       "Teleport",
+		},
 	}
 
 	return buildPromoteOsPackagePipelines(packageDeployments)
@@ -52,10 +59,11 @@ func buildPromoteOsPackagePipelines(packageDeployments []osPackageDeployment) pi
 	clonePath := "/go/src/github.com/gravitational/teleport"
 
 	ghaBuild := ghaBuildType{
-		trigger:      triggerPromote,
-		pipelineName: "publish-os-package-repos",
-		checkoutPath: clonePath,
-		workflows:    buildWorkflows(releaseEnvironmentFilePath, packageDeployments),
+		trigger:                    triggerPromote,
+		pipelineName:               "publish-os-package-repos",
+		checkoutPath:               clonePath,
+		workflows:                  buildWorkflows(releaseEnvironmentFilePath, packageDeployments),
+		enableParallelWorkflowRuns: true,
 	}
 	setupSteps := []step{
 		{
@@ -97,6 +105,7 @@ func buildWorkflows(releaseEnvironmentFilePath string, packageDeployments []osPa
 				timeout:           12 * time.Hour, // DR takes a long time
 				shouldTagWorkflow: true,
 				seriesRun:         true,
+				seriesRunFilter:   fmt.Sprintf(".*%s.*", repoType),
 				inputs:            inputs,
 			})
 		}