Removed old Drone pipelines (#37526)

* Removed old Drone pipelines

* Removed docs webhook and buildbox pipelines

dronegen/buildbox.go

@@ -1,122 +0,0 @@
-/*
- * Teleport
- * Copyright (C) 2023  Gravitational, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package main
-
-import "fmt"
-
-func buildboxPipelineSteps() []step {
-	steps := []step{
-		{
-			Name:  "Check out code",
-			Image: "docker:git",
-			Commands: []string{
-				`git clone --depth 1 --single-branch --branch ${DRONE_SOURCE_BRANCH:-master} https://github.com/gravitational/${DRONE_REPO_NAME}.git .`,
-				`git checkout ${DRONE_COMMIT}`,
-			},
-		},
-		waitForDockerStep(),
-		kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{
-			awsRoleSettings: awsRoleSettings{
-				awsAccessKeyID:     value{fromSecret: "STAGING_BUILDBOX_DRONE_USER_ECR_KEY"},
-				awsSecretAccessKey: value{fromSecret: "STAGING_BUILDBOX_DRONE_USER_ECR_SECRET"},
-				role:               value{fromSecret: "STAGING_BUILDBOX_DRONE_ECR_AWS_ROLE"},
-			},
-			configVolume: volumeRefAwsConfig,
-			name:         "Configure Staging AWS Profile",
-			profile:      "staging",
-		}),
-		kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{
-			awsRoleSettings: awsRoleSettings{
-				awsAccessKeyID:     value{fromSecret: "PRODUCTION_BUILDBOX_DRONE_USER_ECR_KEY"},
-				awsSecretAccessKey: value{fromSecret: "PRODUCTION_BUILDBOX_DRONE_USER_ECR_SECRET"},
-				role:               value{fromSecret: "PRODUCTION_BUILDBOX_DRONE_ECR_AWS_ROLE"},
-			},
-			configVolume: volumeRefAwsConfig,
-			name:         "Configure Production AWS Profile",
-			append:       true,
-			profile:      "production",
-		}),
-	}
-
-	for _, name := range []string{"buildbox", "buildbox-arm", "buildbox-centos7"} {
-		for _, fips := range []bool{false, true} {
-			// FIPS is only supported on centos7
-			if fips && name != "buildbox-centos7" {
-				continue
-			}
-			steps = append(steps, buildboxPipelineStep(name, fips))
-		}
-	}
-	return steps
-}
-
-func buildboxPipelineStep(buildboxName string, fips bool) step {
-	var buildboxTagSuffix string
-	if buildboxName == "buildbox-centos7" {
-		// Drone-managed buildboxes are only amd64
-		buildboxTagSuffix = "-amd64"
-	}
-	if fips {
-		buildboxName += "-fips"
-	}
-	return step{
-		Name:    "Build and push " + buildboxName,
-		Image:   "docker",
-		Pull:    "if-not-exists",
-		Volumes: []volumeRef{volumeRefAwsConfig, volumeRefDocker, volumeRefDockerConfig},
-		Commands: []string{
-			`apk add --no-cache make aws-cli go`,
-			`chown -R $UID:$GID /go`,
-			// Authenticate to staging registry
-			`aws ecr get-login-password --profile staging --region=us-west-2 | docker login -u="AWS" --password-stdin ` + StagingRegistry,
-			// Build buildbox image
-			fmt.Sprintf(`make -C build.assets %s`, buildboxName),
-			// Retag for staging registry
-			fmt.Sprintf(`docker tag %s/gravitational/teleport-%s:$BUILDBOX_VERSION%s %s/gravitational/teleport-%s:$BUILDBOX_VERSION-$DRONE_COMMIT_SHA`, GitHubRegistry, buildboxName, buildboxTagSuffix, StagingRegistry, buildboxName),
-			// Push to staging registry
-			fmt.Sprintf(`docker push %s/gravitational/teleport-%s:$BUILDBOX_VERSION-$DRONE_COMMIT_SHA`, StagingRegistry, buildboxName),
-			// Authenticate to production registry
-			`docker logout ` + StagingRegistry,
-			`aws ecr-public get-login-password --profile production --region=us-east-1 | docker login -u="AWS" --password-stdin ` + ProductionRegistry,
-			// Retag for production registry
-			fmt.Sprintf(`docker tag %s/gravitational/teleport-%s:$BUILDBOX_VERSION%s %s/gravitational/teleport-%s:$BUILDBOX_VERSION`, GitHubRegistry, buildboxName, buildboxTagSuffix, ProductionRegistry, buildboxName),
-			// Push to production registry
-			fmt.Sprintf(`docker push %s/gravitational/teleport-%s:$BUILDBOX_VERSION`, ProductionRegistry, buildboxName),
-		},
-	}
-}
-
-func buildboxPipeline() pipeline {
-	p := newKubePipeline("build-buildboxes")
-	p.Environment = map[string]value{
-		"BUILDBOX_VERSION": buildboxVersion,
-		"UID":              {raw: "1000"},
-		"GID":              {raw: "1000"},
-	}
-
-	// only on master for now; add the release branch name when forking a new release series.
-	p.Trigger = pushTriggerForBranch("master", "branch/*")
-	p.Workspace = workspace{Path: "/go/src/github.com/gravitational/teleport"}
-	p.Volumes = []volume{volumeAwsConfig, volumeDocker, volumeDockerConfig}
-	p.Services = []service{
-		dockerService(),
-	}
-	p.Steps = buildboxPipelineSteps()
-	return p
-}

dronegen/common.go

@@ -19,10 +19,7 @@
 package main
 
 import (
-	"bytes"
 	"fmt"
-	"log"
-	"os/exec"
 	"strings"
 )
 
@@ -109,32 +106,6 @@ var (
 	}
 )
 
-var buildboxVersion value
-
-func init() {
-	v, err := exec.Command("make", "-s", "-C", "build.assets", "print-buildbox-version").Output()
-	if err != nil {
-		log.Fatalf("could not get buildbox version: %v", err)
-	}
-	buildboxVersion = value{raw: string(bytes.TrimSpace(v))}
-}
-
-func pushTriggerForBranch(branches ...string) trigger {
-	t := trigger{
-		Event: triggerRef{Include: []string{"push"}},
-		Repo:  triggerRef{Include: []string{"gravitational/teleport"}},
-	}
-	t.Branch.Include = append(t.Branch.Include, branches...)
-	return t
-}
-
-func cronTrigger(cronJobNames []string) trigger {
-	return trigger{
-		Cron: triggerRef{Include: cronJobNames},
-		Repo: triggerRef{Include: []string{"gravitational/teleport"}},
-	}
-}
-
 func cloneRepoCommands(cloneDirectory, commit string) []string {
 	return []string{
 		fmt.Sprintf("mkdir -pv %q", cloneDirectory),

dronegen/container_image_triggers.go

@@ -97,72 +97,6 @@ func NewPromoteTrigger(branchMajorVersion string) *TriggerInfo {
 	}
 }
 
-func NewCronTrigger(latestMajorVersions []string) *TriggerInfo {
-	if len(latestMajorVersions) == 0 {
-		return nil
-	}
-
-	majorVersionVarBasePath := "/go/vars/full-version"
-
-	supportedVersions := make([]*ReleaseVersion, 0, len(latestMajorVersions))
-	if len(latestMajorVersions) > 0 {
-		latestMajorVersion := latestMajorVersions[0]
-		supportedVersions = append(supportedVersions, &ReleaseVersion{
-			MajorVersion:        latestMajorVersion,
-			ShellVersion:        readCronShellVersionCommand(majorVersionVarBasePath, latestMajorVersion),
-			RelativeVersionName: "current-version",
-			SetupSteps:          []step{getLatestSemverStep(latestMajorVersion, majorVersionVarBasePath)},
-		})
-
-		if len(latestMajorVersions) > 1 {
-			for i, majorVersion := range latestMajorVersions[1:] {
-				supportedVersions = append(supportedVersions, &ReleaseVersion{
-					MajorVersion: majorVersion,
-					ShellVersion: readCronShellVersionCommand(majorVersionVarBasePath, majorVersion),
-					// Omitted because it doesn't matter here - latest tags should always be built
-					// ShellIsPrerelease:   // ,
-					RelativeVersionName: fmt.Sprintf("previous-version-%d", i+1),
-					SetupSteps:          []step{getLatestSemverStep(majorVersion, majorVersionVarBasePath)},
-				})
-			}
-		}
-	}
-
-	return &TriggerInfo{
-		Trigger: cronTrigger([]string{"teleport-container-images-cron"}),
-		Name:    "cron",
-		Flags: &TriggerFlags{
-			ShouldAffectProductionImages: true,
-			ShouldBuildNewImages:         true,
-			UseUniqueStagingTag:          true,
-			ShouldOnlyPublishFullSemver:  false,
-		},
-		SupportedVersions: supportedVersions,
-	}
-}
-
-func getLatestSemverStep(majorVersion string, majorVersionVarBasePath string) step {
-	// We don't use "/go/src/github.com/gravitational/teleport" here as a later stage
-	// may need to clone a different version, and "/go" persists between steps
-	cloneDirectory := "/tmp/teleport"
-	majorVersionVarPath := fmt.Sprintf("%s-%s", majorVersionVarBasePath, majorVersion)
-	return step{
-		Name:  fmt.Sprintf("Find the latest available semver for %s", majorVersion),
-		Image: fmt.Sprintf("golang:%s", GoVersion),
-		Commands: append(
-			cloneRepoCommands(cloneDirectory, fmt.Sprintf("branch/%s", majorVersion)),
-			fmt.Sprintf("mkdir -pv $(dirname %q)", majorVersionVarPath),
-			fmt.Sprintf("cd %q", path.Join(cloneDirectory, "build.assets", "tooling", "cmd", "query-latest")),
-			fmt.Sprintf("go run . %q | sed 's/v//' > %q", majorVersion, majorVersionVarPath),
-			fmt.Sprintf("echo Found full semver \"$(cat %q)\" for major version %q", majorVersionVarPath, majorVersion),
-		),
-	}
-}
-
-func readCronShellVersionCommand(majorVersionDirectory, majorVersion string) string {
-	return fmt.Sprintf("v$(cat '%s-%s')", majorVersionDirectory, majorVersion)
-}
-
 func recordPrereleaseStatus(shellVersion, recordFilePath string) step {
 	clonePath := "/tmp/repo"
 	commands := []string{

dronegen/container_images.go

@@ -32,7 +32,7 @@ import (
 const branchMajorVersion int = 15
 const latestReleaseVersion int = 14
 
-func buildPipelineVersions() (string, []string) {
+func buildPipelineVersions() string {
 	branchMajorSemver := fmt.Sprintf("v%d", branchMajorVersion)
 	// Note that this only matters in the context of the master branch
 	updateVersionCount := 3
@@ -41,19 +41,14 @@ func buildPipelineVersions() (string, []string) {
 		imageUpdateSemvers[i] = fmt.Sprintf("v%d", latestReleaseVersion-i)
 	}
 
-	return branchMajorSemver, imageUpdateSemvers
+	return branchMajorSemver
 }
 
 func buildContainerImagePipelines() []pipeline {
-	branchMajorSemver, imageUpdateSemvers := buildPipelineVersions()
+	branchMajorSemver := buildPipelineVersions()
 
 	triggers := []*TriggerInfo{
 		NewPromoteTrigger(branchMajorSemver),
-		NewCronTrigger(imageUpdateSemvers),
-	}
-
-	if configureForPRTestingOnly {
-		triggers = append(triggers, NewTestTrigger(prBranch, branchMajorSemver))
 	}
 
 	pipelines := make([]pipeline, 0, len(triggers))

dronegen/container_images_repos.go

@@ -26,6 +26,12 @@ import (
 	"golang.org/x/exp/maps"
 )
 
+const (
+	ProductionRegistryOrg string = "gravitational"
+	PublicEcrRegion       string = "us-east-1"
+	StagingEcrRegion      string = "us-west-2"
+)
+
 // Describes a registry and repo that images are to be published to.
 type ContainerRepo struct {
 	Name            string                            // Human readable name for the repo. Does not need to match remote value.
@@ -51,17 +57,6 @@ func NewEcrContainerRepo(accessKeyIDSecret, secretAccessKeySecret, roleSecret, d
 	profileName := fmt.Sprintf("ecr-%s", name)
 
 	registryOrg := ProductionRegistryOrg
-	if configureForPRTestingOnly {
-		accessKeyIDSecret = testingSecretPrefix + accessKeyIDSecret
-		secretAccessKeySecret = testingSecretPrefix + secretAccessKeySecret
-		roleSecret = testingSecretPrefix + roleSecret
-		registryOrg = testingECRRegistryOrg
-
-		if !isPublic {
-			domain = testingECRDomain
-			ecrRegion = testingECRRegion
-		}
-	}
 
 	loginCommands := []string{
 		"apk add --no-cache aws-cli",

dronegen/container_images_testing.go

@@ -1,87 +0,0 @@
-/*
- * Teleport
- * Copyright (C) 2023  Gravitational, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package main
-
-// This file contains variables and functions to make testing of the container image build process
-// more simple and easier.
-
-// To run one of these pipelines locally:
-// # Drone requires certain variables to be set
-// export DRONE_REMOTE_URL="https://github.com/gravitational/teleport"
-// export DRONE_SOURCE_BRANCH="$(git branch --show-current)"
-// # `drone exec` does not support `exec` or `kubernetes` pipelines
-// sed -i '' 's/type\: kubernetes/type\: docker/' .drone.yml && sed -i '' 's/type\: exec/type\: docker/' .drone.yml
-// # Drone has a bug where "workspace" is appended to "/drone/src". This fixes that by updating references
-// sed -i '' 's~/go/~/drone/src/go/~g' .drone.yml
-// # Pull the current branch instead of v11
-// sed -i '' "s~git checkout -qf \"\$(cat '/go/vars/full-version/v11')\"~git checkout -qf \"${DRONE_SOURCE_BRANCH}\"~" .drone.yml
-// # `drone exec` does not properly map the workspace path. This creates a volume to be shared between steps
-// #  at the correct path
-// DOCKER_VOLUME_NAME="go"
-// docker volume create "$DOCKER_VOLUME_NAME"
-// drone exec --trusted --pipeline teleport-container-images-current-version-cron --clone=false --volume "${DOCKER_VOLUME_NAME}:/go"
-// # Cleanup
-// docker volume rm "$DOCKER_VOLUME_NAME"
-
-// If you are working on a PR/testing changes to this file you should configure the following for Drone testing:
-// 1. Publish the branch you're working on
-// 2. Set `prBranch` to the name of the branch in (1)
-// 3. Set `configureForPRTestingOnly` to true
-// 4. Create a public and private ECR repos for "teleport", "teleport-ent", "teleport-operator", "teleport-lab"
-// 5. Set `testingECRRegistryOrg` to the org name(s) used in (4)
-// 6. Set the `ECRTestingDomain` to the domain used for the private ECR repos
-// 7. Create two separate IAM users, each with full access to either the public ECR repo OR the private ECR repo
-// 8. Set the Drone secrets for the secret names listed in "GetContainerRepos" to the credentials in (7, 8), prefixed by the value of `testingSecretPrefix`
-//
-// On each commit, after running `make dronegen``, run the following commands and resign the file:
-// # Pull the current branch instead of v11 so the appropriate dockerfile gets loaded
-// sed -i '' "s~git checkout -qf \"\$(cat '/go/vars/full-version/v11')\"~git checkout -qf \"${DRONE_SOURCE_BRANCH}\"~" .drone.yml
-//
-// When finishing up your PR check the following:
-// * The testing secrets added to Drone have been removed
-// * `configureForPRTestingOnly` has been set to false, and `make dronegen` has been reran afterwords
-
-const (
-	configureForPRTestingOnly bool   = false
-	testingSecretPrefix       string = "TEST_"
-	testingECRRegistryOrg     string = "u8j2q1d9"
-	testingECRRegion          string = "us-east-2"
-	prBranch                  string = "" // "fred/multiarch-teleport-actual-container-images"
-	testingECRDomain          string = "278576220453.dkr.ecr.us-east-2.amazonaws.com"
-)
-
-const (
-	ProductionRegistryOrg string = "gravitational"
-	PublicEcrRegion       string = "us-east-1"
-	StagingEcrRegion      string = "us-west-2"
-)
-
-func NewTestTrigger(triggerBranch, testMajorVersion string) *TriggerInfo {
-	// baseTrigger := NewTagTrigger(testMajorVersion)
-	// baseTrigger := NewPromoteTrigger(testMajorVersion)
-	baseTrigger := NewCronTrigger([]string{testMajorVersion})
-	baseTrigger.Name = "Test trigger on push"
-	baseTrigger.Trigger = trigger{
-		Repo:   triggerRef{Include: []string{"gravitational/teleport"}},
-		Event:  triggerRef{Include: []string{"push"}},
-		Branch: triggerRef{Include: []string{triggerBranch}},
-	}
-
-	return baseTrigger
-}

dronegen/main.go

@@ -37,8 +37,6 @@ func main() {
 	pipelines = append(pipelines, tagPipelines()...)
 	pipelines = append(pipelines, cronPipelines()...)
 	pipelines = append(pipelines, promoteBuildPipelines()...)
-	pipelines = append(pipelines, updateDocsPipeline())
-	pipelines = append(pipelines, buildboxPipeline())
 	pipelines = append(pipelines, buildContainerImagePipelines()...)
 	pipelines = append(pipelines, publishReleasePipeline())
 

dronegen/misc.go

@@ -1,24 +0,0 @@
-/*
- * Teleport
- * Copyright (C) 2023  Gravitational, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package main
-
-func updateDocsPipeline() pipeline {
-	// TODO: migrate
-	return pipeline{}
-}