gravitational/teleport
1
0
Fork 0
mirror of https://github.com/gravitational/teleport.git synced 2026-10-11 22:49:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate publishing of new APT and YUM repos to GHA (#24350)

Browse Source 
* Deleted OPRT from this repo

* Migrated dronegen to call gha workflow

* Removed dead pipelines

* Removed unused function to appease linter

* Accounted for private repo promotions

* Updated to use package name filter

* Accounted for teleport-ent-updater publishing
This commit is contained in:
fheinecke
2023-04-14 19:42:53 +00:00
committed by GitHub
parent b1941aaa85
commit 5690cb007e
18 changed files with 231 additions and 3907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.drone.yml
+191 -354
View File
@@ -6504,42 +6504,12 @@ image_pull_secrets:
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/os_repos.go (main.buildNeverTriggerPipeline)
# Generated at dronegen/gha.go (main.ghaBuildPipeline)
################################################
kind: pipeline
type: kubernetes
name: migrate-apt-new-repos
trigger:
event:
include:
- custom
repo:
include:
- non-existent-repository
branch:
include:
- non-existent-branch
clone:
disable: true
steps:
- name: Placeholder
image: alpine:latest
commands:
- echo "This command, step, and pipeline never runs"
image_pull_secrets:
- DOCKERHUB_CREDENTIALS
---
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/os_repos.go (main.(*OsPackageToolPipelineBuilder).buildBaseOsPackagePipeline)
################################################
kind: pipeline
type: kubernetes
name: publish-apt-new-repos
name: publish-normal-apt-new-repos
trigger:
event:
include:
@@ -6549,21 +6519,14 @@ trigger:
- production
repo:
include:
- gravitational/teleport
- gravitational/teleport-private
- gravitational/*
workspace:
path: /go
clone:
disable: true
steps:
- name: Verify build is tagged
image: alpine:latest
pull: if-not-exists
commands:
- '[ -n ${DRONE_TAG} ] || (echo ''DRONE_TAG is not set. Is the commit tagged?''
&& exit 1)'
- name: Check out code
image: alpine/git:latest
image: docker:git
pull: if-not-exists
commands:
- mkdir -pv "/go/src/github.com/gravitational/teleport"
@@ -6571,176 +6534,35 @@ steps:
- git init
- git remote add origin ${DRONE_REMOTE_URL}
- git fetch origin --tags
- git checkout -qf "${DRONE_TAG}"
- name: Assume Download AWS Role
image: amazon/aws-cli
pull: if-not-exists
commands:
- aws sts get-caller-identity
- |-
printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\n" \
$(aws sts assume-role \
--role-arn "$AWS_ROLE" \
--role-session-name $(echo "drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER}" | sed "s|/|-|g") \
--query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
--output text) \
> /root/.aws/credentials
- unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
- aws sts get-caller-identity --profile default
- git checkout -qf "${DRONE_COMMIT_SHA}"
- mkdir -m 0700 /root/.ssh && echo "$GITHUB_PRIVATE_KEY" > /root/.ssh/id_rsa &&
chmod 600 /root/.ssh/id_rsa
- ssh-keyscan -H github.com > /root/.ssh/known_hosts 2>/dev/null && chmod 600 /root/.ssh/known_hosts
- git submodule update --init e
- mkdir -pv /go/cache
- rm -f /root/.ssh/id_rsa
environment:
AWS_ACCESS_KEY_ID:
from_secret: AWS_ACCESS_KEY_ID
AWS_ROLE:
from_secret: AWS_ROLE
AWS_SECRET_ACCESS_KEY:
from_secret: AWS_SECRET_ACCESS_KEY
volumes:
- name: awsconfig
path: /root/.aws
depends_on:
- Verify build is tagged
- Check out code
- name: Download artifacts for "${DRONE_TAG}"
image: amazon/aws-cli
commands:
- mkdir -pv "$ARTIFACT_PATH"
- rm -rf "$ARTIFACT_PATH"/*
- if [ "${DRONE_REPO_PRIVATE}" = true ]; then ENT_FILTER="*ent"; fi
- FILTER="$${ENT_FILTER}*.deb*"
- aws s3 sync --no-progress --delete --exclude "*" --include "$FILTER" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/
"$ARTIFACT_PATH"
environment:
ARTIFACT_PATH: /go/artifacts
AWS_S3_BUCKET:
from_secret: AWS_S3_BUCKET
volumes:
- name: awsconfig
path: /root/.aws
depends_on:
- Assume Download AWS Role
- Verify build is tagged
- Check out code
- name: Assume Upload AWS Role
image: amazon/aws-cli
pull: if-not-exists
commands:
- aws sts get-caller-identity
- |-
printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\n" \
$(aws sts assume-role \
--role-arn "$AWS_ROLE" \
--role-session-name $(echo "drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER}" | sed "s|/|-|g") \
--query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
--output text) \
> /root/.aws/credentials
- unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
- aws sts get-caller-identity --profile default
environment:
AWS_ACCESS_KEY_ID:
from_secret: APT_REPO_NEW_AWS_ACCESS_KEY_ID
AWS_ROLE:
from_secret: APT_REPO_NEW_AWS_ROLE
AWS_SECRET_ACCESS_KEY:
from_secret: APT_REPO_NEW_AWS_SECRET_ACCESS_KEY
volumes:
- name: awsconfig
path: /root/.aws
depends_on:
- Download artifacts for "${DRONE_TAG}"
- Verify build is tagged
- Check out code
- name: Check if tag is prerelease
GITHUB_PRIVATE_KEY:
from_secret: GITHUB_PRIVATE_KEY
- name: Determine if release should go to development or production
image: golang:1.18-alpine
commands:
- apk add git
- mkdir -pv "/tmp/repo"
- cd "/tmp/repo"
- git init
- git remote add origin ${DRONE_REMOTE_URL}
- git fetch origin --tags
- git checkout -qf "${DRONE_TAG}"
- cd "/tmp/repo/build.assets/tooling"
- go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is
a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)
depends_on:
- Assume Upload AWS Role
- Verify build is tagged
- Check out code
- name: Publish debs to APT repos for "${DRONE_TAG}"
image: golang:1.18-bullseye
commands:
- apt update
- apt install -y aptly
- mkdir -pv -m0700 "$GNUPGHOME"
- echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME
- chown -R root:root "$GNUPGHOME"
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- export VERSION="${DRONE_TAG}"
- export RELEASE_CHANNEL="stable"
- go run ./cmd/build-os-package-repos apt -bucket "$REPO_S3_BUCKET" -local-bucket-path
"$BUCKET_CACHE_PATH" -version-channel "$VERSION" -release-channel "$RELEASE_CHANNEL"
-artifact-path "$ARTIFACT_PATH" -log-level 4 -aptly-root-dir "$APTLY_ROOT_DIR"
environment:
APTLY_ROOT_DIR: /mnt/aptly
ARTIFACT_PATH: /go/artifacts
AWS_REGION: us-west-2
BUCKET_CACHE_PATH: /tmp/bucket
DEBIAN_FRONTEND: noninteractive
GNUPGHOME: /tmpfs/gnupg
GPG_RPM_SIGNING_ARCHIVE:
from_secret: GPG_RPM_SIGNING_ARCHIVE
REPO_S3_BUCKET:
from_secret: APT_REPO_NEW_AWS_S3_BUCKET
volumes:
- name: apt-persistence
path: /mnt
- name: tmpfs
path: /tmpfs
- name: awsconfig
path: /root/.aws
depends_on:
- Check if tag is prerelease
- Verify build is tagged
- Check out code
volumes:
- name: apt-persistence
claim:
name: drone-s3-aptrepo-pvc
- name: tmpfs
temp:
medium: memory
- name: awsconfig
temp: {}
image_pull_secrets:
- DOCKERHUB_CREDENTIALS
---
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/os_repos.go (main.buildNeverTriggerPipeline)
################################################
kind: pipeline
type: kubernetes
name: migrate-yum-new-repos
trigger:
event:
include:
- custom
repo:
include:
- non-existent-repository
branch:
include:
- non-existent-branch
clone:
disable: true
steps:
- name: Placeholder
image: alpine:latest
- (go run ./cmd/check -tag ${DRONE_TAG} -check prerelease && echo "build" || echo
"promote") > "/go/vars/release-environment.txt"
- name: Delegate build to GitHub
image: golang:1.18-alpine
pull: if-not-exists
commands:
- echo "This command, step, and pipeline never runs"
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e
-tag-workflow -timeout 12h0m0s -workflow deploy-packages.yaml -workflow-ref=refs/heads/master
-input "artifact-tag=${DRONE_TAG}" -input "environment=$(cat "/go/vars/release-environment.txt")"
-input "package-name-filter=$($DRONE_REPO_PRIVATE && echo "*ent*" || echo "")"
-input "release-channel=stable" -input "repo-type=apt" -input "version-channel=${DRONE_TAG}" '
environment:
GHA_APP_KEY:
from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY
image_pull_secrets:
- DOCKERHUB_CREDENTIALS
@@ -6748,12 +6570,12 @@ image_pull_secrets:
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/os_repos.go (main.(*OsPackageToolPipelineBuilder).buildBaseOsPackagePipeline)
# Generated at dronegen/gha.go (main.ghaBuildPipeline)
################################################
kind: pipeline
type: kubernetes
name: publish-yum-new-repos
name: publish-normal-yum-new-repos
trigger:
event:
include:
@@ -6763,21 +6585,14 @@ trigger:
- production
repo:
include:
- gravitational/teleport
- gravitational/teleport-private
- gravitational/*
workspace:
path: /go
clone:
disable: true
steps:
- name: Verify build is tagged
image: alpine:latest
pull: if-not-exists
commands:
- '[ -n ${DRONE_TAG} ] || (echo ''DRONE_TAG is not set. Is the commit tagged?''
&& exit 1)'
- name: Check out code
image: alpine/git:latest
image: docker:git
pull: if-not-exists
commands:
- mkdir -pv "/go/src/github.com/gravitational/teleport"
@@ -6785,147 +6600,167 @@ steps:
- git init
- git remote add origin ${DRONE_REMOTE_URL}
- git fetch origin --tags
- git checkout -qf "${DRONE_TAG}"
- name: Assume Download AWS Role
image: amazon/aws-cli
pull: if-not-exists
commands:
- aws sts get-caller-identity
- |-
printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\n" \
$(aws sts assume-role \
--role-arn "$AWS_ROLE" \
--role-session-name $(echo "drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER}" | sed "s|/|-|g") \
--query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
--output text) \
> /root/.aws/credentials
- unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
- aws sts get-caller-identity --profile default
- git checkout -qf "${DRONE_COMMIT_SHA}"
- mkdir -m 0700 /root/.ssh && echo "$GITHUB_PRIVATE_KEY" > /root/.ssh/id_rsa &&
chmod 600 /root/.ssh/id_rsa
- ssh-keyscan -H github.com > /root/.ssh/known_hosts 2>/dev/null && chmod 600 /root/.ssh/known_hosts
- git submodule update --init e
- mkdir -pv /go/cache
- rm -f /root/.ssh/id_rsa
environment:
AWS_ACCESS_KEY_ID:
from_secret: AWS_ACCESS_KEY_ID
AWS_ROLE:
from_secret: AWS_ROLE
AWS_SECRET_ACCESS_KEY:
from_secret: AWS_SECRET_ACCESS_KEY
volumes:
- name: awsconfig
path: /root/.aws
depends_on:
- Verify build is tagged
- Check out code
- name: Download artifacts for "${DRONE_TAG}"
image: amazon/aws-cli
commands:
- mkdir -pv "$ARTIFACT_PATH"
- rm -rf "$ARTIFACT_PATH"/*
- if [ "${DRONE_REPO_PRIVATE}" = true ]; then ENT_FILTER="*ent"; fi
- FILTER="$${ENT_FILTER}*.rpm*"
- aws s3 sync --no-progress --delete --exclude "*" --include "$FILTER" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/
"$ARTIFACT_PATH"
environment:
ARTIFACT_PATH: /go/artifacts
AWS_S3_BUCKET:
from_secret: AWS_S3_BUCKET
volumes:
- name: awsconfig
path: /root/.aws
depends_on:
- Assume Download AWS Role
- Verify build is tagged
- Check out code
- name: Assume Upload AWS Role
image: amazon/aws-cli
pull: if-not-exists
commands:
- aws sts get-caller-identity
- |-
printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\n" \
$(aws sts assume-role \
--role-arn "$AWS_ROLE" \
--role-session-name $(echo "drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER}" | sed "s|/|-|g") \
--query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
--output text) \
> /root/.aws/credentials
- unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
- aws sts get-caller-identity --profile default
environment:
AWS_ACCESS_KEY_ID:
from_secret: YUM_REPO_NEW_AWS_ACCESS_KEY_ID
AWS_ROLE:
from_secret: YUM_REPO_NEW_AWS_ROLE
AWS_SECRET_ACCESS_KEY:
from_secret: YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY
volumes:
- name: awsconfig
path: /root/.aws
depends_on:
- Download artifacts for "${DRONE_TAG}"
- Verify build is tagged
- Check out code
- name: Check if tag is prerelease
GITHUB_PRIVATE_KEY:
from_secret: GITHUB_PRIVATE_KEY
- name: Determine if release should go to development or production
image: golang:1.18-alpine
commands:
- apk add git
- mkdir -pv "/tmp/repo"
- cd "/tmp/repo"
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- (go run ./cmd/check -tag ${DRONE_TAG} -check prerelease && echo "build" || echo
"promote") > "/go/vars/release-environment.txt"
- name: Delegate build to GitHub
image: golang:1.18-alpine
pull: if-not-exists
commands:
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e
-tag-workflow -timeout 12h0m0s -workflow deploy-packages.yaml -workflow-ref=refs/heads/master
-input "artifact-tag=${DRONE_TAG}" -input "environment=$(cat "/go/vars/release-environment.txt")"
-input "package-name-filter=$($DRONE_REPO_PRIVATE && echo "*ent*" || echo "")"
-input "release-channel=stable" -input "repo-type=yum" -input "version-channel=${DRONE_TAG}" '
environment:
GHA_APP_KEY:
from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY
image_pull_secrets:
- DOCKERHUB_CREDENTIALS
---
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/gha.go (main.ghaBuildPipeline)
################################################
kind: pipeline
type: kubernetes
name: publish-teleport-ent-updater-apt-new-repos
trigger:
event:
include:
- promote
target:
include:
- production
repo:
include:
- gravitational/*
workspace:
path: /go
clone:
disable: true
steps:
- name: Check out code
image: docker:git
pull: if-not-exists
commands:
- mkdir -pv "/go/src/github.com/gravitational/teleport"
- cd "/go/src/github.com/gravitational/teleport"
- git init
- git remote add origin ${DRONE_REMOTE_URL}
- git fetch origin --tags
- git checkout -qf "${DRONE_TAG}"
- cd "/tmp/repo/build.assets/tooling"
- go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is
a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)
depends_on:
- Assume Upload AWS Role
- Verify build is tagged
- Check out code
- name: Publish rpms to YUM repos for "${DRONE_TAG}"
image: golang:1.18-bullseye
commands:
- apt update
- apt install -y createrepo-c
- mkdir -pv "$CACHE_DIR"
- mkdir -pv -m0700 "$GNUPGHOME"
- echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME
- chown -R root:root "$GNUPGHOME"
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- export VERSION="${DRONE_TAG}"
- export RELEASE_CHANNEL="stable"
- go run ./cmd/build-os-package-repos yum -bucket "$REPO_S3_BUCKET" -local-bucket-path
"$BUCKET_CACHE_PATH" -version-channel "$VERSION" -release-channel "$RELEASE_CHANNEL"
-artifact-path "$ARTIFACT_PATH" -log-level 4 -cache-dir "$CACHE_DIR"
- git checkout -qf "${DRONE_COMMIT_SHA}"
- mkdir -m 0700 /root/.ssh && echo "$GITHUB_PRIVATE_KEY" > /root/.ssh/id_rsa &&
chmod 600 /root/.ssh/id_rsa
- ssh-keyscan -H github.com > /root/.ssh/known_hosts 2>/dev/null && chmod 600 /root/.ssh/known_hosts
- git submodule update --init e
- mkdir -pv /go/cache
- rm -f /root/.ssh/id_rsa
environment:
ARTIFACT_PATH: /go/artifacts
AWS_REGION: us-west-2
BUCKET_CACHE_PATH: /mnt/bucket
CACHE_DIR: /mnt/createrepo_cache
DEBIAN_FRONTEND: noninteractive
GNUPGHOME: /tmpfs/gnupg
GPG_RPM_SIGNING_ARCHIVE:
from_secret: GPG_RPM_SIGNING_ARCHIVE
REPO_S3_BUCKET:
from_secret: YUM_REPO_NEW_AWS_S3_BUCKET
volumes:
- name: yum-persistence
path: /mnt
- name: tmpfs
path: /tmpfs
- name: awsconfig
path: /root/.aws
depends_on:
- Check if tag is prerelease
- Verify build is tagged
- Check out code
volumes:
- name: yum-persistence
claim:
name: drone-s3-yumrepo-pvc
- name: tmpfs
temp:
medium: memory
- name: awsconfig
temp: {}
GITHUB_PRIVATE_KEY:
from_secret: GITHUB_PRIVATE_KEY
- name: Determine if release should go to development or production
image: golang:1.18-alpine
commands:
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- (go run ./cmd/check -tag ${DRONE_TAG} -check prerelease && echo "build" || echo
"promote") > "/go/vars/release-environment.txt"
- name: Delegate build to GitHub
image: golang:1.18-alpine
pull: if-not-exists
commands:
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e
-tag-workflow -timeout 12h0m0s -workflow deploy-packages.yaml -workflow-ref=refs/heads/master
-input "artifact-tag=${DRONE_TAG}" -input "environment=$(cat "/go/vars/release-environment.txt")"
-input "package-name-filter=teleport-ent-updater*" -input "release-channel=stable"
-input "repo-type=apt" -input "version-channel=cloud" '
environment:
GHA_APP_KEY:
from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY
image_pull_secrets:
- DOCKERHUB_CREDENTIALS
---
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/gha.go (main.ghaBuildPipeline)
################################################
kind: pipeline
type: kubernetes
name: publish-teleport-ent-updater-yum-new-repos
trigger:
event:
include:
- promote
target:
include:
- production
repo:
include:
- gravitational/*
workspace:
path: /go
clone:
disable: true
steps:
- name: Check out code
image: docker:git
pull: if-not-exists
commands:
- mkdir -pv "/go/src/github.com/gravitational/teleport"
- cd "/go/src/github.com/gravitational/teleport"
- git init
- git remote add origin ${DRONE_REMOTE_URL}
- git fetch origin --tags
- git checkout -qf "${DRONE_COMMIT_SHA}"
- mkdir -m 0700 /root/.ssh && echo "$GITHUB_PRIVATE_KEY" > /root/.ssh/id_rsa &&
chmod 600 /root/.ssh/id_rsa
- ssh-keyscan -H github.com > /root/.ssh/known_hosts 2>/dev/null && chmod 600 /root/.ssh/known_hosts
- git submodule update --init e
- mkdir -pv /go/cache
- rm -f /root/.ssh/id_rsa
environment:
GITHUB_PRIVATE_KEY:
from_secret: GITHUB_PRIVATE_KEY
- name: Determine if release should go to development or production
image: golang:1.18-alpine
commands:
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- (go run ./cmd/check -tag ${DRONE_TAG} -check prerelease && echo "build" || echo
"promote") > "/go/vars/release-environment.txt"
- name: Delegate build to GitHub
image: golang:1.18-alpine
pull: if-not-exists
commands:
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e
-tag-workflow -timeout 12h0m0s -workflow deploy-packages.yaml -workflow-ref=refs/heads/master
-input "artifact-tag=${DRONE_TAG}" -input "environment=$(cat "/go/vars/release-environment.txt")"
-input "package-name-filter=teleport-ent-updater*" -input "release-channel=stable"
-input "repo-type=yum" -input "version-channel=cloud" '
environment:
GHA_APP_KEY:
from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY
image_pull_secrets:
- DOCKERHUB_CREDENTIALS
@@ -20230,8 +20065,10 @@ clone:
depends_on:
- promote-build
- teleport-container-images-branch-promote
- publish-apt-new-repos
- publish-yum-new-repos
- publish-normal-apt-new-repos
- publish-normal-yum-new-repos
- publish-teleport-ent-updater-apt-new-repos
- publish-teleport-ent-updater-yum-new-repos
- promote-teleport-oci-distroless-images
- promote-teleport-kube-agent-updater-oci-images
steps:
@@ -20343,6 +20180,6 @@ image_pull_secrets:
- DOCKERHUB_CREDENTIALS
---
kind: signature
hmac: 2b0baa890b89809154cbdae3134c92f8ad3d221bf4b205c6ce858eed78e536ab
hmac: 9e770f0d365dd5fd55104a3d293b09bb6e8a9ab63d1471a80fe13aada2d1d140
...
build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go
-240
View File
@@ -1,240 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"io/fs"
"path/filepath"
"strings"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
)
type AptRepoTool struct {
config *AptConfig
aptly *Aptly
gpg *GPG
s3Manager *S3manager
supportedOSs map[string][]string
}
// Instantiates a new apt repo tool instance and performs any required setup/config.
func NewAptRepoTool(config *AptConfig, supportedOSs map[string][]string) (*AptRepoTool, error) {
aptly, err := NewAptly(config.aptlyPath)
if err != nil {
return nil, trace.Wrap(err, "failed to create a new aptly instance")
}
gpg, err := NewGPG()
if err != nil {
return nil, trace.Wrap(err, "failed to create a new GPG instance")
}
s3Manager, err := NewS3Manager(config.S3Config)
if err != nil {
return nil, trace.Wrap(err, "failed to create a new s3manager instance")
}
return &AptRepoTool{
aptly: aptly,
config: config,
gpg: gpg,
s3Manager: s3Manager,
supportedOSs: supportedOSs,
}, nil
}
// Runs the tool, creating and updating APT repos based upon the current configuration.
func (art *AptRepoTool) Run() error {
start := time.Now()
logrus.Infoln("Starting APT repo build process...")
logrus.Debugf("Using config: %+v", spew.Sdump(art.config))
isFirstRun, err := art.aptly.IsFirstRun()
if err != nil {
return trace.Wrap(err, "failed to check if Aptly needs (re)built")
}
if isFirstRun {
logrus.Warningln("First run or disaster recovery detected, attempting to rebuild existing repos from APT repository...")
err = art.s3Manager.DownloadExistingRepo()
if err != nil {
return trace.Wrap(err, "failed to sync existing repo from S3 bucket")
}
_, err = art.recreateExistingRepos(art.config.localBucketPath)
if err != nil {
return trace.Wrap(err, "failed to recreate existing repos")
}
} else {
logrus.Debugf("Not first run of tool, skipping Aptly repository rebuild process")
}
// Note: this logic will only push the artifact into the `art.supportedOSs` repos.
// This behavior is intended to allow deprecating old OS versions in the future
// without removing the associated repos entirely.
artifactRepos, err := art.getArtifactRepos()
if err != nil {
return trace.Wrap(err, "failed to create repos")
}
err = art.importNewDebs(artifactRepos)
if err != nil {
return trace.Wrap(err, "failed to import new debs")
}
err = art.publishRepos()
if err != nil {
return trace.Wrap(err, "failed to publish repos")
}
// Both Hashicorp and Docker publish their key to this path
err = art.gpg.WritePublicKeyToFile(filepath.Join(art.aptly.rootDir, "public", "gpg"))
if err != nil {
return trace.Wrap(err, "failed to write GPG public key")
}
art.s3Manager.ChangeLocalBucketPath(filepath.Join(art.aptly.rootDir, "public"))
err = art.s3Manager.UploadBuiltRepo()
if err != nil {
return trace.Wrap(err, "failed to sync changes to S3 bucket")
}
// Future work: add literals to config?
err = art.s3Manager.UploadRedirectURL("index.html", "https://goteleport.com/docs/installation/#linux")
if err != nil {
return trace.Wrap(err, "failed to redirect index page to Teleport docs")
}
logrus.Infof("APT repo build process completed in %s", time.Since(start).Round(time.Millisecond))
return nil
}
func (art *AptRepoTool) publishRepos() error {
// Pull in all Aptly repos, not just the latest ones to ensure they all get built into APT repos correctly
repos, err := art.aptly.GetAllRepos()
if err != nil {
return trace.Wrap(err, "failed to get all Aptly repos")
}
// Build a map keyed by os info with value of all repos that support the os in the key
// This will be used to structure the publish command
logrus.Debugf("Categorizing repos according to OS info: %v", RepoNames(repos))
categorizedRepos := make(map[string][]*Repo)
for _, r := range repos {
if osRepos, ok := categorizedRepos[r.OSInfo()]; ok {
categorizedRepos[r.OSInfo()] = append(osRepos, r)
} else {
categorizedRepos[r.OSInfo()] = []*Repo{r}
}
}
logrus.Debugf("Categorized repos: %v", categorizedRepos)
for osInfo, osRepoList := range categorizedRepos {
if len(osRepoList) < 1 {
continue
}
err := art.aptly.PublishRepos(osRepoList, osRepoList[0].os, osRepoList[0].osVersion)
if err != nil {
return trace.Wrap(err, "failed to publish for os %q", osInfo)
}
}
return nil
}
func (art *AptRepoTool) recreateExistingRepos(localPublishedPath string) ([]*Repo, error) {
logrus.Infoln("Recreating previously published repos...")
createdRepos, err := art.aptly.CreateReposFromPublishedPath(localPublishedPath)
if err != nil {
return nil, trace.Wrap(err, "failed to recreate existing repos")
}
for _, repo := range createdRepos {
err := art.aptly.ImportDebsFromExistingRepo(repo)
if err != nil {
return nil, trace.Wrap(err, "failed to import debs from existing repo %q", repo.Name())
}
}
logrus.Infof("Recreated and imported pre-existing artifacts for %d repos", len(createdRepos))
return createdRepos, nil
}
func (art *AptRepoTool) getArtifactRepos() ([]*Repo, error) {
logrus.Infoln("Creating or getting Aptly repos for artifact requirements...")
artifactRepos, err := art.aptly.CreateReposFromArtifactRequirements(art.supportedOSs,
art.config.releaseChannel, art.config.versionChannel)
if err != nil {
return nil, trace.Wrap(err, "failed to create or get repos from artifact requirements")
}
logrus.Infof("Created or got %d artifact Aptly repos", len(artifactRepos))
return artifactRepos, nil
}
func (art *AptRepoTool) importNewDebs(repos []*Repo) error {
logrus.Debugf("Importing new debs into %d repos: %q", len(repos), strings.Join(RepoNames(repos), "\", \""))
err := filepath.WalkDir(art.config.artifactPath,
func(debPath string, d fs.DirEntry, err error) error {
return art.importNewDebsWalker(debPath, d, err, repos)
},
)
if err != nil {
return trace.Wrap(err, "failed to find and import debs")
}
return nil
}
// This should not be used outside of importNewDebs
func (art *AptRepoTool) importNewDebsWalker(debPath string, d fs.DirEntry, err error, repos []*Repo) error {
if err != nil {
return trace.Wrap(err, "failure while searching %s for debs", debPath)
}
if d.IsDir() {
return nil
}
fileName := d.Name()
if filepath.Ext(fileName) != ".deb" {
return nil
}
// Import new artifacts into all repos that match the artifact's requirements
for _, repo := range repos {
// Other checks could be added here to ensure that a given deb gets added to the correct repo
// such as name or parent directory, facilitating os-specific artifacts
if repo.versionChannel != art.config.versionChannel || repo.releaseChannel != art.config.releaseChannel {
continue
}
err = art.aptly.ImportDeb(repo.Name(), debPath)
if err != nil {
return trace.Wrap(err, "failed to import deb from %s", debPath)
}
}
return nil
}
build.assets/tooling/cmd/build-os-package-repos/aptly.go
-676
View File
@@ -1,676 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
)
// This provides wrapper functions for the Aptly command. Aptly is written in Go but it doesn't appear
// to have a good binary API to use, only a CLI tool and REST API.
type Aptly struct {
rootDir string
}
// Instantiates Aptly, performing any system configuration needed.
func NewAptly(rootDir string) (*Aptly, error) {
a := &Aptly{
rootDir: rootDir,
}
err := a.ensureDefaultConfigExists()
if err != nil {
return nil, trace.Wrap(err, "failed to ensure Aptly default config exists")
}
err = a.updateConfiguration()
if err != nil {
return nil, trace.Wrap(err, "failed to load Aptly configuration")
}
return a, nil
}
func (*Aptly) ensureDefaultConfigExists() error {
// If the default config doesn't exist then it will be created the first time an Aptly command is
// ran, which messes up the output.
// Note: it is important to not use any repo-related commands here as they have a side effect of
// also creating the Aptly rootDir structure which is usually undesirable here
_, err := BuildAndRunCommand("aptly", "config", "show")
if err != nil {
return trace.Wrap(err, "failed to create default Aptly config")
}
return nil
}
func (a *Aptly) updateConfiguration() error {
aptlyConfigMap, err := loadAptlyConfigMap()
if err != nil {
return trace.Wrap(err, "failed to load Aptly config map")
}
// Additional config can be handled here if needed in the future
aptlyConfigMap["rootDir"] = a.rootDir
logrus.Debugf("Built Aptly config: %v", aptlyConfigMap)
saveAptlyConfigMap(aptlyConfigMap)
configOutput, err := BuildAndRunCommand("aptly", "config", "show")
if err != nil {
return trace.Wrap(err, "failed to check Aptly config")
}
logrus.Debugln("Aptly config on disk:")
logrus.Debugf("%v", configOutput)
return nil
}
func saveAptlyConfigMap(aptlyConfigMap map[string]interface{}) error {
aptlyConfigData, err := json.MarshalIndent(aptlyConfigMap, "", " ")
if err != nil {
return trace.Wrap(err, "failed to marshal Aptly config with data %v", aptlyConfigMap)
}
aptlyConfigPath, err := getAptlyConfigPath()
if err != nil {
return trace.Wrap(err, "failed to get Aptly config path")
}
err = os.WriteFile(aptlyConfigPath, aptlyConfigData, 0644)
if err != nil {
return trace.Wrap(err, "failed to write Aptly config to %q with data %q", aptlyConfigPath, string(aptlyConfigData))
}
return nil
}
func loadAptlyConfigMap() (map[string]interface{}, error) {
aptlyConfigPath, err := getAptlyConfigPath()
if err != nil {
return nil, trace.Wrap(err, "failed to get Aptly config path")
}
aptlyConfigData, err := os.ReadFile(aptlyConfigPath)
if err != nil {
return nil, trace.Wrap(err, "failed to read Aptly config file at %q", aptlyConfigPath)
}
var aptlyConfigMap map[string]interface{}
if err := json.Unmarshal(aptlyConfigData, &aptlyConfigMap); err != nil {
return nil, trace.Wrap(err, "failed to unmarshal Aptly config from file %q with data %q", aptlyConfigPath, string(aptlyConfigData))
}
return aptlyConfigMap, nil
}
// This follows the config logic specified here: https://www.aptly.info/doc/configuration/
func getAptlyConfigPath() (string, error) {
userHomeDir, err := os.UserHomeDir()
if err != nil {
return "", trace.Wrap(err, "failed to get user home directory path")
}
userAptlyConfigPath := path.Join(userHomeDir, ".aptly.conf")
_, err = os.Stat(userAptlyConfigPath)
if err == nil {
return userAptlyConfigPath, nil
}
if !errors.Is(err, os.ErrNotExist) {
return "", trace.Wrap(err, "failed to check if %q exists", userAptlyConfigPath)
}
systemAptlyConfigPath := "/etc/aptly.conf"
_, err = os.Stat(systemAptlyConfigPath)
if err == nil {
return systemAptlyConfigPath, nil
}
if !errors.Is(err, os.ErrNotExist) {
return "", trace.Wrap(err, "failed to check if %q exists", systemAptlyConfigPath)
}
return "", trace.Errorf("Aptly config not found at %q or %q", userAptlyConfigPath, systemAptlyConfigPath)
}
func (a *Aptly) IsFirstRun() (bool, error) {
_, err := os.Stat(a.rootDir)
if err == nil {
return false, nil
}
if errors.Is(err, os.ErrNotExist) {
return true, nil
}
return false, trace.Wrap(err, "failed to check if %q exists", a.rootDir)
}
// Creates the provided repo `r` via Aptly. Returns true if the repo was created, false otherewise.
func (a *Aptly) CreateRepoIfNotExists(r *Repo) (bool, error) {
logrus.Debugf("Creating repo %q if it doesn't already exist..", r.Name())
doesRepoExist, err := a.DoesRepoExist(r)
if err != nil {
return false, trace.Wrap(err, "failed to check whether or not the repo %q already exists", r.Name())
}
if doesRepoExist {
logrus.Debugf("Repo %q already exists, skipping creation", r.Name())
return false, nil
}
distributionArg := fmt.Sprintf("-distribution=%s", r.osVersion)
componentArg := fmt.Sprintf("-component=%s/%s", r.releaseChannel, r.versionChannel)
_, err = BuildAndRunCommand("aptly", "repo", "create", distributionArg, componentArg, r.Name())
if err != nil {
return false, trace.Wrap(err, "failed to create repo %q", r.Name())
}
logrus.Debugf("Created repo %q", r.Name())
return true, nil
}
// Checks to see if the Aptly described by repo `r` exists. Returns true if it exists, false otherwise.
func (a *Aptly) DoesRepoExist(r *Repo) (bool, error) {
logrus.Debugf("Checking if repo %q exists...", r.Name())
existingRepoNames, err := a.GetExistingRepoNames()
if err != nil {
return false, trace.Wrap(err, "failed to get existing repo names")
}
return slices.Contains(existingRepoNames, r.Name()), nil
}
// Gets a list of the name of Aptly repos that already exists.
func (a *Aptly) GetExistingRepoNames() ([]string, error) {
logrus.Debugln("Getting a list of pre-existing repos...")
// The output of the command will be simiar to:
// ```
// <repo name 1>
// ...
// <repo name N>
// ```
output, err := BuildAndRunCommand("aptly", "repo", "list", "-raw")
if err != nil {
return nil, trace.Wrap(err, "failed to get a list of existing repos")
}
// Split the command output by new line
parsedRepoNames := strings.Split(output, "\n")
// The names may have whitespace and the command may print an extra blank line, so we remove those here
var validRepoNames []string
for _, parsedRepoName := range parsedRepoNames {
if trimmedRepoName := strings.TrimSpace(parsedRepoName); trimmedRepoName != "" {
validRepoNames = append(validRepoNames, trimmedRepoName)
}
}
logrus.Debugf("Found %d repos: %q", len(validRepoNames), strings.Join(validRepoNames, "\", \""))
return validRepoNames, nil
}
// Imports a deb at `debPath` into the Aptly repo of name `repoName`.
// If `debPath` is a folder, the folder will be searched recursively for *.deb files
// which are then imported into the repo.
func (a *Aptly) ImportDeb(repoName string, debPath string) error {
logrus.Infof("Importing deb(s) from %q into repo %q...", debPath, repoName)
_, err := BuildAndRunCommand("aptly", "repo", "add", repoName, debPath)
if err != nil {
return trace.Wrap(err, "failed to add %q to repo %q", debPath, repoName)
}
return nil
}
// This function imports deb files from a preexisting published repo, typically created from a previous run of this tool.
func (a *Aptly) ImportDebsFromExistingRepo(repo *Repo) error {
logrus.Infof("Importing pre-existing debs from repo %q...", repo.Name())
publishedRepoAbsolutePath, err := repo.PublishedRepoAbsolutePath()
if err != nil {
return trace.Wrap(err, "failed to get the absolute path of the published repo %q", repo.Name())
}
logrus.Debugf("Looking in %q for Packages files...", publishedRepoAbsolutePath)
err = filepath.WalkDir(publishedRepoAbsolutePath,
func(packagesPath string, d fs.DirEntry, err error) error {
if err != nil {
return trace.Wrap(err, "failure while searching %s for Packages files", packagesPath)
}
if d.IsDir() {
return nil
}
if d.Name() != "Packages" {
return nil
}
logrus.Debugf("Matched %q as a Packages file, attempting to import listed debs into %q...", packagesPath, repo.Name())
err = a.importDebsFromPackagesFile(repo, packagesPath)
if err != nil {
return trace.Wrap(err, "failed to import debs into repo %q from packages file %q", repo.Name(), packagesPath)
}
return nil
},
)
if err != nil {
return trace.Wrap(err, "failed to find and import debs from existing repo %q at published path %q", repo.Name(), publishedRepoAbsolutePath)
}
return nil
}
func (a *Aptly) importDebsFromPackagesFile(repo *Repo, packagesPath string) error {
logrus.Debugf("Importing debs from %q into %q", packagesPath, repo.Name())
debRelativeFilePaths, err := parsePackagesFile(packagesPath)
if err != nil {
return trace.Wrap(err, "failed to parse packages file %q for deb file paths", packagesPath)
}
logrus.Debugf("Found %d debs listed in %q: %q", len(debRelativeFilePaths), packagesPath, strings.Join(debRelativeFilePaths, "\", \""))
for _, debRelativeFilePath := range debRelativeFilePaths {
debPath := path.Join(repo.publishedSourcePath, repo.os, debRelativeFilePath)
logrus.Debugf("Constructed deb absolute path %q", debPath)
err = a.ImportDeb(repo.Name(), debPath)
if err != nil {
return trace.Wrap(err, "failed to import deb into repo %q from %q", repo.Name(), debPath)
}
}
return nil
}
func parsePackagesFile(packagesPath string) ([]string, error) {
logrus.Debugf("Parsing packages file %q", packagesPath)
file, err := os.Open(packagesPath)
if err != nil {
logrus.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
debRelativeFilePaths := make([]string, 0, 1) // If the package file exists then it should contain at least one deb file path
for scanner.Scan() {
line := scanner.Text()
// Skip empty lines
if line == "" {
continue
}
key, value, err := parsePackagesFileLine(line)
if err != nil {
return nil, trace.Wrap(err, "failed to parse line %q in packages file %q", line, packagesPath)
}
if key != "Filename" {
continue
}
logrus.Debugf("Found deb file listed at relative path %q", value)
debRelativeFilePaths = append(debRelativeFilePaths, value)
}
if err := scanner.Err(); err != nil {
return nil, trace.Wrap(err, "received error while reading %q", packagesPath)
}
return debRelativeFilePaths, nil
}
func parsePackagesFileLine(line string) (string, string, error) {
splitLine := strings.SplitN(line, ": ", 2)
if len(splitLine) != 2 {
return "", "", trace.Errorf("packages file line %q is malformed", line)
}
key := splitLine[0]
value := splitLine[1]
if key == "" {
return "", "", trace.Errorf("packages file line %q contains an empty key", line)
}
if value == "" {
return "", "", trace.Errorf("packages file line %q contains an empty value", line)
}
return key, value, nil
}
// Publishes the Aptly repos defined in the `repos` slice to the `repoOS` subpath.
func (a *Aptly) PublishRepos(repos []*Repo, repoOS string, repoOSVersion string) error {
repoNames := RepoNames(repos)
logrus.Infof("Publishing repos for OS %q: %q...", repoOS, strings.Join(repoNames, "\", \""))
// Trying to publish to an already published OS/OS version will fail, and dropping a published
// OS/OS version when no new components (release channel/major version) are added is
// computationally expensive
areSomeUnpublished, areSomePublished, err := a.getRepoSlicePublishedState(repos)
if err != nil {
return trace.Wrap(err, "failed to determine if repos for have been published or not")
}
logrus.Debugln("Repo OS/OS version combo publish state:")
logrus.Debugf("Are some unpublished: %v", areSomeUnpublished)
logrus.Debugf("Are some published: %v", areSomePublished)
logrus.Debugf("Repos: %v", RepoNames(repos))
// If all repos have been published
if areSomePublished && !areSomeUnpublished {
// Update rather than republish
_, err := BuildAndRunCommand("aptly", "publish", "update", repoOSVersion, repoOS)
if err != nil {
return trace.Wrap(err, "failed to update publish repos with OS %q and OS version %q", repoOS, repoOSVersion)
}
return nil
}
// If some have been published and some have not
// This will occur if there is a new major release, a OS version is supported, or a new release channel is added
if areSomePublished && areSomeUnpublished {
// Drop the currently published APT repo so that it can be rebuilt from scratch
_, err := BuildAndRunCommand("aptly", "publish", "drop", repoOSVersion, repoOS)
if err != nil {
return trace.Wrap(err, "failed to update publish repos with OS %q and OS version %q", repoOS, repoOSVersion)
}
}
// If all repos have not been published (or were just unpublished/dropped)
// Build the command args and publish all the repos
args := []string{"publish", "repo"}
if len(repos) > 1 {
componentsArgument := fmt.Sprintf("-component=%s", strings.Repeat(",", len(repos)-1))
args = append(args, componentsArgument)
}
args = append(args, repoNames...)
args = append(args, repoOS)
// Full command is `aptly publish repo -component=<, repeating len(repos) - 1 times> <repo names> <repo OS>`
_, err = BuildAndRunCommand("aptly", args...)
if err != nil {
return trace.Wrap(err, "failed to publish repos")
}
return nil
}
// This function determines if `repos` contains repos that have an OS/OS version combo
// that have not yet been published, and if it contains repos that have an OS/OS version
// combo that have been published.
//
// Returns:
//
// 1. true if `repos` contains at least one repo who's OS/OS version combo has not been
// published yet
//
// 2. true if `repos` contains at least one repo who's OS/OS version combo has been
// published
//
// Example:
//
// `getRepoSlicePublishedState([<repo for debian-trixie-stable-v6>,
// <repo for debian-buster-stable-v7>])` will return
//
// 1. `true, false, nil` if a repo for debian/trixie, debian/buster has not been published yet
//
// 2. `false, true, nil` if a repo for debian/trixie, debian/buster has been published
//
// 3. `true, true, nil` if a repo for debian/trixie has not been published yet but debian/buster
// has, or vice versa
func (a *Aptly) getRepoSlicePublishedState(repos []*Repo) (bool, bool, error) {
publishedRepoNames, err := a.GetPublishedRepoNames()
if err != nil {
return false, false, trace.Wrap(err, "failed to get a list of published repos' names")
}
containsUnpublishedRepo := false
containsPublishedRepo := false
for _, repo := range repos {
repoName := repo.Name()
hasRepoBeenPublished := slices.Contains(publishedRepoNames, repoName)
logrus.Debugf("Repo %q has been published: %v", repoName, hasRepoBeenPublished)
containsUnpublishedRepo = containsUnpublishedRepo || !hasRepoBeenPublished
containsPublishedRepo = containsPublishedRepo || hasRepoBeenPublished
// No need to keep checking if they're both already true
if containsUnpublishedRepo && containsPublishedRepo {
break
}
}
// failsafe in case of some underlying logic change
if !containsUnpublishedRepo && !containsPublishedRepo {
return false, false, trace.Errorf("something went very wrong here")
}
return containsUnpublishedRepo, containsPublishedRepo, nil
}
func (a *Aptly) GetPublishedRepoNames() ([]string, error) {
logrus.Debugln("Getting a list of published repos...")
// The output of the command will be simiar to:
// ```
// Published repositories:
// * <os 1>/<osVersion 1> [<APT repo supported arch 1>, ..., <APT repo supported arch M>] publishes {<APT repo component 1>: [<Aptly repo name>]}, ..., {<APT repo component N>: [<Aptly repo name>]}
// ...
// * <os 1>/<osVersion J> [<APT repo supported arch 1>, ..., <APT repo supported arch M>] publishes {<APT repo component 1>: [<Aptly repo name>]}, ..., {<APT repo component N>: [<Aptly repo name>]}
// * <os 2>/<osVersion 1> [<APT repo supported arch 1>, ..., <APT repo supported arch M>] publishes {<APT repo component 1>: [<Aptly repo name>]}, ..., {<APT repo component N>: [<Aptly repo name>]}
// ...
// * <os I>/<osVersion J> [<APT repo supported arch 1>, ..., <APT repo supported arch M>] publishes {<APT repo component 1>: [<Aptly repo name>]}, ..., {<APT repo component N>: [<Aptly repo name>]}
// ```
//
// If no repos have been published then the output will be similar to:
// ```
// No snapshots/local repos have been published. Publish a snapshot by running `aptly publish snapshot ...`.
// ```
// Note that the `-raw` argument is not used here as it does not provide sufficient information
output, err := BuildAndRunCommand("aptly", "publish", "list")
if err != nil {
return nil, trace.Wrap(err, "failed to get a list of published repos")
}
// Split the command output by new line
publishedRepoLines := strings.Split(output, "\n")
// In all cases the first line should exist and not be parsed
publishedRepoLines = publishedRepoLines[1:]
repoNameRegexStr := ": \\[(.+?)\\]"
repoNameRegex, err := regexp.Compile(repoNameRegexStr)
if err != nil {
return nil, trace.Wrap(err, "failed to compile repo name regex %q", repoNameRegexStr)
}
var publishedRepoNames []string
for _, publishedRepoLine := range publishedRepoLines {
// The names may have whitespace and the command may print an extra blank line, so we remove those here
if trimmedRepoLine := strings.TrimSpace(publishedRepoLine); trimmedRepoLine != "" {
// Additional parsing should go here if needed in the future
repoNameMatches := repoNameRegex.FindAllStringSubmatch(publishedRepoLine, -1)
if repoNameMatches == nil {
return nil, trace.Errorf("failed to match repo names in line %q with regex %q", publishedRepoLine, repoNameRegexStr)
}
for _, repoNameMatch := range repoNameMatches {
// `repoNameRegexStr` is written such that there will be exactly one match and one group in repoNameMatch
// for example repoNameMatch could be [": [debian-bookworm-stable-v6]", "debian-bookworm-stable-v6"]
publishedRepoName := repoNameMatch[1]
publishedRepoNames = append(publishedRepoNames, publishedRepoName)
}
}
}
logrus.Debugf("Found %d published repos: %q", len(publishedRepoNames), publishedRepoNames)
return publishedRepoNames, nil
}
// Creates Aptly repos from a local path that has previously published Apt repos created by this tool.
// Returns a list of repo objects describing the created Aptly repos.
func (a *Aptly) CreateReposFromPublishedPath(localPublishedPath string) ([]*Repo, error) {
// The file tree that we care about here will be of the following structure:
// `/<bucketPath>/<os>/dists/<os version>/<release channel>/<major version>/...`
logrus.Infof("Recreating previously published repos from %q...", localPublishedPath)
createdRepos := []*Repo{}
osSubdirectories, err := getSubdirectories(localPublishedPath)
if err != nil {
return nil, trace.Wrap(err, "failed to get OS subdirectories in %s", localPublishedPath)
}
for _, os := range osSubdirectories {
osVersionParentDirectory := filepath.Join(localPublishedPath, os, "dists")
osVersionSubdirectories, err := getSubdirectories(osVersionParentDirectory)
if err != nil {
return nil, trace.Wrap(err, "failed to get OS version subdirectories in %s", localPublishedPath)
}
for _, osVersion := range osVersionSubdirectories {
releaseChannelParentDirectory := filepath.Join(osVersionParentDirectory, osVersion)
releaseChannelSubdirectories, err := getSubdirectories(releaseChannelParentDirectory)
if err != nil {
return nil, trace.Wrap(err, "failed to get release channel subdirectories in %s", localPublishedPath)
}
for _, releaseChannel := range releaseChannelSubdirectories {
versionChannelParentDirectory := filepath.Join(releaseChannelParentDirectory, releaseChannel)
versionChannelSubdirectories, err := getSubdirectories(versionChannelParentDirectory)
if err != nil {
return nil, trace.Wrap(err, "failed to get version channel subdirectories in %s", localPublishedPath)
}
for _, versionChannel := range versionChannelSubdirectories {
r := &Repo{
os: os,
osVersion: osVersion,
releaseChannel: releaseChannel,
versionChannel: versionChannel,
publishedSourcePath: localPublishedPath,
}
wasRepoCreated, err := a.CreateRepoIfNotExists(r)
if err != nil {
return nil, trace.Wrap(err, "failed to create repo %q", r.Name())
}
if wasRepoCreated {
createdRepos = append(createdRepos, r)
}
}
}
}
}
logrus.Infof("Recreated %d repos", len(createdRepos))
return createdRepos, nil
}
// Creates or gets Aptly repos for all permutations of the provided requirements.
// Returns a list of repo objects describing the Aptly repos, regardless of if they
// already existed.
//
// supportedOSInfo should be a dictionary keyed by OS name, with values being a list of
// supported OS version codenames.
func (a *Aptly) CreateReposFromArtifactRequirements(supportedOSInfo map[string][]string,
releaseChannel string, versionChannel string) ([]*Repo, error) {
logrus.Infoln("Creating new repos from artifact requirements:")
logrus.Infof("Supported OSs: %+v", supportedOSInfo)
logrus.Infof("Release channel: %q", releaseChannel)
logrus.Infof("Version channel: %q", versionChannel)
artifactRequirementRepos := []*Repo{}
for os, osVersions := range supportedOSInfo {
for _, osVersion := range osVersions {
r := &Repo{
os: os,
osVersion: osVersion,
releaseChannel: releaseChannel,
versionChannel: versionChannel,
}
_, err := a.CreateRepoIfNotExists(r)
if err != nil {
return nil, trace.Wrap(err, "failed to create repo %q", r.Name())
}
artifactRequirementRepos = append(artifactRequirementRepos, r)
}
}
return artifactRequirementRepos, nil
}
// Returns a list of all Aptly reported repos
func (a *Aptly) GetAllRepos() ([]*Repo, error) {
repoNames, err := a.GetExistingRepoNames()
if err != nil {
return nil, trace.Wrap(err, "failed to get existing repo names")
}
repos := make([]*Repo, len(repoNames))
for i, repoName := range repoNames {
repo, err := NewRepoFromName(repoName)
if err != nil {
return nil, trace.Wrap(err, "failed to build repo struct for repo name %q", repoName)
}
repos[i] = repo
}
return repos, nil
}
func getSubdirectories(basePath string) ([]string, error) {
logrus.Debugf("Getting subdirectories of %q...", basePath)
files, err := os.ReadDir(basePath)
if err != nil {
return nil, trace.Wrap(err, "failed to read directory %q", basePath)
}
subdirectories := []string{}
for _, file := range files {
if !file.IsDir() {
continue
}
subdirectory := file.Name()
logrus.Debugf("Found subdirectory %q", subdirectory)
subdirectories = append(subdirectories, subdirectory)
}
return subdirectories, nil
}
build.assets/tooling/cmd/build-os-package-repos/command_executor.go
-53
View File
@@ -1,53 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"errors"
"os/exec"
"strings"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
)
// Builds an runs a command with the provided arguments. Extensively logs command
// details to the debug log. Returns stdout and stderr combined, along with an
// error iff one occurred.
func BuildAndRunCommand(command string, args ...string) (string, error) {
cmd := exec.Command(command, args...)
logrus.Debugf("Running command \"%s '%s'\"", command, strings.Join(args, "' '"))
output, err := cmd.CombinedOutput()
if output != nil {
logrus.Debugf("Command output: %s", string(output))
}
if err != nil {
var exitError *exec.ExitError
if errors.As(err, &exitError) {
exitCode := exitError.ExitCode()
logrus.Debugf("Command exited with exit code %d", exitCode)
} else {
logrus.Debugln("Command failed without an exit code")
}
return "", trace.Wrap(err, "Command failed, see debug output for additional details")
}
logrus.Debugln("Command exited successfully")
return string(output), nil
}
build.assets/tooling/cmd/build-os-package-repos/config.go
-282
View File
@@ -1,282 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"flag"
"os"
"strings"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
)
const StableChannelFlagValue string = "stable"
type LoggerConfig struct {
logLevel uint
logJSON bool
}
func NewLoggerConfigWithFlagset(fs *flag.FlagSet) *LoggerConfig {
lc := &LoggerConfig{}
fs.UintVar(&lc.logLevel, "log-level", uint(logrus.InfoLevel), "Log level from 0 to 6, 6 being the most verbose")
fs.BoolVar(&lc.logJSON, "log-json", false, "True if the log entries should use JSON format, false for text logging")
return lc
}
func (lc *LoggerConfig) Check() error {
if err := lc.validateLogLevel(); err != nil {
return trace.Wrap(err, "failed to validate the log level flag")
}
return nil
}
func (lc *LoggerConfig) validateLogLevel() error {
if lc.logLevel > 6 {
return trace.BadParameter("the log-level flag should be between 0 and 6")
}
return nil
}
type S3Config struct {
bucketName string
localBucketPath string
maxConcurrentSyncs int
}
func NewS3ConfigWithFlagset(fs *flag.FlagSet) *S3Config {
s3c := &S3Config{}
fs.StringVar(&s3c.bucketName, "bucket", "", "The name of the S3 bucket where the repo should be synced to/from")
fs.StringVar(&s3c.localBucketPath, "local-bucket-path", "/bucket", "The local path where the bucket should be synced to")
fs.IntVar(&s3c.maxConcurrentSyncs, "max-concurrent-syncs", 16, "The maximum number of S3 bucket syncs that may run in parallel (-1 for unlimited, 16 default)")
return s3c
}
func (s3c *S3Config) Check() error {
if err := s3c.validateBucketName(); err != nil {
return trace.Wrap(err, "failed to validate the bucket name flag")
}
if err := s3c.validateLocalBucketPath(); err != nil {
return trace.Wrap(err, "failed to validate the local bucket path flag")
}
if err := s3c.validateMaxConcurrentSyncs(); err != nil {
return trace.Wrap(err, "failed to validate the max concurrent syncs flag")
}
return nil
}
func (s3c *S3Config) validateBucketName() error {
if s3c.bucketName == "" {
return trace.BadParameter("the bucket flag should not be empty")
}
return nil
}
func (s3c *S3Config) validateLocalBucketPath() error {
if s3c.localBucketPath == "" {
return trace.BadParameter("the local-bucket-path flag should not be empty")
}
if stat, err := os.Stat(s3c.localBucketPath); err == nil && !stat.IsDir() {
return trace.BadParameter("the local bucket path points to a file instead of a directory")
}
return nil
}
func (s3c *S3Config) validateMaxConcurrentSyncs() error {
if s3c.maxConcurrentSyncs < -1 {
return trace.BadParameter("the max-concurrent-syncs flag must be greater than -1")
}
return nil
}
// This type is common to all other config types
type Config struct {
*LoggerConfig
*S3Config
artifactPath string
printHelp bool
releaseChannel string
versionChannel string
}
func NewConfigWithFlagset(fs *flag.FlagSet) *Config {
c := &Config{}
c.LoggerConfig = NewLoggerConfigWithFlagset(fs)
c.S3Config = NewS3ConfigWithFlagset(fs)
fs.StringVar(&c.artifactPath, "artifact-path", "/artifacts", "Path to the filesystem tree containing the *.deb or *.rpm files to add to the repos")
fs.Visit(func(f *flag.Flag) {
if f.Name == "-h" || f.Name == "--help" {
c.printHelp = true
}
})
fs.StringVar(&c.releaseChannel, "release-channel", "", "The release channel of the repos that the artifacts should be added to")
fs.StringVar(&c.versionChannel, "version-channel", "", "The version channel of the repos that the artifacts should be added to. Semver values will be truncated to major version. Examples: \"v1.2.3\" (truncated to \"v1\"), \"cloud\".")
return c
}
func (c *Config) Check() error {
if err := c.LoggerConfig.Check(); err != nil {
return trace.Wrap(err, "failed to validate logger config")
}
if err := c.S3Config.Check(); err != nil {
return trace.Wrap(err, "failed to validate S3 config")
}
if err := c.validateArtifactPath(); err != nil {
return trace.Wrap(err, "failed to validate the artifact path flag")
}
if err := c.validateVersionChannel(); err != nil {
return trace.Wrap(err, "failed to validate the version channel flag")
}
if err := c.validateReleaseChannel(); err != nil {
return trace.Wrap(err, "failed to validate the release channel flag")
}
return nil
}
func (c *Config) validateArtifactPath() error {
if c.artifactPath == "" {
return trace.BadParameter("the artifact-path flag should not be empty")
}
if stat, err := os.Stat(c.artifactPath); os.IsNotExist(err) {
return trace.BadParameter("the artifact-path %q does not exist", c.artifactPath)
} else if !stat.IsDir() {
return trace.BadParameter("the artifact-path %q is not a directory", c.artifactPath)
}
return nil
}
func (c *Config) validateVersionChannel() error {
if c.versionChannel == "" {
return trace.BadParameter("the version-channel flag should not be empty")
}
if semver.IsValid(c.versionChannel) {
c.versionChannel = semver.Major(c.versionChannel)
}
return nil
}
func (c *Config) validateReleaseChannel() error {
if c.releaseChannel == "" {
return trace.BadParameter("the release-channel flag should not be empty")
}
// Not sure what other channels we'd want to support, but they should be listed here
validReleaseChannels := []string{StableChannelFlagValue}
for _, validReleaseChannel := range validReleaseChannels {
if c.releaseChannel == validReleaseChannel {
return nil
}
}
return trace.BadParameter("the release channel contains an invalid value. Valid values are: %s", strings.Join(validReleaseChannels, ","))
}
// APT-specific config
type AptConfig struct {
*Config
aptlyPath string
}
func NewAptConfigWithFlagSet(fs *flag.FlagSet) (*AptConfig, error) {
ac := &AptConfig{}
ac.Config = NewConfigWithFlagset(fs)
homeDir, err := os.UserHomeDir()
if err != nil {
return nil, trace.Wrap(err, "failed to get user's home directory path")
}
fs.StringVar(&ac.aptlyPath, "aptly-root-dir", homeDir, "The Aptly \"rootDir\" (see https://www.aptly.info/doc/configuration/ for details)")
return ac, nil
}
func (ac *AptConfig) validateAptlyPath() error {
if ac.aptlyPath == "" {
return trace.BadParameter("the aptly-root-dir flag should not be empty")
}
return nil
}
func (ac *AptConfig) Check() error {
if err := ac.Config.Check(); err != nil {
return trace.Wrap(err, "failed to validate common config")
}
if err := ac.validateAptlyPath(); err != nil {
return trace.Wrap(err, "failed to validate the aptly-root-dir path flag")
}
return nil
}
// YUM-specific config
type YumConfig struct {
*Config
cacheDir string
}
func NewYumConfigWithFlagSet(fs *flag.FlagSet) *YumConfig {
yc := &YumConfig{}
yc.Config = NewConfigWithFlagset(fs)
fs.StringVar(&yc.cacheDir, "cache-dir", "/tmp/createrepo/cache", "The createrepo checksum caching directory (see https://linux.die.net/man/8/createrepo for details")
return yc
}
func (yc *YumConfig) validateCacheDir() error {
if yc.cacheDir == "" {
return trace.BadParameter("the cache-dir flag should not be empty")
}
return nil
}
func (yc *YumConfig) Check() error {
if err := yc.Config.Check(); err != nil {
return trace.Wrap(err, "failed to validate common config")
}
if err := yc.validateCacheDir(); err != nil {
return trace.Wrap(err, "failed to validate the cache-dir path flag")
}
return nil
}
build.assets/tooling/cmd/build-os-package-repos/createrepo.go
-83
View File
@@ -1,83 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"os"
"os/exec"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
)
type CreateRepo struct {
cacheDir string
binaryName string
}
// Instantiates createrepo, ensuring all system requirements for performing createrepo operations
// have been met
func NewCreateRepo(cacheDir string) (*CreateRepo, error) {
cr := &CreateRepo{
cacheDir: cacheDir,
// `createrepo_c` is the "new" (as in 9 years old) replacement for `createrepo`
// This can be replace with "createrepo" in the unlikely chance that there is
// a problem
binaryName: "createrepo_c",
}
err := cr.ensureBinaryExists()
if err != nil {
return nil, trace.Wrap(err, "failed to ensure CreateRepo binary exists")
}
// Ensure the cache dir exists
err = os.MkdirAll(cr.cacheDir, 0660)
if err != nil {
return nil, trace.Wrap(err, "failed to ensure %q exists", cr.cacheDir)
}
return cr, nil
}
func (cr *CreateRepo) ensureBinaryExists() error {
_, err := exec.LookPath(cr.binaryName)
if err != nil {
return trace.Wrap(err, "failed to verify that %q binary exists", cr.binaryName)
}
return nil
}
func (cr *CreateRepo) CreateOrUpdateRepo(repoPath string) error {
// <cr.binaryName> --cachedir <cr.cacheDir> --update <repoPath>
logrus.Debugf("Updating repo metadata for repo at %q", repoPath)
args := []string{
"--cachedir",
cr.cacheDir,
"--update",
repoPath,
}
_, err := BuildAndRunCommand(cr.binaryName, args...)
if err != nil {
return trace.Wrap(err, "createrepo create/update command failed on path %q with cache directory %q", repoPath, cr.cacheDir)
}
return nil
}
build.assets/tooling/cmd/build-os-package-repos/gpg.go
-119
View File
@@ -1,119 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"os"
"strings"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
)
type GPG struct{}
// Instantiates GPG, ensuring all system requirements for using GPG are fulfilled
func NewGPG() (*GPG, error) {
g := &GPG{}
err := g.ensureFirstRunHasOccurred()
if err != nil {
return nil, trace.Wrap(err, "failed to setup GPG")
}
err = g.ensureSecretKeyExists()
if err != nil {
return nil, trace.Wrap(err, "failed to ensure a secret key exists")
}
return g, nil
}
// The first time GPG is run for a user with any "meaningful" arguments it will
// generate several files and log what it created to stdout. These logs can
// disrupt parsing of GPG command outputs, so we force it to happen here, once,
// rather than try and handle it on each GPG call.
func (*GPG) ensureFirstRunHasOccurred() error {
_, err := BuildAndRunCommand("gpg", "--fingerprint")
if err != nil {
return trace.Wrap(err, "failed to ensure GPG has been ran once")
}
return nil
}
func (*GPG) ensureSecretKeyExists() error {
output, err := BuildAndRunCommand("gpg", "--list-secret-keys", "--with-colons")
if err != nil {
return trace.Wrap(err, "failed to ensure GPG secret key exists")
}
outputLineCount := strings.Count(output, "\n")
if outputLineCount < 1 {
return trace.Errorf("failed to find a GPG secret key")
}
return nil
}
// Creates a detached, armored signature for the provided file using the default GPG key
func (*GPG) SignFile(filePath string) error {
// While this could be done via a Go module, the x/crypto/openpgp library has been frozen
// and deprecated for almost 18 months. Others exist, but given the security implications of
// using a less reputable Go module I've decided to just call `gpg` via shell instead.
// Additionally this works and is just _so easy_ that it's probably not worth the effort to
// use another library that reinvents the wheel.
logrus.Debugf("Signing repo metadata at %q", filePath)
// gpg --batch --yes --detach-sign --armor <filePath>
_, err := BuildAndRunCommand("gpg", "--batch", "--yes", "--detach-sign", "--armor", filePath)
if err != nil {
return trace.Wrap(err, "failed to run GPG signing command on %q", filePath)
}
return nil
}
// Get the armored default public GPG key, ready to be written to a file
func (*GPG) GetPublicKey() (string, error) {
// For reference here is how another company formats their key:
// https://download.docker.com/linux/rhel/gpg
logrus.Debug("Attempting to get the default public GPG key")
key, err := BuildAndRunCommand("gpg", "--export", "--armor", "--no-version")
if err != nil {
return "", trace.Wrap(err, "failed to export the default public GPG key")
}
return key, nil
}
func (g *GPG) WritePublicKeyToFile(filePath string) error {
logrus.Debugf("Writing the default armored public GPG key to %q", filePath)
key, err := g.GetPublicKey()
if err != nil {
return trace.Wrap(err, "failed to retrieve public key")
}
err = os.WriteFile(filePath, []byte(key), 0664)
if err != nil {
return trace.Wrap(err, "failed to write key to %q", filePath)
}
return nil
}
build.assets/tooling/cmd/build-os-package-repos/main.go
-119
View File
@@ -1,119 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"os"
"strings"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
)
func main() {
err := run()
if err != nil {
logrus.Fatal(err.Error())
}
}
func buildSubcommandRunners() ([]Runner, error) {
ar, err := NewAptRunner()
if err != nil {
return nil, trace.Wrap(err, "failed to instantiate new APT runner")
}
yr, err := NewYumRunner()
if err != nil {
return nil, trace.Wrap(err, "failed to instantiate new YUM runner")
}
// These should be sorted alphabetically by `Name()`
return []Runner{
*ar,
*yr,
}, nil
}
func run() error {
subcommands, err := buildSubcommandRunners()
if err != nil {
return trace.Wrap(err, "failed to build subcommand runners")
}
// 2 = program name + subcommand
if len(os.Args) < 2 {
logHelp(subcommands)
return trace.Errorf("subcommand not provided")
}
subcommandName := strings.ToLower(os.Args[1])
for _, subcommand := range subcommands {
if strings.ToLower(subcommandName) != subcommand.Name() {
continue
}
// 2 = program name + subcommand, skip them and get subcommand arguments
args := os.Args[2:]
err := subcommand.Init(args)
if err != nil {
return trace.Wrap(err, "failed to initialize runner for subcommand %q", subcommandName)
}
setupLogger(subcommand.GetLoggerConfig())
err = subcommand.Run()
if err != nil {
return trace.Wrap(err, "failed to run subcommand %q", subcommandName)
}
return nil
}
if subcommandName == "-h" {
logHelp(subcommands)
return nil
}
logHelp(subcommands)
return trace.Errorf("no subcommands found matching %q", subcommandName)
}
func logHelp(subcommands []Runner) {
executableName := os.Args[0]
fmt.Printf("%s - OS package repo builder/updater\n", executableName)
fmt.Println()
fmt.Println("Commands:")
fmt.Println()
for _, subcommand := range subcommands {
fmt.Printf("\t%s\t%s\n", subcommand.Name(), subcommand.Info())
}
fmt.Println()
fmt.Printf("Use \"%s <command> -h\" for more information about a command.\n", executableName)
fmt.Println()
}
func setupLogger(config *LoggerConfig) {
if config.logJSON {
logrus.SetFormatter(&logrus.JSONFormatter{})
} else {
logrus.SetFormatter(&logrus.TextFormatter{})
}
logrus.SetOutput(os.Stdout)
logrus.SetLevel(logrus.Level(config.logLevel))
logrus.Debugf("Setup logger with config: %+v", config)
}
build.assets/tooling/cmd/build-os-package-repos/repo.go
-103
View File
@@ -1,103 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/gravitational/trace"
)
type Repo struct {
os string
osVersion string
releaseChannel string
versionChannel string
publishedSourcePath string
}
// Returns a unique name for the repo.
func (r *Repo) Name() string {
return fmt.Sprintf("%s-%s-%s-%s", r.os, r.osVersion, r.releaseChannel, r.versionChannel)
}
func NewRepoFromName(name string) (*Repo, error) {
splitName := strings.Split(name, "-")
if len(splitName) != 4 {
return nil, trace.Errorf("the provided repo name %q is not a valid repo name", name)
}
return &Repo{
os: splitName[0],
osVersion: splitName[1],
releaseChannel: splitName[2],
versionChannel: splitName[3],
}, nil
}
// Returns the APT component to be associated with all debs in the Aptly repo.
func (r *Repo) Component() string {
return fmt.Sprintf("%s/%s", r.releaseChannel, r.versionChannel)
}
// Returns a string that identifies the specific OS and OS version the Aptly repo targets.
func (r *Repo) OSInfo() string {
return fmt.Sprintf("%s/%s", r.os, r.osVersion)
}
// Returns true if the repo is a recreation of a published repo, false otherewise.
// If true, publishedSourcePath is a valid existing path on the filesystem.
func (r *Repo) WasCreatedFromPublishedSource() (bool, error) {
if r.publishedSourcePath == "" {
return false, nil
}
_, err := os.Stat(r.publishedSourcePath)
if os.IsNotExist(err) {
return false, trace.Errorf("the published source path of repo %q was not empty (%q), but does not exist on disk", r.Name(), r.publishedSourcePath)
}
return true, nil
}
// Returns the absolute path to the published repo on disk that this repo was created from.
func (r *Repo) PublishedRepoAbsolutePath() (string, error) {
wasCreatedFromPublishedSource, err := r.WasCreatedFromPublishedSource()
if err != nil {
return "", trace.Wrap(err, "failed to verify if the repo %q is a recreation of an existing published repo", r.Name())
}
if !wasCreatedFromPublishedSource {
return "", trace.Errorf("repo %q was not created from a publish source and therefor has no published source path", r.Name())
}
// `/<publishedSourcePath>/<os>/dists/<os version>/<release channel>/<major version>/`
return filepath.Join(r.publishedSourcePath, r.os, "dists", r.osVersion, r.releaseChannel, r.versionChannel), nil
}
// Helper function that calls `Name()` on all repos in the provided list.
func RepoNames(repos []*Repo) []string {
repoNames := make([]string, len(repos))
for i, repo := range repos {
repoNames[i] = repo.Name()
}
return repoNames
}
build.assets/tooling/cmd/build-os-package-repos/runners.go
-203
View File
@@ -1,203 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"flag"
"github.com/gravitational/trace"
)
// Pattern from https://www.digitalocean.com/community/tutorials/how-to-use-the-flag-package-in-go
type Runner interface {
Init([]string) error
Run() error
GetLoggerConfig() *LoggerConfig
Name() string
Info() string
}
// APT implementation
type AptRunner struct {
flags *flag.FlagSet
config *AptConfig
supportedOSs map[string][]string
}
func NewAptRunner() (*AptRunner, error) {
runner := &AptRunner{
supportedOSs: map[string][]string{
// When adding a new supportedOS, update the lib/web/scipts/node-join/install.sh script
// Otherwise, it will keep using the binary installation instead of the deb repo.
"debian": { // See https://wiki.debian.org/DebianReleases#Production_Releases for details
"stretch", // 9
"buster", // 10
"bullseye", // 11
"bookworm", // 12
"trixie", // 13
},
"ubuntu": { // See https://wiki.ubuntu.com/Releases for details
"xenial", // 16.04 LTS
"yakkety", // 16.10 (EOL)
"zesty", // 17.04 (EOL)
"artful", // 17.10 (EOL)
"bionic", // 18.04 LTS
"cosmic", // 18.10 (EOL)
"disco", // 19.04 (EOL)
"eoan", // 19.10 (EOL)
"focal", // 20.04 LTS
"groovy", // 20.10 (EOL)
"hirsuite", // 21.04 (EOL)
"impish", // 21.10 (EOL)
"jammy", // 22.04 LTS
},
},
}
runner.flags = flag.NewFlagSet(runner.Name(), flag.ExitOnError)
config, err := NewAptConfigWithFlagSet(runner.flags)
if err != nil {
return nil, trace.Wrap(err, "failed to create a new APT config instance")
}
runner.config = config
return runner, nil
}
func (ar AptRunner) Init(args []string) error {
err := ar.flags.Parse(args)
if err != nil {
return trace.Wrap(err, "failed to parse arguments")
}
err = ar.config.Check()
if err != nil {
return trace.Wrap(err, "failed to validate APT config arguments")
}
return nil
}
func (ar AptRunner) Run() error {
if ar.config.printHelp {
ar.flags.Usage()
return nil
}
art, err := NewAptRepoTool(ar.config, ar.supportedOSs)
if err != nil {
return trace.Wrap(err, "failed to create a new APT repo tool instance")
}
err = art.Run()
if err != nil {
return trace.Wrap(err, "APT runner failed")
}
return nil
}
func (AptRunner) Name() string {
return "apt"
}
func (AptRunner) Info() string {
return "builds APT repos"
}
func (ar AptRunner) GetLoggerConfig() *LoggerConfig {
return ar.config.LoggerConfig
}
// YUM implementation
type YumRunner struct {
flags *flag.FlagSet
config *YumConfig
supportedOSs map[string][]string
}
func NewYumRunner() (*YumRunner, error) {
runner := &YumRunner{
supportedOSs: map[string][]string{
// When adding a new supportedOS, update the lib/web/scipts/node-join/install.sh script
// Otherwise, it will keep using the binary installation instead of the yum repo.
"rhel": { // See https://access.redhat.com/articles/3078 for details
"7",
"8",
"9",
},
"centos": { // See https://endoflife.date/centos for details
"7",
"8",
"9",
},
// "$releasever" is a hot mess for Amazon Linux. No good documentation on this outside of just running
// a container or EC2 instance and manually checking $releasever values
"amzn": {
// "latest" // 1, aka 2018.03.0.20201028.0
"2", // 2, aka 2.0.20201111.0
// "2022.0.20220531" // 2022 (new naming scheme, preview) aka 2022.0.20220531
},
},
}
runner.flags = flag.NewFlagSet(runner.Name(), flag.ExitOnError)
runner.config = NewYumConfigWithFlagSet(runner.flags)
return runner, nil
}
func (yr YumRunner) Init(args []string) error {
err := yr.flags.Parse(args)
if err != nil {
return trace.Wrap(err, "failed to parse arguments")
}
err = yr.config.Check()
if err != nil {
return trace.Wrap(err, "failed to validate YUM config arguments")
}
return nil
}
func (yr YumRunner) Run() error {
yrt, err := NewYumRepoTool(yr.config, yr.supportedOSs)
if err != nil {
return trace.Wrap(err, "failed to create a new YUM repo tool instance")
}
err = yrt.Run()
if err != nil {
return trace.Wrap(err, "YUM runner failed")
}
return nil
}
func (YumRunner) Name() string {
return "yum"
}
func (YumRunner) Info() string {
return "builds YUM repos"
}
func (yr YumRunner) GetLoggerConfig() *LoggerConfig {
return yr.config.LoggerConfig
}
build.assets/tooling/cmd/build-os-package-repos/s3logger.go
-37
View File
@@ -1,37 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"github.com/sirupsen/logrus"
)
// This maps the s3sync logging functions to logrus so that they match the
// rest of the logging framework's settings.
// Their docs outline the custom logger configuration here:
// https://github.com/seqsense/s3sync#sets-the-custom-logger
type s3logger struct{}
// Maps the s3sync log function to logrus.
func (s3logger) Log(v ...interface{}) {
logrus.Debugln(v...)
}
// Maps the s3sync logf function to logrus.
func (s3logger) Logf(format string, v ...interface{}) {
logrus.Debugf(format, v...)
}
build.assets/tooling/cmd/build-os-package-repos/s3manager.go
-511
View File
@@ -1,511 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"bytes"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/gravitational/trace"
"github.com/inhies/go-bytesize"
"github.com/seqsense/s3sync"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
)
type S3manager struct {
syncManager *s3sync.Manager
uploader *s3manager.Uploader
downloader *s3manager.Downloader
bucketLocalPath string
bucketName string
bucketURL *url.URL
maxConcurrentSyncs int
downloadedBytes int64
}
func NewS3Manager(config *S3Config) (*S3manager, error) {
// Right now the AWS session is only used by this manager, but if it ends
// up being needed elsewhere then it should probably be moved to an arg
awsSession, err := session.NewSession()
if err != nil {
return nil, trace.Wrap(err, "failed to create a new AWS session")
}
syncManagerMaxConcurrentSyncs := config.maxConcurrentSyncs
if syncManagerMaxConcurrentSyncs < 0 {
// This isn't unlimited but due to the s3sync library's parallelism implementation
// this must be limited to a "reasonable" number
syncManagerMaxConcurrentSyncs = 128
}
s := &S3manager{
bucketName: config.bucketName,
bucketURL: &url.URL{
Scheme: "s3",
Host: config.bucketName,
},
syncManager: s3sync.New(awsSession, s3sync.WithParallel(syncManagerMaxConcurrentSyncs)),
uploader: s3manager.NewUploader(awsSession),
downloader: s3manager.NewDownloader(awsSession),
maxConcurrentSyncs: config.maxConcurrentSyncs,
}
s.ChangeLocalBucketPath(config.localBucketPath)
s3sync.SetLogger(&s3logger{})
return s, nil
}
func (s *S3manager) ChangeLocalBucketPath(newBucketPath string) error {
s.bucketLocalPath = newBucketPath
// Ensure the local bucket path exists as it will be needed by all functions
err := os.MkdirAll(s.bucketLocalPath, 0660)
if err != nil {
return trace.Wrap(err, "failed to ensure path %q exists", s.bucketLocalPath)
}
return nil
}
func (s *S3manager) DownloadExistingRepo() error {
err := deleteAllFilesInDirectory(s.bucketLocalPath)
if err != nil {
return trace.Wrap(err, "failed to remove all filesystem entries in %q", s.bucketLocalPath)
}
downloadGroup := &errgroup.Group{}
downloadGroup.SetLimit(s.maxConcurrentSyncs)
linkMap := make(map[string]string)
var continuationToken *string
for {
listObjResponse, err := s.downloader.S3.ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: &s.bucketName,
ContinuationToken: continuationToken,
})
if err != nil {
return trace.Wrap(err, "failed to list objects for bucket %q", s.bucketName)
}
for _, s3object := range listObjResponse.Contents {
s.processS3ObjectDownload(s3object, downloadGroup, &linkMap)
}
continuationToken = listObjResponse.NextContinuationToken
if continuationToken == nil {
break
}
}
// Even if an error has occurred we should wait to exit until all running syncs have
// completed, even if not successful
logrus.Info("Waiting for download to complete...")
err = downloadGroup.Wait()
if err != nil {
return trace.Wrap(err, "failed to perform S3 sync from remote bucket %q to local bucket %q", s.bucketName, s.bucketLocalPath)
}
// Links must be created after their target exists
err = createLinks(linkMap)
if err != nil {
return trace.Wrap(err, "failed to create filesystem links for bucket %q", s.bucketName)
}
logrus.Infof("Downloaded %s bytes", bytesize.New(float64(s.downloadedBytes)))
return nil
}
func (s *S3manager) processS3ObjectDownload(s3object *s3.Object, downloadGroup *errgroup.Group, linkMap *map[string]string) {
downloadGroup.Go(func() error {
objectLink, err := s.getObjectLink(s3object)
if err != nil {
return trace.Wrap(err, "failed to get object link for key %q in bucket %q", *s3object.Key, s.bucketName)
}
// If the link does not start with a '/' then it is not a filesystem link
if objectLink != nil && len(*objectLink) > 0 && (*objectLink)[0] == '/' {
localObjectPath := filepath.Join(s.bucketLocalPath, *s3object.Key)
linkTarget := filepath.Join(s.bucketLocalPath, *objectLink)
(*linkMap)[localObjectPath] = linkTarget
return nil
}
err = s.downloadFile(s3object)
if err != nil {
return trace.Wrap(err, "failed to download S3 file %q from bucket %q", *s3object.Key, s.bucketName)
}
return nil
})
}
func createLinks(linkMap map[string]string) error {
for file, target := range linkMap {
logrus.Infof("Creating a symlink from %q to %q", file, target)
err := os.MkdirAll(filepath.Dir(file), 0660)
if err != nil {
return trace.Wrap(err, "failed to create directory structure for %q", file)
}
err = os.Symlink(target, file)
if err != nil {
return trace.Wrap(err, "failed to symlink %q to %q", file, target)
}
}
return nil
}
// This could potentially be made more efficient by running `os.RemoveAll` in a goroutine
// as random access on storage devices performs better at a higher queue depth
func deleteAllFilesInDirectory(dir string) error {
// Note that os.ReadDir does not follow/eval links which is important here
dirEntries, err := os.ReadDir(dir)
if err != nil {
return trace.Wrap(err, "failed to list directory entries for directory %q", dir)
}
for _, dirEntry := range dirEntries {
dirEntryPath := filepath.Join(dir, dirEntry.Name())
err = os.RemoveAll(dirEntryPath)
if err != nil {
return trace.Wrap(err, "failed to remove directory entry %q", dirEntryPath)
}
}
return nil
}
func (s *S3manager) getObjectLink(s3object *s3.Object) (*string, error) {
s3HeadObjectOutput, err := s.downloader.S3.HeadObject(&s3.HeadObjectInput{
Bucket: &s.bucketName,
Key: s3object.Key,
// Probably unnecessary but this will cause an error to be thrown if somebody is
// modifying the object while this program is running
IfMatch: s3object.ETag,
IfUnmodifiedSince: s3object.LastModified,
})
if err != nil {
return nil, trace.Wrap(err, "failed to retrieve metadata for key %q in bucket %q", *s3object.Key, s.bucketName)
}
return s3HeadObjectOutput.WebsiteRedirectLocation, nil
}
// s3sync has a bug when downloading a single file so this call reimplements s3sync's download
func (s *S3manager) downloadFile(s3object *s3.Object) error {
logrus.Infof("Downloading %q...", *s3object.Key)
localObjectPath := filepath.Join(s.bucketLocalPath, *s3object.Key)
err := os.MkdirAll(filepath.Dir(localObjectPath), 0660)
if err != nil {
return trace.Wrap(err, "failed to create directory structure for %q", localObjectPath)
}
fileWriter, err := os.Create(localObjectPath)
if err != nil {
return trace.Wrap(err, "failed to open %q for writing", localObjectPath)
}
defer fileWriter.Close()
fileDownloadByteCount, err := s.downloader.Download(fileWriter, &s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(*s3object.Key),
})
if err != nil {
return trace.Wrap(err, "failed to download object %q from bucket %q to local path %q", *s3object.Key, s.bucketName, localObjectPath)
}
s.downloadedBytes += fileDownloadByteCount
err = os.Chtimes(localObjectPath, *s3object.LastModified, *s3object.LastModified)
if err != nil {
return trace.Wrap(err, "failed to update the access and modification time on file %q to %v", localObjectPath, *s3object.LastModified)
}
logrus.Infof("Download %q complete", *s3object.Key)
return nil
}
func (s *S3manager) UploadBuiltRepo() error {
err := s.sync(false)
if err != nil {
return trace.Wrap(err, "failed to upload bucket")
}
return nil
}
func (s *S3manager) UploadBuiltRepoWithRedirects(extensionToMatch, relativeRedirectDir string) error {
uploadGroup := &errgroup.Group{}
uploadGroup.SetLimit(s.maxConcurrentSyncs)
walkErr := filepath.WalkDir(s.bucketLocalPath, func(absPath string, info fs.DirEntry, err error) error {
logrus.Debugf("Starting on %q...", absPath)
if err != nil {
return trace.Wrap(err, "failed to walk over directory %q on path %q", s.bucketLocalPath)
}
syncFunc, err := s.syncGenericFsObject(absPath, info)
if err != nil {
return trace.Wrap(err, "failed to get syncing function for %q", absPath)
}
uploadGroup.Go(syncFunc)
logrus.Debugf("Upload for %q queued", absPath)
return nil
})
// Even if an error has occurred we should wait to exit until all running syncs have
// completed, even if not successful
logrus.Info("Waiting for sync to complete...")
syncErr := uploadGroup.Wait()
// Future work: add upload logging information once
// https://github.com/seqsense/s3sync/commit/29b3fcb259293d80634cb3916e0f28467d017087 has been released
logrus.Info("Sync has completed")
errs := make([]error, 0, 2)
if walkErr != nil {
errs = append(errs, trace.Wrap(walkErr, "failed to walk over entries in %q", s.bucketLocalPath))
}
if syncErr != nil {
errs = append(errs, trace.Wrap(syncErr, "failed to perform S3 sync from local bucket %q to remote bucket %q", s.bucketLocalPath, s.bucketName))
}
if len(errs) > 0 {
return trace.Wrap(trace.NewAggregate(errs...), "one or more erros occurred while uploading built repo %q", s.bucketLocalPath)
}
return nil
}
func (s *S3manager) syncGenericFsObject(absPath string, dirEntryInfo fs.DirEntry) (func() error, error) {
// Don't do anything with non-empty directories as they will be caught later by their contents
if dirEntryInfo.IsDir() {
f, err := s.buildSyncDirFunc(absPath)
if err != nil {
return nil, trace.Wrap(err, "failed to build directory syncing function to sync %q", absPath)
}
return f, nil
} else
// If symbolic link
if dirEntryInfo.Type()&fs.ModeSymlink != 0 {
f, err := s.buildSyncSymbolicLinkFunc(absPath)
if err != nil {
return nil, trace.Wrap(err, "failed to build symbolic link file syncing function to sync %q", absPath)
}
return f, nil
}
// sync a single file or directory
f, err := s.buildSyncSingleFsEntryFunc(absPath)
if err != nil {
return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath)
}
return f, nil
}
func (s *S3manager) buildSyncDirFunc(absPath string) (func() error, error) {
isDirEmpty, err := isDirectoryEmpty(absPath)
if err != nil {
return nil, trace.Wrap(err, "failed to determine if directory %q is empty", absPath)
}
if !isDirEmpty {
logrus.Debug("Skipping non-empty directory")
return func() error { return nil }, nil
}
// If the directory has no contents, call sync normally which will create the directory remotely if not exists
f, err := s.buildSyncSingleFsEntryFunc(absPath)
if err != nil {
return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath)
}
return f, nil
}
func (s *S3manager) buildSyncSymbolicLinkFunc(absPath string) (func() error, error) {
actualFilePath, err := filepath.EvalSymlinks(absPath)
if err != nil {
return nil, trace.Wrap(err, "failed to follow symlink for path %q", absPath)
}
isInBucket, err := isPathChildOfAnother(s.bucketLocalPath, actualFilePath)
if err != nil {
return nil, trace.Wrap(err, "failed to determine if %q is a child of %q", actualFilePath, s.bucketLocalPath)
}
if isInBucket {
// This will re-upload every redirect file ever created. Implementing "sync" functionality would
// require significantly more engineering effort and this cost is low so this shouldn't be a
// problem.
return func() error {
err := s.UploadRedirectFile(absPath, actualFilePath)
if err != nil {
return trace.Wrap(err, "failed to upload a redirect file to S3 for %q targeting %q", absPath, actualFilePath)
}
return nil
}, nil
}
// If not in bucket, call sync normally which will follow the symlink to the actual file and upload it
f, err := s.buildSyncSingleFsEntryFunc(absPath)
if err != nil {
return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath)
}
return f, nil
}
func (s *S3manager) buildSyncSingleFsEntryFunc(absPath string) (func() error, error) {
relPath, err := filepath.Rel(s.bucketLocalPath, absPath)
if err != nil {
return nil, trace.Wrap(err, "failed to get %q relative to %q", absPath, s.bucketLocalPath)
}
remoteURL := getURLWithPath(*s.bucketURL, relPath)
return func() error {
err := s.syncManager.Sync(absPath, remoteURL)
if err != nil {
return trace.Wrap(err, "failed to sync from %q to %q", absPath, remoteURL)
}
return nil
}, nil
}
func getURLWithPath(baseURL url.URL, path string) string {
// Because this function is pass-by-value it should not modify `baseUrl`, where doing this directly on the
// provided parameter would modify it
baseURL.Path = path
return baseURL.String()
}
func isPathChildOfAnother(baseAbsPath string, testAbsPath string) (bool, error) {
// General implementation from https://stackoverflow.com/questions/28024731/check-if-given-path-is-a-subdirectory-of-another-in-golang
relPath, err := filepath.Rel(baseAbsPath, testAbsPath)
if err != nil {
return false, trace.Wrap(err, "failed to get the path of %q relative to %q", testAbsPath, baseAbsPath)
}
return !strings.HasPrefix(relPath, fmt.Sprintf("..%c", os.PathSeparator)) && relPath != "..", nil
}
func (s *S3manager) UploadRedirectFile(localAbsSrcPath, localAbsRemoteTargetPath string) error {
relSrcPath, err := filepath.Rel(s.bucketLocalPath, localAbsSrcPath)
if err != nil {
return trace.Wrap(err, "failed to get %q relative to %q", localAbsSrcPath, s.bucketLocalPath)
}
relTargetPath, err := filepath.Rel(s.bucketLocalPath, localAbsRemoteTargetPath)
if err != nil {
return trace.Wrap(err, "failed to get %q relative to %q", localAbsRemoteTargetPath, s.bucketLocalPath)
}
logrus.Infof("Creating a redirect file from %q to %q", relSrcPath, relTargetPath)
// S3 requires a prepended "/" to inform the redirect metadata that the target is another S3 object
// in the same bucket
s3TargetPath := filepath.Join("/", relTargetPath)
// Upload an empty file that when requested will redirect to the real one
_, err = s.uploader.Upload(&s3manager.UploadInput{
Bucket: &s.bucketName,
Key: &relSrcPath,
Body: bytes.NewReader([]byte{}),
WebsiteRedirectLocation: &s3TargetPath,
})
if err != nil {
return trace.Wrap(err, "failed to upload an empty redirect file to %q in bucket %q", relSrcPath, s.bucketName)
}
return nil
}
func (s *S3manager) UploadRedirectURL(remoteAbsSourcePath, targetURL string) error {
logrus.Infof("Creating redirect from %q to %q", remoteAbsSourcePath, targetURL)
_, err := s.uploader.Upload(&s3manager.UploadInput{
Bucket: &s.bucketName,
Key: &remoteAbsSourcePath,
Body: bytes.NewReader([]byte{}),
WebsiteRedirectLocation: &targetURL,
})
if err != nil {
return trace.Wrap(err, "failed to upload URL redirect file targeting %q to %q", targetURL, remoteAbsSourcePath)
}
return nil
}
func isDirectoryEmpty(dirPath string) (bool, error) {
// Pulled from https://stackoverflow.com/questions/30697324/how-to-check-if-directory-on-path-is-empty
f, err := os.Open(dirPath)
if err != nil {
return false, trace.Wrap(err, "failed to open directory %q", dirPath)
}
defer f.Close()
_, err = f.Readdirnames(1)
if err == io.EOF {
return true, nil
}
if err != nil {
return false, trace.Wrap(err, "failed to read the name of directories in %q", dirPath)
}
return false, nil
}
func (s *S3manager) sync(download bool) error {
var src, dest string
if download {
src = s.bucketURL.String()
dest = s.bucketLocalPath
} else {
src = s.bucketLocalPath
dest = s.bucketURL.String()
}
logrus.Infof("Performing S3 sync from %q to %q...", src, dest)
err := s.syncManager.Sync(src, dest)
if err != nil {
return trace.Wrap(err, "failed to sync %q to %q", src, dest)
}
logrus.Infoln("S3 sync complete")
return nil
}
build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh
-63
View File
@@ -1,63 +0,0 @@
#!/bin/bash
# shellcheck disable=SC2016,SC1004,SC2174,SC2155
set -xeu
# These must be set for the script to run
: "$AWS_ACCESS_KEY_ID"
: "$AWS_SECRET_ACCESS_KEY"
: "$AWS_SESSION_TOKEN"
ART_VERSION_TAG="8.3.15"
ARTIFACT_PATH="/go/artifacts"
CACHE_DIR="/mnt/createrepo_cache"
GNUPGHOME="/tmpfs/gnupg"
REPO_S3_BUCKET="fred-test1"
BUCKET_CACHE_PATH="/mnt/bucket"
export AWS_REGION="us-west-2"
: '
Run command:
docker run \
--rm -it \
-v "$(git rev-parse --show-toplevel)":/go/src/github.com/gravitational/teleport/ \
-v "$HOME/.aws":"/root/.aws" \
-e AWS_PROFILE="$AWS_PROFILE" \
-e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
-e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
-e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \
-e DEBIAN_FRONTEND="noninteractive" \
golang:1.18.4-bullseye /go/src/github.com/gravitational/teleport/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh
'
# Download the artifacts
apt update
apt install -y wget
mkdir -pv "$ARTIFACT_PATH"
cd "$ARTIFACT_PATH"
wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.x86_64.rpm"
wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.arm64.rpm"
wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.i386.rpm"
wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.arm.rpm"
apt install -y createrepo-c gnupg
mkdir -pv "$CACHE_DIR"
mkdir -pv -m0700 "$GNUPGHOME"
chown -R root:root "$GNUPGHOME"
export GPG_TTY=$(tty)
gpg --batch --gen-key <<EOF
Key-Type: 1
Key-Length: 2048
Subkey-Type: 1
Subkey-Length: 2048
Name-Real: Test RPM key
Name-Email: [email protected]
Expire-Date: 0
%no-protection
EOF
cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
export VERSION="v${ART_VERSION_TAG}"
export RELEASE_CHANNEL="stable"
go run ./cmd/build-os-package-repos yum -bucket "$REPO_S3_BUCKET" -local-bucket-path \
"$BUCKET_CACHE_PATH" -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" \
-artifact-path "$ARTIFACT_PATH" -log-level 4 -cache-dir "$CACHE_DIR"
build.assets/tooling/cmd/build-os-package-repos/yum_repo_tool.go
-511
View File
@@ -1,511 +0,0 @@
/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"io"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
// The golang docs are wrong/out of date for this package. Check github instead.
"github.com/cavaliergopher/rpm"
"github.com/davecgh/go-spew/spew"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
)
type YumRepoTool struct {
config *YumConfig
s3Manager *S3manager
createRepo *CreateRepo
gpg *GPG
supportedOSs map[string][]string
}
const ArtifactExtension string = ".rpm"
// Instantiates a new yum repo tool instance and performs any required setup/config.
func NewYumRepoTool(config *YumConfig, supportedOSs map[string][]string) (*YumRepoTool, error) {
cr, err := NewCreateRepo(config.cacheDir)
if err != nil {
trace.Wrap(err, "failed to instantiate new CreateRepo instance")
}
s3Manager, err := NewS3Manager(config.S3Config)
if err != nil {
return nil, trace.Wrap(err, "failed to create a new s3manager instance")
}
gpg, err := NewGPG()
if err != nil {
return nil, trace.Wrap(err, "failed to create a new GPG instance")
}
return &YumRepoTool{
config: config,
s3Manager: s3Manager,
createRepo: cr,
gpg: gpg,
supportedOSs: supportedOSs,
}, nil
}
func (yrt *YumRepoTool) Run() error {
start := time.Now()
logrus.Infoln("Starting YUM repo build process...")
logrus.Debugf("Using config: %+v", spew.Sdump(yrt.config))
isFirstRun, err := yrt.isFirstRun()
if err != nil {
return trace.Wrap(err, "failed to determine if YUM repos have been built before")
}
if isFirstRun {
logrus.Warningln("First run or disaster recovery detected, attempting to rebuild existing repos from YUM repository...")
err = yrt.s3Manager.DownloadExistingRepo()
if err != nil {
return trace.Wrap(err, "failed to sync existing repo from S3 bucket")
}
// Additional first time setup can be done here, but shouldn't be needed
} else {
logrus.Debugf("Not first run of tool, skipping S3 resync")
}
// Both Hashicorp and Docker publish their key to this path
relativeGpgPublicKeyPath := "gpg"
err = yrt.gpg.WritePublicKeyToFile(path.Join(yrt.config.localBucketPath, relativeGpgPublicKeyPath))
if err != nil {
return trace.Wrap(err, "failed to write GPG public key")
}
sourceArtifactPaths, err := yrt.getSourceArtifactPaths()
if err != nil {
return trace.Wrap(err, "failed to get the file paths of available RPM artifacts")
}
// This can be anywhere under repoPath. Hardcoding it rather than putting it in config as it should not change
// between runs/versions.
relativeBucketArtifactPath := "RPMs"
bucketArtifactPaths, err := yrt.copyArtifactsToBucket(sourceArtifactPaths, relativeBucketArtifactPath)
if err != nil {
return trace.Wrap(err, "failed to transfer available RPM artifacts to a bucket subdirectory")
}
err = yrt.addArtifacts(bucketArtifactPaths, relativeGpgPublicKeyPath)
if err != nil {
return trace.Wrap(err, "failed to add artifacts")
}
err = yrt.s3Manager.UploadBuiltRepoWithRedirects(ArtifactExtension, relativeBucketArtifactPath)
if err != nil {
return trace.Wrap(err, "failed to sync changes to S3 bucket")
}
// Future work: add literals to config?
err = yrt.s3Manager.UploadRedirectURL("index.html", "https://goteleport.com/docs/installation/#linux")
if err != nil {
return trace.Wrap(err, "failed to redirect index page to Teleport docs")
}
logrus.Infof("YUM repo build process completed in %s", time.Since(start).Round(time.Millisecond))
return nil
}
func (yrt *YumRepoTool) isFirstRun() (bool, error) {
yumSyncPath := yrt.config.localBucketPath
logrus.Debugf("Checking if %q exists...", yumSyncPath)
files, err := os.ReadDir(yumSyncPath)
if err != nil {
return false, trace.Wrap(err, "failed to list files in %q", yumSyncPath)
}
logrus.Debugf("Found %d files in %q:", len(files), yumSyncPath)
for _, file := range files {
logrus.Debug(file.Name())
}
return len(files) == 0, nil
}
func (yrt *YumRepoTool) getSourceArtifactPaths() ([]string, error) {
artifactPath := yrt.config.artifactPath
logrus.Infof("Looking for artifacts in %q...", artifactPath)
fileDirEntries, err := os.ReadDir(artifactPath)
if err != nil {
return nil, trace.Wrap(err, "failed to list files in %q", artifactPath)
}
logrus.Infof("Found %d possible artifacts in %q", len(fileDirEntries), artifactPath)
// This allocates a capacity of the maximum that is possibly needed, but it is probably
// better than reallocating the underlying array by appending each time
validArtifactPaths := make([]string, 0, len(fileDirEntries))
for _, fileDirEntry := range fileDirEntries {
fileName := fileDirEntry.Name()
if path.Ext(fileName) != ArtifactExtension {
logrus.Debugf("The file %q does not have a %q extension, skipping...", fileName, ArtifactExtension)
continue
}
filePath := path.Join(artifactPath, fileName)
validArtifactPaths = append(validArtifactPaths, filePath)
logrus.Debugf("Found artifact %q", filePath)
}
logrus.Infof("Found %d artifacts", len(validArtifactPaths))
logrus.Debugf("Source artifact paths: %v", validArtifactPaths)
return validArtifactPaths, nil
}
func sortArtifactsByArch(artifactPaths []string) (map[string][]string, error) {
logrus.Info("Determining ISA of targeted artifacts...")
// Four is probably a decent guess for the number of ISAs we build for. This would cover:
// i386, x86_64, arm, arm64
archPackageMap := make(map[string][]string, 4)
for _, artifactPath := range artifactPaths {
logrus.Debugf("Attempting to open RPM %q...", artifactPath)
rpmPackage, err := rpm.Open(artifactPath)
if err != nil {
return nil, trace.Wrap(err, "failed to read package %q", artifactPath)
}
arch := rpmPackage.Architecture()
baseArch, err := getBaseArchForArch(arch)
if err != nil {
return nil, trace.Wrap(err, "failed to determine base architecture for artifact %q", artifactPath)
}
logrus.Debugf("Found %q with ISA %q and base ISA %q", artifactPath, arch, baseArch)
if rpmPackagePaths, ok := archPackageMap[baseArch]; ok {
archPackageMap[baseArch] = append(rpmPackagePaths, artifactPath)
} else {
archPackageMap[baseArch] = []string{artifactPath}
}
}
logrus.Infof("Found %d ISAs: %v", len(archPackageMap), archPackageMap)
return archPackageMap, nil
}
// Implementation pulled from https://github.com/rpm-software-management/yum/blob/master/rpmUtils/arch.py#L429
func getBaseArchForArch(arch string) (string, error) {
archTypes := map[string][]string{
"i386": {
"athlon",
"geode",
"i686",
"i586",
"i486",
"i386",
},
"x86_64": {
"amd64",
"ia32e",
"x86_64",
},
// This does not cover ARMv8 and above which have several strange corner cases
"arm": {
"armv2",
"armv3",
"armv4",
"armv5",
"armv6",
"armv7",
"arm",
},
"aarch64": {
"arm64",
"aarch64",
},
}
for baseArch, archTypes := range archTypes {
for _, archType := range archTypes {
if strings.HasPrefix(arch, archType) {
return baseArch, nil
}
}
}
return "", trace.Errorf("failed to determine base arch for architecture %q", arch)
}
func (yrt *YumRepoTool) addArtifacts(bucketArtifactPaths []string, relativeGpgPublicKeyPath string) error {
logrus.Info("Adding artifacts to repos...")
archs, err := sortArtifactsByArch(bucketArtifactPaths)
if err != nil {
return trace.Wrap(err, "failed to get artifacts by architecture")
}
repoCount := 0
for os, osVersions := range yrt.supportedOSs {
osPath := path.Join(yrt.config.localBucketPath, os)
for _, osVersion := range osVersions {
for arch, packages := range archs {
relativeRepoPath := path.Join(
osVersion,
"Teleport",
arch,
yrt.config.releaseChannel,
yrt.config.versionChannel,
)
repoPath := path.Join(osPath, relativeRepoPath)
err := yrt.updateRepoWithArtifacts(packages, repoPath)
if err != nil {
return trace.Wrap(err, "failed to add artifact for YUM repo %q", relativeRepoPath)
}
repoFilePath := filepath.Join(repoPath, "teleport.repo")
err = yrt.createRepoFile(repoFilePath, os, osVersion, arch, relativeGpgPublicKeyPath)
if err != nil {
return trace.Wrap(err, "failed to create repo file for os %q at %q", os, repoFilePath)
}
repoCount++
}
}
}
logrus.Infof("Updated %d repos with %d artifacts", repoCount, len(bucketArtifactPaths))
return nil
}
func (yrt *YumRepoTool) updateRepoWithArtifacts(packagePaths []string, repoPath string) error {
logrus.Infof("Updating repo at %q with packages %v", repoPath, packagePaths)
// A soft copy here will have a significant performance impact, and S3 sync will follow links
err := yrt.copyArtifactsToRepo(packagePaths, repoPath)
if err != nil {
return trace.Wrap(err, "failed to copy artifacts to repo %q", repoPath)
}
err = yrt.updateRepoMetadata(repoPath)
if err != nil {
return trace.Wrap(err, "failed to update repo %q metadata", repoPath)
}
err = yrt.signRepoMetadata(repoPath)
if err != nil {
return trace.Wrap(err, "failed to sign repo %q metadata", repoPath)
}
logrus.Infof("Finished updating repo %q", repoPath)
return nil
}
func (yrt *YumRepoTool) copyArtifactsToRepo(artifactPaths []string, repoPath string) error {
// The "repo_rpms" directory here is arbitrary and not tied to anything else.
repoArtifactFolder := path.Join(repoPath, "repo_rpms")
_, err := copyArtifacts(artifactPaths, repoArtifactFolder, false)
if err != nil {
return trace.Wrap(err, "failed to copy artifacts %d artifacts to repo directory at %s", len(artifactPaths), repoArtifactFolder)
}
return nil
}
// Flattens artifactPaths into one directory and returns the created files in that directory
func (yrt *YumRepoTool) copyArtifactsToBucket(artifactPaths []string, bucketArtifactSubdirectory string) ([]string, error) {
bucketArtifactFolder := path.Join(yrt.config.localBucketPath, bucketArtifactSubdirectory)
// A "hard" copy is performed here because the bucket will usually be stored on a non-ephemeral filesystem path.
// If the artifacts are linked rather than copied then every time the uploaded bucket is synced on future runs
// the sync will re-download the real artifacts.
destinationArtifactPaths, err := copyArtifacts(artifactPaths, bucketArtifactFolder, true)
if err != nil {
return nil, trace.Wrap(err, "failed to copy artifacts %d artifacts to bucket directory at %s", len(artifactPaths), destinationArtifactPaths)
}
return destinationArtifactPaths, nil
}
func copyArtifacts(artifactPaths []string, destinationDirectory string, shouldHardCopy bool) ([]string, error) {
copyType := "soft"
if shouldHardCopy {
copyType = "hard"
}
logrus.Debugf("Copying %d artifacts to %q via a %s copy...", len(artifactPaths), destinationDirectory, copyType)
err := os.MkdirAll(destinationDirectory, 0660)
if err != nil {
return nil, trace.Wrap(err, "failed to ensure destination directory %q exists", destinationDirectory)
}
destinationArtifactPaths := make([]string, len(artifactPaths))
for i, artifactPath := range artifactPaths {
artifactDestinationPath := path.Join(destinationDirectory, path.Base(artifactPath))
if shouldHardCopy {
hardCopyFile(artifactPath, artifactDestinationPath)
} else {
softCopyFile(artifactPath, artifactDestinationPath)
}
destinationArtifactPaths[i] = artifactDestinationPath
}
logrus.Debugf("Successfully copied %d artifact(s) to %q", len(destinationArtifactPaths), destinationDirectory)
return destinationArtifactPaths, nil
}
func (yrt *YumRepoTool) updateRepoMetadata(repoPath string) error {
// Ensure the directory exists
err := os.MkdirAll(repoPath, 0660)
if err != nil {
return trace.Wrap(err, "failed to ensure repo directory %q exists", repoPath)
}
err = yrt.createRepo.CreateOrUpdateRepo(repoPath)
if err != nil {
return trace.Wrap(err, "failed to update repo metadata for %q", repoPath)
}
return nil
}
func (yrt *YumRepoTool) signRepoMetadata(repoPath string) error {
repomdPath := path.Join(repoPath, "repodata", "repomd.xml")
err := yrt.gpg.SignFile(repomdPath)
if err != nil {
return trace.Wrap(err, "failed to sign file %q", repomdPath)
}
return nil
}
// Creates an os-specific ".repo" file for yum-config-manager akin to
// https://rpm.releases.teleport.dev/teleport.repo
func (yrt *YumRepoTool) createRepoFile(filePath, osName, osVersion, arch, relativeGpgPublicKeyPath string) error {
// Future work: maybe move domain name to config?
domainName := "yum.releases.teleport.dev"
sectionName := "teleport"
// See these for config details:
// https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/sec-configuring_yum_and_yum_repositories
// https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/sec-using_yum_variables
repoOptions := map[string]string{
"name": "Gravitational Teleport packages",
"baseurl": (&url.URL{
Scheme: "https",
Host: domainName,
Path: strings.Join(
[]string{
osName,
osVersion,
"Teleport",
arch,
yrt.config.releaseChannel,
yrt.config.versionChannel,
},
"/",
),
}).String(),
"enabled": "1",
"gpgcheck": "1",
"repo_gpgcheck": "1",
"gpgkey": (&url.URL{
Scheme: "https",
Host: domainName,
Path: relativeGpgPublicKeyPath,
}).String(),
}
// + 2 = repo header line, new line
repoFileLines := make([]string, 0, len(repoOptions)+2)
repoFileLines = append(repoFileLines, fmt.Sprintf("[%s]", sectionName))
for key, value := range repoOptions {
repoFileLines = append(repoFileLines, fmt.Sprintf("%s=%s", key, value))
}
repoFileLines = append(repoFileLines, "")
repoFileContent := strings.Join(repoFileLines, "\n")
err := os.WriteFile(filePath, []byte(repoFileContent), 0660)
if err != nil {
return trace.Wrap(err, "failed to create repo file at %q", filePath)
}
logrus.Infof("Created repo file at %q", filePath)
logrus.Debugf("Repo file contents:\n%s", repoFileContent)
return nil
}
// Guaranteed to perform a "copy" operation rather than just linking. Should only be used
// where linking is not acceptable.
func hardCopyFile(src, dest string) error {
// Implementation is a modified version of method 1 from
// https://opensource.com/article/18/6/copying-files-go
start := time.Now()
logrus.Debugf("Beginning hard file copy from %q to %q...", src, dest)
sourceFileStat, err := os.Stat(src)
if err != nil {
return trace.Wrap(err, "failed to get filesystem data for %q", src)
}
if !sourceFileStat.Mode().IsRegular() {
return trace.Errorf("Source file %q is not a regular file and cannot be copied", src)
}
sourceHandle, err := os.Open(src)
if err != nil {
return trace.Wrap(err, "failed to open source file %q for reading", src)
}
defer sourceHandle.Close()
destinationHandle, err := os.Create(dest)
if err != nil {
return trace.Wrap(err, "failed to open destination file %q for writing", dest)
}
defer destinationHandle.Close()
_, err = io.Copy(destinationHandle, sourceHandle)
if err != nil {
return trace.Wrap(err, "failed to copy source file %q to destination file %q", src, dest)
}
logrus.Debugf("File transfer from %q to %q completed in %s", src, dest, time.Since(start).Round(time.Millisecond))
return nil
}
// Copies or links the src file to dest path. The implementation of this function is subject
// to change. If a guaranteed is needed then `hardCopyFile` should be used instead.
func softCopyFile(src, dest string) error {
// Profiling has shown that disk reads/writes are a significant bottleneck with the
// APT side of the tool. This will reduce roughly 25GB of read/writes to nearly 0.
start := time.Now()
logrus.Debugf("Beginning soft file copy from %q to %q...", src, dest)
err := os.Symlink(src, dest)
if err != nil {
return trace.Wrap(err, "failed to link %q to %q", src, dest)
}
logrus.Debugf("File transfer from %q to %q completed in %s", src, dest, time.Since(start).Round(time.Nanosecond))
return nil
}
dronegen/apt.go
-54
View File
@@ -1,54 +0,0 @@
// Copyright 2021 Gravitational, Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import "path"
// This function calls the build-apt-repos tool which handles the APT portion of RFD 0058.
func promoteAptPipeline() pipeline {
return getAptPipelineBuilder().buildPromoteOsPackagePipeline()
}
func migrateAptPipeline(triggerBranch string, migrationVersions []string) pipeline {
return getAptPipelineBuilder().buildMigrateOsPackagePipeline(triggerBranch, migrationVersions)
}
func getAptPipelineBuilder() *OsPackageToolPipelineBuilder {
optpb := NewOsPackageToolPipelineBuilder(
"drone-s3-aptrepo-pvc",
"deb",
"apt",
NewRepoBucketSecrets(
"APT_REPO_NEW_AWS_S3_BUCKET",
"APT_REPO_NEW_AWS_ACCESS_KEY_ID",
"APT_REPO_NEW_AWS_SECRET_ACCESS_KEY",
"APT_REPO_NEW_AWS_ROLE",
),
)
optpb.environmentVars["APTLY_ROOT_DIR"] = value{
raw: path.Join(optpb.pvcMountPoint, "aptly"),
}
optpb.requiredPackages = []string{
"aptly",
}
optpb.extraArgs = []string{
"-aptly-root-dir \"$APTLY_ROOT_DIR\"",
}
return optpb
}
dronegen/common.go
-19
View File
@@ -19,7 +19,6 @@ import (
"fmt"
"log"
"os/exec"
"path"
"strings"
)
@@ -337,24 +336,6 @@ func cloneRepoStep(clonePath, commit string) step {
}
}
func verifyNotPrereleaseStep() step {
clonePath := "/tmp/repo"
commands := []string{
"apk add git",
}
commands = append(commands, cloneRepoCommands(clonePath, "${DRONE_TAG}")...)
commands = append(commands,
fmt.Sprintf("cd %q", path.Join(clonePath, "build.assets", "tooling")),
"go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)",
)
return step{
Name: "Check if tag is prerelease",
Image: fmt.Sprintf("golang:%s-alpine", GoVersion),
Commands: commands,
}
}
func sliceSelect[T, V any](slice []T, selector func(T) V) []V {
selectedValues := make([]V, len(slice))
for i, entry := range slice {
dronegen/os_repos.go
+40 -417
View File
@@ -17,443 +17,66 @@ package main
import (
"fmt"
"path"
"strings"
"time"
)
func buildOsRepoPipelines() []pipeline {
pipelines := promoteBuildOsRepoPipelines()
pipelines = append(pipelines, artifactMigrationPipeline()...)
return pipelines
}
func promoteBuildOsRepoPipelines() []pipeline {
aptPipeline := promoteAptPipeline()
yumPipeline := promoteYumPipeline()
pipelines := []pipeline{}
// Normal release pipelines
pipelines = append(pipelines, buildPromoteOsPackagePipelines("${DRONE_TAG}", `$($DRONE_REPO_PRIVATE && echo "*ent*" || echo "")`, "normal")...)
// teleport-ent-updater to stable/cloud only pipelines
pipelines = append(pipelines, buildPromoteOsPackagePipelines("cloud", "teleport-ent-updater*", "teleport-ent-updater")...)
return pipelines
}
func buildPromoteOsPackagePipelines(versionChannel, packageNameFilter, pipelineNameSuffix string) []pipeline {
return []pipeline{
aptPipeline,
yumPipeline,
buildPromoteOsPackagePipeline("apt", versionChannel, packageNameFilter, pipelineNameSuffix),
buildPromoteOsPackagePipeline("yum", versionChannel, packageNameFilter, pipelineNameSuffix),
}
}
// Used for one-off migrations of older versions.
// Use cases include:
// - We want to support another OS while providing backwards compatibility
// - We want to support another OS version while providing backwards compatibility
// - A customer wants to be able to install an older version via APT/YUM even if we
// no longer support it
// - RPM migrations after new YUM pipeline is done
func artifactMigrationPipeline() []pipeline {
migrationVersions := []string{
// These versions were migrated as a part of the new `promoteAptPipeline`
// "v6.2.31",
// "v7.3.17",
// "v7.3.18",
// "v7.3.19",
// "v7.3.20",
// "v7.3.21",
// "v7.3.23",
// "v8.3.3",
// "v8.3.4",
// "v8.3.5",
// "v8.3.6",
// "v8.3.7",
// "v8.3.8",
// "v8.3.9",
// "v8.3.10",
// "v8.3.11",
// "v8.3.12",
// "v8.3.14",
// "v8.3.15",
// "v8.3.16",
// "v9.0.0",
// "v9.0.1",
// "v9.0.2",
// "v9.0.3",
// "v9.0.4",
// "v9.1.0",
// "v9.1.1",
// "v9.1.2",
// "v9.1.3",
// "v9.2.0",
// "v9.2.1",
// "v9.2.2",
// "v9.2.3",
// "v9.2.4",
// "v9.3.0",
// "v9.3.2",
// "v9.3.4",
// "v9.3.5",
// "v9.3.6",
// "v9.3.7",
// "v9.3.9",
// "v9.3.10",
// "v9.3.12",
// "v9.3.13",
// "v9.3.14",
// "v10.0.0",
// "v10.0.1",
// "v10.0.2",
// "v10.1.2",
// "v10.1.4",
}
// Pushing to this branch will trigger the listed versions to be migrated. Typically this should be
// the branch that these changes are being committed to.
migrationBranch := "" // "rfd/0058-package-distribution"
func buildPromoteOsPackagePipeline(repoType, versionChannel, packageNameFilter, pipelineNameSuffix string) pipeline {
releaseEnvironmentFilePath := "/go/vars/release-environment.txt"
clonePath := "/go/src/github.com/gravitational/teleport"
aptPipeline := migrateAptPipeline(migrationBranch, migrationVersions)
yumPipeline := migrateYumPipeline(migrationBranch, migrationVersions)
return []pipeline{
aptPipeline,
yumPipeline,
}
}
type RepoBucketSecrets struct {
awsRoleSettings
bucketName value
}
func NewRepoBucketSecrets(bucketName, accessKeyID, secretAccessKey, role string) *RepoBucketSecrets {
return &RepoBucketSecrets{
awsRoleSettings: awsRoleSettings{
awsAccessKeyID: value{fromSecret: accessKeyID},
awsSecretAccessKey: value{fromSecret: secretAccessKey},
role: value{fromSecret: role},
pipeline := ghaBuildPipeline(ghaBuildType{
trigger: triggerPromote,
pipelineName: fmt.Sprintf("publish-%s-%s-new-repos", pipelineNameSuffix, repoType),
ghaWorkflow: "deploy-packages.yaml",
timeout: 12 * time.Hour, // DR takes a long time
workflowRef: "refs/heads/master",
inputs: map[string]string{
"repo-type": repoType,
"environment": fmt.Sprintf("$(cat %q)", releaseEnvironmentFilePath),
"artifact-tag": "${DRONE_TAG}",
"release-channel": "stable",
"version-channel": versionChannel,
"package-name-filter": packageNameFilter,
},
bucketName: value{fromSecret: bucketName},
}
}
})
type OsPackageToolPipelineBuilder struct {
clameName string
packageType string
packageManagerName string
volumeName string
pipelineNameSuffix string
artifactPath string
pvcMountPoint string
bucketSecrets *RepoBucketSecrets
extraArgs []string
requiredPackages []string
setupCommands []string
environmentVars map[string]value
}
// This function configures the build tool with it's requirements and sensible defaults.
// If additional configuration required then the returned struct should be modified prior
// to calling "build" functions on it.
func NewOsPackageToolPipelineBuilder(claimName, packageType, packageManagerName string, bucketSecrets *RepoBucketSecrets) *OsPackageToolPipelineBuilder {
optpb := &OsPackageToolPipelineBuilder{
clameName: claimName,
packageType: packageType,
packageManagerName: packageManagerName,
bucketSecrets: bucketSecrets,
extraArgs: []string{},
setupCommands: []string{},
requiredPackages: []string{},
volumeName: fmt.Sprintf("%s-persistence", packageManagerName),
pipelineNameSuffix: fmt.Sprintf("%s-new-repos", packageManagerName),
artifactPath: "/go/artifacts",
pvcMountPoint: "/mnt",
}
optpb.environmentVars = map[string]value{
"REPO_S3_BUCKET": optpb.bucketSecrets.bucketName,
"AWS_REGION": {
raw: "us-west-2",
},
"BUCKET_CACHE_PATH": {
// If we need to cache the bucket on the PVC for some reason in the future
// uncomment this line
// raw: path.Join(pvcMountPoint, "bucket-cache"),
raw: "/tmp/bucket",
},
"ARTIFACT_PATH": {
raw: optpb.artifactPath,
},
"GNUPGHOME": {
raw: "/tmpfs/gnupg",
},
"GPG_RPM_SIGNING_ARCHIVE": {
fromSecret: "GPG_RPM_SIGNING_ARCHIVE",
},
"DEBIAN_FRONTEND": {
raw: "noninteractive",
},
}
return optpb
}
func (optpb *OsPackageToolPipelineBuilder) buildPromoteOsPackagePipeline() pipeline {
pipelineName := fmt.Sprintf("publish-%s", optpb.pipelineNameSuffix)
checkoutPath := "/go/src/github.com/gravitational/teleport"
commitName := "${DRONE_TAG}"
p := optpb.buildBaseOsPackagePipeline(pipelineName, checkoutPath, commitName)
p.Trigger = triggerPromote
p.Trigger.Repo.Include = []string{
"gravitational/teleport",
"gravitational/teleport-private",
}
setupSteps := []step{
verifyTaggedStep(),
cloneRepoStep(checkoutPath, commitName),
}
setupStepNames := make([]string, 0, len(setupSteps))
for _, setupStep := range setupSteps {
setupStepNames = append(setupStepNames, setupStep.Name)
}
versionSteps := optpb.getDroneTagVersionSteps(checkoutPath)
for i := range versionSteps {
versionStep := &versionSteps[i]
if versionStep.DependsOn == nil {
versionStep.DependsOn = setupStepNames
continue
}
versionStep.DependsOn = append(versionStep.DependsOn, setupStepNames...)
}
p.Steps = append(setupSteps, versionSteps...)
return p
}
func (optpb *OsPackageToolPipelineBuilder) buildMigrateOsPackagePipeline(triggerBranch string, migrationVersions []string) pipeline {
pipelineName := fmt.Sprintf("migrate-%s", optpb.pipelineNameSuffix)
checkoutPath := "/go/src/github.com/gravitational/teleport"
// DRONE_TAG is not available outside of promotion pipelines and will cause drone to fail with a
// "migrate-apt-new-repos: bad substitution" error if used here
commitName := "${DRONE_COMMIT}"
// If migrations are not configured then don't run
if triggerBranch == "" || len(migrationVersions) == 0 {
return buildNeverTriggerPipeline(pipelineName)
}
p := optpb.buildBaseOsPackagePipeline(pipelineName, checkoutPath, commitName)
p.Trigger = trigger{
Repo: triggerRef{Include: []string{"gravitational/teleport"}},
Event: triggerRef{Include: []string{"push"}},
Branch: triggerRef{Include: []string{triggerBranch}},
}
for _, migrationVersion := range migrationVersions {
// Not enabling parallelism here so that multiple migrations don't run at once
p.Steps = append(p.Steps, optpb.getVersionSteps(checkoutPath, migrationVersion, false, false)...)
}
setStepResourceLimits(p.Steps)
return p
}
// Builds a pipeline that is syntactically correct but should never trigger to create
// a placeholder pipeline
func buildNeverTriggerPipeline(pipelineName string) pipeline {
p := newKubePipeline(pipelineName)
p.Trigger = trigger{
Event: triggerRef{Include: []string{"custom"}},
Repo: triggerRef{Include: []string{"non-existent-repository"}},
Branch: triggerRef{Include: []string{"non-existent-branch"}},
}
p.Steps = []step{
pipeline.Steps = []step{
pipeline.Steps[0],
{
Name: "Placeholder",
Image: "alpine:latest",
Name: "Determine if release should go to development or production",
Image: fmt.Sprintf("golang:%s-alpine", GoVersion),
Commands: []string{
"echo \"This command, step, and pipeline never runs\"",
fmt.Sprintf("cd %q", path.Join(clonePath, "build.assets", "tooling")),
fmt.Sprintf(`(go run ./cmd/check -tag ${DRONE_TAG} -check prerelease && echo "build" || echo "promote") > %q`, releaseEnvironmentFilePath),
},
},
pipeline.Steps[1],
}
return p
}
// Functions that use this method should add at least:
// * a Trigger
// * Steps for checkout
func (optpb *OsPackageToolPipelineBuilder) buildBaseOsPackagePipeline(pipelineName, checkoutPath, commit string) pipeline {
p := newKubePipeline(pipelineName)
p.Workspace = workspace{Path: "/go"}
p.Volumes = []volume{
{
Name: optpb.volumeName,
Claim: &volumeClaim{
Name: optpb.clameName,
},
},
volumeTmpfs,
volumeAwsConfig,
}
p.Steps = []step{cloneRepoStep(checkoutPath, commit)}
setStepResourceLimits(p.Steps)
return p
}
func setStepResourceLimits(steps []step) {
// Not currently supported
// for i := range steps {
// step := &steps[i]
// if step.Resources == nil {
// step.Resources = &containerResources{}
// }
// if step.Resources.Requests == nil {
// step.Resources.Requests = &resourceSet{}
// }
// step.Resources.Requests.Cpu = 100
// step.Resources.Requests.Memory = (*resourceQuantity)(resource.NewQuantity(100*1024*1024, resource.BinarySI))
// }
}
func (optpb *OsPackageToolPipelineBuilder) getDroneTagVersionSteps(codePath string) []step {
return optpb.getVersionSteps(codePath, "${DRONE_TAG}", true, true)
}
// Version should start with a 'v', i.e. v1.2.3 or v9.0.1, or should be an environment var
// i.e. ${DRONE_TAG}
func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version string, enableParallelism bool, enablePrereleaseCheck bool) []step {
var bucketFolder string
switch version[0:1] {
// If environment var
case "$":
// Remove the 'v' at runtime as the value isn't known at compile time
// This will change "${SOME_VAR}" to "${SOME_VAR##v}". `version` isn't actually
// an environment variable - it's a Drone substitution variable. See
// https://docs.drone.io/pipeline/environment/substitution/ for details.
bucketFolder = fmt.Sprintf("%s##v}", version[:len(version)-1])
// If static string
case "v":
// Remove the 'v' at compile time as the value is known then
bucketFolder = version[1:]
}
toolSetupCommands := []string{}
if len(optpb.requiredPackages) > 0 {
toolSetupCommands = []string{
"apt update",
fmt.Sprintf("apt install -y %s", strings.Join(optpb.requiredPackages, " ")),
}
}
toolSetupCommands = append(toolSetupCommands, optpb.setupCommands...)
assumeDownloadRoleStep := kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{
awsRoleSettings: awsRoleSettings{
awsAccessKeyID: value{fromSecret: "AWS_ACCESS_KEY_ID"},
awsSecretAccessKey: value{fromSecret: "AWS_SECRET_ACCESS_KEY"},
role: value{fromSecret: "AWS_ROLE"},
},
configVolume: volumeRefAwsConfig,
name: "Assume Download AWS Role",
})
downloadStep := step{
Name: fmt.Sprintf("Download artifacts for %q", version),
Image: "amazon/aws-cli",
Environment: map[string]value{
"AWS_S3_BUCKET": {
fromSecret: "AWS_S3_BUCKET",
},
"ARTIFACT_PATH": {
raw: optpb.artifactPath,
},
},
Volumes: []volumeRef{volumeRefAwsConfig},
Commands: []string{
"mkdir -pv \"$ARTIFACT_PATH\"",
// Clear out old versions from previous steps
"rm -rf \"$ARTIFACT_PATH\"/*",
// Conditionally match ONLY enterprise and fips binaries based off of file name,
// if running in the context of a private repo (teleport-private)
"if [ \"${DRONE_REPO_PRIVATE}\" = true ]; then ENT_FILTER=\"*ent\"; fi",
fmt.Sprintf("FILTER=\"$${ENT_FILTER}*.%s*\"", optpb.packageType),
strings.Join(
[]string{
"aws s3 sync",
"--no-progress",
"--delete",
"--exclude \"*\"",
"--include \"$FILTER\"",
fmt.Sprintf("s3://$AWS_S3_BUCKET/teleport/tag/%s/", bucketFolder),
"\"$ARTIFACT_PATH\"",
},
" ",
),
},
}
assumeUploadRoleStep := kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{
awsRoleSettings: optpb.bucketSecrets.awsRoleSettings,
configVolume: volumeRefAwsConfig,
name: "Assume Upload AWS Role",
})
verifyNotPrereleaseStep := verifyNotPrereleaseStep()
buildAndUploadStep := step{
Name: fmt.Sprintf("Publish %ss to %s repos for %q", optpb.packageType, strings.ToUpper(optpb.packageManagerName), version),
Image: fmt.Sprintf("golang:%s-bullseye", GoVersion),
Environment: optpb.environmentVars,
Commands: append(
toolSetupCommands,
[]string{
"mkdir -pv -m0700 \"$GNUPGHOME\"",
"echo \"$GPG_RPM_SIGNING_ARCHIVE\" | base64 -d | tar -xzf - -C $GNUPGHOME",
"chown -R root:root \"$GNUPGHOME\"",
fmt.Sprintf("cd %q", path.Join(codePath, "build.assets", "tooling")),
fmt.Sprintf("export VERSION=%q", version),
"export RELEASE_CHANNEL=\"stable\"", // The tool supports several release channels but I'm not sure where this should be configured
strings.Join(
append(
[]string{
// This just makes the (long) command a little more readable
"go run ./cmd/build-os-package-repos",
optpb.packageManagerName,
"-bucket \"$REPO_S3_BUCKET\"",
"-local-bucket-path \"$BUCKET_CACHE_PATH\"",
"-version-channel \"$VERSION\"",
"-release-channel \"$RELEASE_CHANNEL\"",
"-artifact-path \"$ARTIFACT_PATH\"",
"-log-level 4", // Set this to 5 for debug logging
},
optpb.extraArgs...,
),
" ",
),
}...,
),
Volumes: []volumeRef{
{
Name: optpb.volumeName,
Path: optpb.pvcMountPoint,
},
volumeRefTmpfs,
volumeRefAwsConfig,
},
}
var steps []step
steps = append(steps, assumeDownloadRoleStep)
steps = append(steps, downloadStep)
steps = append(steps, assumeUploadRoleStep)
if enablePrereleaseCheck {
steps = append(steps, verifyNotPrereleaseStep)
}
steps = append(steps, buildAndUploadStep)
if enableParallelism {
for i := 1; i < len(steps); i++ {
steps[i].DependsOn = []string{steps[i-1].Name}
}
}
return steps
return pipeline
}
dronegen/yum.go
-63
View File
@@ -1,63 +0,0 @@
// Copyright 2021 Gravitational, Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"path"
)
// This function calls the build-apt-repos tool which handles the APT portion of RFD 0058.
func promoteYumPipeline() pipeline {
return getYumPipelineBuilder().buildPromoteOsPackagePipeline()
}
func migrateYumPipeline(triggerBranch string, migrationVersions []string) pipeline {
return getYumPipelineBuilder().buildMigrateOsPackagePipeline(triggerBranch, migrationVersions)
}
func getYumPipelineBuilder() *OsPackageToolPipelineBuilder {
optpb := NewOsPackageToolPipelineBuilder(
"drone-s3-yumrepo-pvc",
"rpm",
"yum",
NewRepoBucketSecrets(
"YUM_REPO_NEW_AWS_S3_BUCKET",
"YUM_REPO_NEW_AWS_ACCESS_KEY_ID",
"YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY",
"YUM_REPO_NEW_AWS_ROLE",
),
)
optpb.environmentVars["CACHE_DIR"] = value{
raw: path.Join(optpb.pvcMountPoint, "createrepo_cache"),
}
optpb.environmentVars["BUCKET_CACHE_PATH"] = value{
raw: path.Join(optpb.pvcMountPoint, "bucket"),
}
optpb.requiredPackages = []string{
"createrepo-c",
}
optpb.setupCommands = []string{
"mkdir -pv \"$CACHE_DIR\"",
}
optpb.extraArgs = []string{
"-cache-dir \"$CACHE_DIR\"",
}
return optpb
}