crowdsec/.github/workflows/go-tests.yml

---
# This workflow is actually running
# tests (with localstack) but the
# name is used for the badge in README.md

name: Build

on:
  push:
    branches:
      - master
      - releases/**
    paths-ignore:
      - "README.md"
  pull_request:
    paths-ignore:
      - "README.md"

permissions:
  contents: read

# these env variables are for localstack, so we can emulate aws services
env:
  AWS_HOST: localstack
  # these are to mimic aws config
  AWS_ACCESS_KEY_ID: test
  AWS_SECRET_ACCESS_KEY: test
  AWS_REGION: us-east-1
  CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true

jobs:
  build:
    name: "Build + tests"
    runs-on: ubuntu-latest
    services:
      localstack:
        image: localstack/localstack:3.0
        ports:
          - 4566:4566 # Localstack exposes all services on the same port
        env:
          DEBUG: ""
          LAMBDA_EXECUTOR: ""
          KINESIS_ERROR_PROBABILITY: ""
          KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
          LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
          # e.g sqs url will get localhost if we don't set this env to map our service
        options: >-
          --name=localstack
          --health-cmd="curl -sS 127.0.0.1:4566 || exit 1"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=3
      zoo1:
        image: confluentinc/cp-zookeeper:7.4.3
        ports:
          - "2181:2181"
        env:
          ZOOKEEPER_CLIENT_PORT: 2181
          ZOOKEEPER_SERVER_ID: 1
          ZOOKEEPER_SERVERS: zoo1:2888:3888
        options: >-
          --name=zoo1
          --health-cmd "jps -l | grep zookeeper"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      kafka1:
        image: crowdsecurity/kafka-ssl
        ports:
          - "9093:9093"
          - "9092:9092"
          - "9999:9999"
        env:
          KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://127.0.0.1:19092,LISTENER_DOCKER_EXTERNAL://127.0.0.1:9092,LISTENER_DOCKER_EXTERNAL_SSL://127.0.0.1:9093
          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_SSL:SSL
          KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
          KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
          KAFKA_BROKER_ID: 1
          KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
          KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
          KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
          KAFKA_JMX_PORT: 9999
          KAFKA_JMX_HOSTNAME: "127.0.0.1"
          KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
          KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
          KAFKA_SSL_KEYSTORE_FILENAME: kafka.kafka1.keystore.jks
          KAFKA_SSL_KEYSTORE_CREDENTIALS: kafka1_keystore_creds
          KAFKA_SSL_KEY_CREDENTIALS: kafka1_sslkey_creds
          KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.kafka1.truststore.jks
          KAFKA_SSL_TRUSTSTORE_CREDENTIALS: kafka1_truststore_creds
          KAFKA_SSL_ENABLED_PROTOCOLS: TLSv1.2
          KAFKA_SSL_PROTOCOL: TLSv1.2
          KAFKA_SSL_CLIENT_AUTH: none
          KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
        options: >-
          --name=kafka1
          --health-cmd "kafka-broker-api-versions --version"
          --health-interval 10s
          --health-timeout 10s
          --health-retries 5

      loki:
        image: grafana/loki:2.9.1
        ports:
          - "3100:3100"
        options: >-
          --name=loki1
          --health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
          --health-interval 30s
          --health-timeout 10s
          --health-retries 5
          --health-start-period 30s

      victorialogs:
        image: victoriametrics/victoria-logs:v1.5.0-victorialogs
        ports:
          - "9428:9428"
        options: >-
          --name=victorialogs1
          --health-cmd "wget -q -O - http://0.0.0.0:9428"
          --health-interval 30s
          --health-timeout 10s
          --health-retries 5
          --health-start-period 30s
    steps:
      - name: Check out CrowdSec repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          submodules: true
          persist-credentials: false

      - name: "Set up Go"
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
        with:
          go-version-file: go.mod

      - name: Run "go generate" and check for changes
        run: |
          set -e
          # ensure the version of 'protoc' matches the one that generated the files
          PROTOBUF_VERSION="35.1"
          JAVY_VERSION="9.1.0"
          # sha256 of the release assets, verified before use
          PROTOBUF_SHA256="6930ebf62bd4ea607b98fff052596c6ee564b9835b4ce172c75a3f53ae9d91b7"
          JAVY_SHA256="a68b122d48eb3dfc1b801d4e14c39271fde3638243d3272d206e376ac9189e39"
          # don't pollute the repo
          pushd $HOME
          curl --fail --location --proto '=https' --tlsv1.2 -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
          echo "${PROTOBUF_SHA256}  protoc.zip" | sha256sum -c -
          unzip protoc.zip -d $HOME/.protoc
          curl --fail --location --proto '=https' --tlsv1.2 -o javy.gz https://github.com/bytecodealliance/javy/releases/download/v${JAVY_VERSION}/javy-x86_64-linux-v${JAVY_VERSION}.gz
          echo "${JAVY_SHA256}  javy.gz" | sha256sum -c -
          mkdir -p $HOME/.javy
          gzip -dc javy.gz > $HOME/.javy/javy
          chmod +x $HOME/.javy/javy
          popd
          export PATH="$HOME/.protoc/bin:$HOME/.javy/:$PATH"
          go install google.golang.org/protobuf/cmd/[email protected]
          go install google.golang.org/grpc/cmd/[email protected]
          go generate ./...
          protoc --version
          if [[ $(git status --porcelain) ]]; then
            echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code."
            git diff
            exit 1
          else
            echo "No changes detected after running 'go generate'."
          fi

      - name: Create localstack streams
        run: |
          aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
          aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2

      - name: Generate codecov configuration
        run: |
          .github/generate-codecov-yml.sh > .github/codecov.yml

      - name: Ensure we can do a dynamic build
        run: |
          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
          make build

      - name: Ensure we can do a static build
        run: |
          make clean build BUILD_STATIC=1

      - name: Unit tests
        run: |
          go install gotest.tools/[email protected]
          # make the repo read-only, with the exception of coverage output
          touch coverage.out
          chmod -R a-w .
          chmod u+w coverage.out
          make testcover
          # ignore changes to codecov.yml
          if [[ $(git status --porcelain -- . ":(exclude).github/codecov.yml") ]]; then
            echo "Error: Unit tests should not create or alter files inside the repository. Please use the appropriate testing helpers or otherwise temporary locations."
            git diff --name-only
            exit 1
          fi
          chmod -R u+w .

      # check if some component stubs are missing
      - name: "Build profile: minimal"
        run: |
          make build BUILD_PROFILE=minimal

      # check that the freebsd build is not broken
      - name: "Build target: freebsd"
        run: |
          make build BUILD_STATIC=1 BUILD_RE2_WASM=1 BUILD_SQLITE=modernc GOOS=freebsd

      - name: Upload unit coverage to Codecov
        uses: codecov/codecov-action@0b35c9ecc4f0529d0eb674914510c22f85b196b4 # v7.1.0
        with:
          files: coverage.out
          flags: unit-linux
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: golangci-lint
        uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0
        with:
          version: v2.13
          args: --issues-exit-code=1 --timeout 10m
          only-new-issues: false