datawave/.github/workflows/compose-tests.yml

name: Compose

on:
  workflow_call:
    inputs:
      allow-snapshots:
        description: 'Allow snapshots in the pom'
        required: true
        type: boolean
      ref:
        description: 'Git ref (branch/tag/SHA) to check out. Defaults to the workflow default ref.'
        required: false
        type: string
        default: ''

jobs:
  configure-runner:
    runs-on: ubuntu-latest
    steps:
      - id: set_output_step # at least one step is required in a job
        run: |
          echo "Determining runner to be ${{vars.RUNNER_TYPE}}"
          echo "runner=${{vars.RUNNER_TYPE}}" >> $GITHUB_OUTPUT
          if [[ "${{ vars.RUNNER_TYPE }}" == "self-hosted" ]]; then
            echo "We have hit the true case"
            echo "cache_settings=-Dmaven.build.cache.enabled=false -Dmaven.build.cache.remote.enabled=false -Dmaven.build.cache.remote.url=http://localhost:12345" >> $GITHUB_OUTPUT
          else
            echo "We have reached the false case"
            echo cache_settings="" >> $GITHUB_OUTPUT
          fi
          echo $GITHUB_OUTPUT
    outputs:
      runner:  ${{ steps.set_output_step.outputs.runner }}
      cache_settings:  ${{ steps.set_output_step.outputs.cache_settings }}
  compose-build-and-test:
    needs: configure-runner
    runs-on: ${{ needs.configure-runner.outputs.runner }}
    env:
      JAVA_VERSION: '11'
      JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8
      MAVEN_OPTS: "-Djansi.force=true -Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true -XX:ThreadStackSize=1m -Daether.connector.basic.threads=8 -Daether.metadataResolver.threads=8 -Daether.syncContext.named.time=480"
      GITHUB_REGISTRY: ghcr.io
      GITHUB_USERNAME: ${{ secrets.GHCR_WRITE_USER_NAME }}
      GITHUB_PASSWORD: ${{ secrets.GHCR_WRITE_ACCESS_TOKEN }}
      DW_BUILD_CACHE_OPTIONS: ${{ needs.configure-runner.outputs.cache_settings }}
      # Without this compose derives the project name from the directory, so every
      # compose job on a runner shares the project "docker" -- and `up --no-recreate`
      # then attaches to another job's containers instead of starting its own. A
      # unique name also labels this job's containers, networks and volumes with
      # com.docker.compose.project, which is what makes the teardown below scoped.
      COMPOSE_PROJECT_NAME: dw-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.allow-snapshots && 'snap' || 'rel' }}
    steps:
      - name: Set owner of working dir and m2 dir recursively
        #Needed because compose can leave around files owned by root.
        run: |
          if [[ "${{ needs.configure-runner.outputs.runner }}" == "self-hosted" ]]; then
            sudo chown -R $(whoami) . ~/.m2 || true   
          fi
      - name: Checkout Code
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref || github.sha }}
      # TODO: remove this once added to the runners
      - name: Install RPM build tools
        run: |
          if command -v rpmbuild >/dev/null 2>&1; then
            exit 0
          elif command -v dnf >/dev/null 2>&1; then
            sudo dnf install -y rpm-build
          elif command -v apt-get >/dev/null 2>&1; then
            sudo apt-get update
            sudo apt-get install -y rpm
          else
            echo "::error::Unsupported package manager"
            exit 1
          fi
      # Clean up free space
      - uses: ./.github/actions/free-space
        with:
          tool-cache: false
          # Self-hosted runners are persistent, so `docker image prune --all` deletes
          # the layer cache every build depends on and can remove images a co-tenant
          # job is still using. Hosted runners are disposable, so let it run there.
          docker-images: ${{ needs.configure-runner.outputs.runner != 'self-hosted' }}
      - name: Log in to the Container registry
        uses: docker/login-action@v4
        with:
          registry: ${{ env.GITHUB_REGISTRY }}
          username: ${{ secrets.GHCR_WRITE_USER_NAME }}
          password: ${{ secrets.GHCR_WRITE_ACCESS_TOKEN }}
      - name: Configure Maven Settings
        run: |
          mkdir -p ~/.m2
          cat > ~/.m2/settings.xml <<'EOF'
          <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                 xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
          <servers>
           <server>
             <id>github-datawave</id>
             <username>${env.GITHUB_USERNAME}</username>
             <password>${env.GITHUB_PASSWORD}</password>
           </server>
          </servers>
          <mirrors>
           <mirror>
             <id>google-maven-central</id>
             <name>Google Cloud Maven Central Mirror</name>
             <url>https://maven-central.storage-download.googleapis.com/maven2/</url>
             <mirrorOf>central</mirrorOf>
           </mirror>
          </mirrors>
          </settings>
          EOF
      - name: Set up JDK ${{ env.JAVA_VERSION }}
        uses: actions/setup-java@v5
        with:
          distribution: ${{ env.JAVA_DISTRIBUTION }}
          java-version: ${{ env.JAVA_VERSION }}
          overwrite-settings: false
      - uses: actions/cache@v5
        if: ${{ !inputs.allow-snapshots }}
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-build-
            ${{ runner.os }}-maven-format-
            ${{ runner.os }}-maven-
      # Builds the DataWave runtime images. Hadoop, ZooKeeper, and Accumulo use
      # published stack images instead of being rebuilt inside quickstart.
      - name: Docker Compose Update POMs to use Snapshots
        if: ${{ inputs.allow-snapshots }}
        run: |
          export MAVEN_OPTS="${{ env.MAVEN_OPTS }}"
          mvn --show-version --batch-mode --errors -T1C -Dutils -Dservices -Dstarters \
              versions:update-properties versions:update-parent -DallowSnapshots=true -Dincludes=gov.nsa.*
      - name: Docker Compose Build
        run: |
          export MAVEN_OPTS="${{ env.MAVEN_OPTS }}"
          mvn --show-version --batch-mode --errors -T1C -Dutils -Dservices -Dstarters \
              -Pcompose -Dmicroservice-docker -Ddocker-release ${{ needs.configure-runner.outputs.cache_settings }} -Ddeploy -Dtar -DskipTests \
              -Djkube.container-image.tags.1=latest \
              clean install
      - name: Docker Compose Clear Space
        run: |
          # free up some space so that we don't run out. This runs before the stack
          # starts, so there is nothing of this job's to reclaim yet -- the targets are
          # build cache and dangling layers. `until=24h` keeps a concurrently running
          # job's freshly built images and cache out of scope; a bare `system prune`
          # would delete them out from under it.
          docker builder prune --force --filter "until=24h"
          docker image prune --force --filter "until=24h"
          export MAVEN_OPTS="${{ env.MAVEN_OPTS }}"
          mvn --show-version --batch-mode --errors -Dutils -Dservices -Dstarters \
              -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests ${{ needs.configure-runner.outputs.cache_settings }} clean
      - name: Docker Compose Stack Start
        run: |
          cd docker
          export DW_HOST_IP=127.0.0.1
          ./bootstrap.sh
          echo "Starting docker compose"
          docker compose down --volumes --remove-orphans
          docker compose up -d --wait --wait-timeout 600
      # Ahead of the query tests, which pass whether or not the shard table is split.
      - name: Docker Compose Verify Shard Splits
        run: ./docker/scripts/verifySplits.sh
      - name: Docker Compose Microservice Query Integration Tests
        run: |
          cd docker/scripts
          ./testAll.sh
      - name: Dump Logs
        if: failure()
        run: |
          cd docker
          docker compose logs
      - name: Docker Compose Web Tests
        run: |
          HOST_PORT=localhost:9443 DW_DATAWAVE_WEB_REMOTE=true \
            bash contrib/datawave-quickstart/bin/services/datawave/test-web/run.sh --blacklist-files QueryMetrics || true
      - name: Final Operations
        if: always()
        run: |
          cd docker
          docker compose down --volumes --remove-orphans
          if [[ "${{ needs.configure-runner.outputs.runner }}" == "self-hosted" ]]; then
            sudo chown -R $(whoami) .. ~/.m2 || true
          fi