From db2d1e070ebe5783bb7c78668dcdc36b23ffe609 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Fri, 10 Jan 2025 22:32:15 +0100 Subject: [PATCH] chore: lakefs CI steps Signed-off-by: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> --- .github/workflows/build.yml | 37 ++++++++++++++++++++++++++++++ .github/workflows/codecov.yml | 2 +- .github/workflows/python_build.yml | 30 ++++++++++++++++++++++-- docker-compose-lakefs.yml | 33 ++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 docker-compose-lakefs.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e206ab82..a015ff28 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,3 +139,40 @@ jobs: - name: Run tests with native-tls run: | cargo test --no-default-features --features integration_test,s3-native-tls,datafusion + + integration_test_lakefs: + name: Integration Tests (LakeFS v1.48) + runs-on: ubuntu-latest + env: + CARGO_INCREMENTAL: 0 + # Disable full debug symbol generation to speed up CI build and keep memory down + # + RUSTFLAGS: "-C debuginfo=line-tables-only" + # https://github.com/rust-lang/cargo/issues/10280 + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + RUST_BACKTRACE: "1" + RUST_LOG: debug + + steps: + - uses: actions/checkout@v3 + + - name: Install minimal stable with clippy and rustfmt + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: '1.81' + override: true + + - name: Download Lakectl + run: | + wget -q https://github.com/treeverse/lakeFS/releases/download/1.48.1/lakeFS_1.48.1_Linux_x86_64.tar.gz + tar -xf lakeFS_1.48.1_Linux_x86_64.tar.gz -C $GITHUB_WORKSPACE + echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH + + - name: Start emulated services + run: docker compose -f docker-compose-lakefs.yml up -d + + - name: Run tests with rustls (default) + run: | + cargo test --features integration_test_lakefs,lakefs,datafusion + diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index f0e68536..69212c55 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -26,7 +26,7 @@ jobs: uses: taiki-e/install-action@cargo-llvm-cov - uses: Swatinem/rust-cache@v2 - name: Generate code coverage - run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs + run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs --skip test_read_tables_lakefs - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index a76cc1f4..acdd2c9c 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -77,13 +77,38 @@ jobs: run: make setup-dat - name: Run tests - run: uv run pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules + run: uv run --no-sync pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules - name: Test without pandas run: | uv pip uninstall pandas - uv run pytest -m "not pandas and not integration and not benchmark" + uv run --no-sync pytest -m "not pandas and not integration and not benchmark" uv pip install pandas + + test-lakefs: + name: Python Build (Python 3.10 LakeFS Integration tests) + runs-on: ubuntu-latest + env: + RUSTFLAGS: "-C debuginfo=1" + CARGO_INCREMENTAL: 0 + + steps: + - uses: actions/checkout@v3 + + - name: Setup Environment + uses: ./.github/actions/setup-env + + - name: Start emulated services + run: docker compose -f ../docker-compose-lakefs.yml up -d + + - name: Build and install deltalake + run: make develop + + - name: Download Data Acceptance Tests (DAT) files + run: make setup-dat + + - name: Run tests + run: uv run --no-sync pytest -m '(lakefs and integration)' --doctest-modules test-pyspark: name: PySpark Integration Tests @@ -109,6 +134,7 @@ jobs: - name: Run tests run: make test-pyspark + multi-python-running: name: Running with Python ${{ matrix.python-version }} runs-on: ubuntu-latest diff --git a/docker-compose-lakefs.yml b/docker-compose-lakefs.yml new file mode 100644 index 00000000..a1553bca --- /dev/null +++ b/docker-compose-lakefs.yml @@ -0,0 +1,33 @@ +services: + lakefs: + image: docker.io/treeverse/lakefs:1.48 + ports: + - "8000:8000" + environment: + - LAKEFS_DATABASE_TYPE=local + - LAKEFS_BLOCKSTORE_TYPE=local + - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string + - LAKEFS_LOGGING_LEVEL=INFO + - LAKEFS_STATS_ENABLED=${LAKEFS_STATS_ENABLED:-1} + - LAKEFS_INSTALLATION_USER_NAME=delta + - LAKEFS_INSTALLATION_ACCESS_KEY_ID=LAKEFSID + - LAKECTL_CREDENTIALS_ACCESS_KEY_ID=LAKEFSID + - LAKEFS_INSTALLATION_SECRET_ACCESS_KEY=LAKEFSKEY + - LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY=LAKEFSKEY + - LAKECTL_SERVER_ENDPOINT_URL=http://localhost:8000 + entrypoint: ["/bin/sh", "-c"] + command: + - | + lakefs run --local-settings & + echo "---- Creating repository ----" + wait-for -t 60 lakefs:8000 -- lakectl repo create lakefs://bronze local://bronze || true + echo "" + echo "lakeFS Web UI: http://127.0.0.1:8000/ >(.＿.)<" + echo " ( )_ " + echo "" + echo " Access Key ID : $$LAKEFS_INSTALLATION_ACCESS_KEY_ID" + echo " Secret Access Key: $$LAKEFS_INSTALLATION_SECRET_ACCESS_KEY" + echo "" + echo "-------- Let's go and have axolotl fun! --------" + echo "" + wait \ No newline at end of file