chore: lakefs CI steps

Signed-off-by: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com>

Author: ion-elgreco

.github/workflows/build.yml

@@ -139,3 +139,40 @@ jobs:
       - name: Run tests with native-tls
         run: |
           cargo test --no-default-features --features integration_test,s3-native-tls,datafusion
+
+  integration_test_lakefs:
+    name: Integration Tests (LakeFS v1.48)
+    runs-on: ubuntu-latest
+    env:
+      CARGO_INCREMENTAL: 0
+      # Disable full debug symbol generation to speed up CI build and keep memory down
+      # <https://doc.rust-lang.org/cargo/reference/profiles.html>
+      RUSTFLAGS: "-C debuginfo=line-tables-only"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      RUST_BACKTRACE: "1"
+      RUST_LOG: debug
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: '1.81'
+          override: true
+
+      - name: Download Lakectl
+        run: |
+          wget -q https://github.com/treeverse/lakeFS/releases/download/1.48.1/lakeFS_1.48.1_Linux_x86_64.tar.gz
+          tar -xf lakeFS_1.48.1_Linux_x86_64.tar.gz -C $GITHUB_WORKSPACE
+          echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH
+
+      - name: Start emulated services
+        run: docker compose -f docker-compose-lakefs.yml up -d
+
+      - name: Run tests with rustls (default)
+        run: |
+          cargo test --features integration_test_lakefs,lakefs,datafusion
+

.github/workflows/codecov.yml

@@ -26,7 +26,7 @@ jobs:
         uses: taiki-e/install-action@cargo-llvm-cov
       - uses: Swatinem/rust-cache@v2
       - name: Generate code coverage
-        run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs
+        run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs --skip test_read_tables_lakefs
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:

.github/workflows/python_build.yml

@@ -77,13 +77,38 @@ jobs:
         run: make setup-dat
 
       - name: Run tests
-        run: uv run pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules
+        run: uv run --no-sync pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules
 
       - name: Test without pandas
         run: |
           uv pip uninstall pandas
-          uv run pytest -m "not pandas and not integration and not benchmark"
+          uv run --no-sync pytest -m "not pandas and not integration and not benchmark"
           uv pip install pandas
+  
+  test-lakefs:
+    name: Python Build (Python 3.10 LakeFS Integration tests)
+    runs-on: ubuntu-latest
+    env:
+      RUSTFLAGS: "-C debuginfo=1"
+      CARGO_INCREMENTAL: 0
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
+
+      - name: Start emulated services
+        run: docker compose -f ../docker-compose-lakefs.yml up -d
+
+      - name: Build and install deltalake
+        run: make develop
+
+      - name: Download Data Acceptance Tests (DAT) files
+        run: make setup-dat
+
+      - name: Run tests
+        run: uv run --no-sync pytest -m '(lakefs and integration)' --doctest-modules
 
   test-pyspark:
     name: PySpark Integration Tests
@@ -109,6 +134,7 @@ jobs:
       - name: Run tests
         run: make test-pyspark
 
+
   multi-python-running:
     name: Running with Python ${{ matrix.python-version }}
     runs-on: ubuntu-latest

docker-compose-lakefs.yml

@@ -0,0 +1,33 @@
+services:
+  lakefs:
+    image: docker.io/treeverse/lakefs:1.48
+    ports:
+      - "8000:8000"
+    environment:
+      - LAKEFS_DATABASE_TYPE=local
+      - LAKEFS_BLOCKSTORE_TYPE=local
+      - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string
+      - LAKEFS_LOGGING_LEVEL=INFO
+      - LAKEFS_STATS_ENABLED=${LAKEFS_STATS_ENABLED:-1}
+      - LAKEFS_INSTALLATION_USER_NAME=delta
+      - LAKEFS_INSTALLATION_ACCESS_KEY_ID=LAKEFSID
+      - LAKECTL_CREDENTIALS_ACCESS_KEY_ID=LAKEFSID
+      - LAKEFS_INSTALLATION_SECRET_ACCESS_KEY=LAKEFSKEY
+      - LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY=LAKEFSKEY
+      - LAKECTL_SERVER_ENDPOINT_URL=http://localhost:8000
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+        - |
+          lakefs run --local-settings &
+          echo "---- Creating repository ----"
+          wait-for -t 60 lakefs:8000 -- lakectl repo create lakefs://bronze local://bronze || true
+          echo ""
+          echo "lakeFS Web UI: http://127.0.0.1:8000/      >(.＿.)<"
+          echo "                                             (  )_ "
+          echo ""
+          echo "                Access Key ID    : $$LAKEFS_INSTALLATION_ACCESS_KEY_ID"
+          echo "                Secret Access Key: $$LAKEFS_INSTALLATION_SECRET_ACCESS_KEY"
+          echo ""
+          echo "-------- Let's go and have axolotl fun! --------"
+          echo ""
+          wait