xet-core/.github/workflows/ci.yml

name: xet-core CI

on:
  push:
    branches:
      - main
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  fmt:
    name: Rustfmt
    runs-on:
      group: aws-general-8-plus
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
      - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
        with:
          toolchain: nightly
          components: rustfmt
      - name: Format
        run: |
          cargo fmt --manifest-path ./Cargo.toml --all -- --check
          cargo fmt --manifest-path ./hf_xet/Cargo.toml --all -- --check
  detect-unused-dependencies:
    runs-on:
      group: aws-general-8-plus
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
      - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
        with:
          toolchain: 1.94.1
      - name: Install cargo-machete
        run: cargo install --locked cargo-machete@0.9.2
      - name: Run cargo-machete
        run: cargo machete
  check-bench-compiles:
    name: Check benchmarks compile
    runs-on:
      group: aws-general-8-plus
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
      - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
        with:
          toolchain: 1.94.1
      - uses: ./.github/actions/cache-rust-build
      - name: Compile benchmarks
        run: |
          cargo bench --no-run --workspace --exclude git_xet
  build_and_test-linux:
    runs-on:
      group: aws-general-8-plus
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
      - name: Install Rust 1.94
        uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
        with:
          toolchain: 1.94.1
          components: clippy
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
        with:
          python-version: '3.10'
      - name: Install libpython for pyo3 linking
        run: |
          set -eux
          py_ver=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
          sudo apt-get update
          if ! apt-cache show "libpython${py_ver}-dev" >/dev/null 2>&1; then
            sudo apt-get install -y software-properties-common
            sudo add-apt-repository -y ppa:deadsnakes/ppa
            sudo apt-get update
          fi
          sudo apt-get install -y "libpython${py_ver}-dev"
      - uses: ./.github/actions/cache-rust-build
      - name: Lint
        run: |
          cargo clippy -r --verbose -- -D warnings # elevates warnings to errors
          cargo clippy -r --verbose --manifest-path hf_xet/Cargo.toml -- -D warnings # elevates warnings to errors
      - name: Set up Git LFS
        run: |
          curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
          sudo apt-get install git-lfs
          git lfs install
      - name: Build and Test
        run: |
          cargo test --verbose --no-fail-fast --features "strict simulation git-xet-for-integration-test"
      - name: Build and Test hf_xet
        run: |
          cd hf_xet && cargo test --verbose --no-fail-fast
      - name: Create venv
        run: python3 -m venv .venv
      - name: Build wheel
        uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad  # v1
        with:
          command: develop
          sccache: 'true'
          working-directory: hf_xet
      - name: Python integration tests (hf_xet)
        run: |
          source .venv/bin/activate
          pip install pytest
          pytest hf_xet/tests/ -v
      - name: Check Cargo.lock has no uncommitted changes
        run: |
          # the build and test steps would update Cargo.lock if it is out of date
          test -z "$(git status --porcelain Cargo.lock)" || (echo "Cargo.lock has uncommitted changes!" && exit 1)
  build_and_test-win:
    runs-on: windows-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - name: Install Rust 1.94
        uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
        with:
          toolchain: 1.94.1
      - uses: ./.github/actions/cache-rust-build
      - name: Build and Test
        run: |
          cargo test --verbose --no-fail-fast --features "strict simulation git-xet-for-integration-test"
      - name: Build and Test hf_xet
        run: |
          cd hf_xet && cargo test --verbose --no-fail-fast
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
        with:
          python-version: '3.10'
      - name: Create venv
        run: python -m venv .venv
      - name: Build wheel
        uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad  # v1
        with:
          command: develop
          sccache: 'true'
          working-directory: hf_xet
      - name: Python integration tests (hf_xet)
        shell: bash
        run: |
          source .venv/Scripts/activate
          pip install pytest
          pytest hf_xet/tests/ -v
  build_and_test-macos:
    runs-on: macos-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - name: Install Rust 1.94
        uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
        with:
          toolchain: 1.94.1
      - name: Set up Git LFS
        run: |
          brew install git-lfs
          git lfs install
      - uses: ./.github/actions/cache-rust-build
      - name: Build and Test
        run: |
          cargo test --verbose --no-fail-fast --features "strict simulation git-xet-for-integration-test"
      - name: Build and Test hf_xet
        run: |
          cd hf_xet && cargo test --verbose --no-fail-fast
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
        with:
          python-version: '3.10'
      - name: Create venv
        run: python3 -m venv .venv
      - name: Build wheel
        uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad  # v1
        with:
          command: develop
          sccache: 'true'
          working-directory: hf_xet
      - name: Python integration tests (hf_xet)
        run: |
          source .venv/bin/activate
          pip install pytest
          pytest hf_xet/tests/ -v
  build_and_test-wasm:
    name: Build WASM
    runs-on:
      group: aws-general-8-plus
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
      - name: Install Rust 1.94
        uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
        with:
          toolchain: 1.94.1
      - uses: ./.github/actions/build-wasm
      - name: Check hf_xet_thin_wasm Cargo.lock has no uncommitted changes
        working-directory: wasm/hf_xet_thin_wasm
        run: |
          # the build steps would update Cargo.lock if it is out of date
          test -z "$(git status --porcelain Cargo.lock)" || (echo "hf_xet_thin_wasm Cargo.lock has uncommitted changes!" && exit 1)
      - name: Check hf_xet_wasm Cargo.lock has no uncommitted changes
        working-directory: wasm/hf_xet_wasm
        run: |
          # the build steps would update Cargo.lock if it is out of date
          test -z "$(git status --porcelain Cargo.lock)" || (echo "hf_xet_wasm Cargo.lock has uncommitted changes!" && exit 1)
  cargo-audit:
    name: Cargo Audit
    runs-on:
      group: aws-general-8-plus
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
      - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
        with:
          toolchain: 1.94.1
      - name: Install cargo-audit
        run: cargo install --locked cargo-audit
      - name: Run cargo audit
        run: cargo audit -D warnings