mirror of
https://github.com/huggingface/xet-core.git
synced 2026-06-04 13:30:29 +08:00
0e1f9f4cf0
This PR builds a Git integration called `git-xet` that enables users to upload files using the Xet protocol as part of a standard git push. This integration builds on the Git LFS custom transfer adapter protocol, the same mechanism we now use to handle Git LFS uploads for files larger than 5 GB through multipart PUT. To enable uploads to Xet, users run `git-xet install`, which writes the following configuration to the Git config file at a selected scope [`--system`, `--global` (default), or `--local`]: ``` [lfs "customtransfer.xet"] path = git-xet args = transfer concurrent = true ``` This setup registers a new transfer adapter named xet, allowing Git to delegate LFS file transfers to the git-xet binary when applicable. On the server side, support is rolled out in two stages: Stage 1 (Upload): The Git LFS batch API for the "upload" operation is updated. - If a repo is Xet enabled but users didn't run git-xet install, moon-landing rejects the request when users initiated git push and returns an instruction to install git-xet. - If a repo is Xet enabled and users have git-xet configured correctly, moon-landing accepts the request and replies with CAS server URL and access token, which git-xet will use to upload files to Xet. - If a repo is NOT Xet enabled, upload goes through the LFS path.
104 lines
3.1 KiB
YAML
104 lines
3.1 KiB
YAML
name: xet-core CI
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
fmt:
|
|
name: Rustfmt
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: dtolnay/rust-toolchain@stable
|
|
with:
|
|
toolchain: nightly
|
|
components: rustfmt
|
|
- name: Format
|
|
run: |
|
|
cargo fmt --manifest-path ./Cargo.toml --all -- --check
|
|
cargo fmt --manifest-path ./hf_xet/Cargo.toml --all -- --check
|
|
|
|
build_and_test-linux:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
- name: Install Rust 1.86
|
|
uses: dtolnay/rust-toolchain@1.86.0
|
|
with:
|
|
components: clippy
|
|
- name: Lint
|
|
run: |
|
|
cargo clippy -r --verbose -- -D warnings # elevates warnings to errors
|
|
cargo clippy -r --verbose --manifest-path hf_xet/Cargo.toml -- -D warnings # elevates warnings to errors
|
|
- name: Set up Git LFS
|
|
run: |
|
|
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
|
|
sudo apt-get install git-lfs
|
|
git lfs install
|
|
- name: Build and Test
|
|
run: |
|
|
cargo test --verbose --no-fail-fast --features "strict"
|
|
- name: Check Cargo.lock has no uncommitted changes
|
|
run: |
|
|
# the build and test steps would update Cargo.lock if it is out of date
|
|
test -z "$(git status --porcelain Cargo.lock)" || (echo "Cargo.lock has uncommitted changes!" && exit 1)
|
|
build_and_test-win:
|
|
runs-on: windows-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
- name: Install Rust 1.86
|
|
uses: dtolnay/rust-toolchain@1.86.0
|
|
with:
|
|
components: clippy
|
|
- name: Build and Test
|
|
run: |
|
|
cargo test --verbose --no-fail-fast --features "strict"
|
|
build_and_test-macos:
|
|
runs-on: macos-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
- name: Install Rust 1.86
|
|
uses: dtolnay/rust-toolchain@1.86.0
|
|
with:
|
|
components: clippy
|
|
- name: Set up Git LFS
|
|
run: |
|
|
brew install git-lfs
|
|
git lfs install
|
|
- name: Build and Test
|
|
run: |
|
|
cargo test --verbose --no-fail-fast --features "strict"
|
|
build_and_test-wasm:
|
|
name: Build WASM
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
- name: Install Rust nightly
|
|
uses: dtolnay/rust-toolchain@nightly
|
|
with:
|
|
targets: wasm32-unknown-unknown
|
|
components: rust-src
|
|
- name: Install wasm-bindgen-cli and wasm-pack
|
|
run: |
|
|
cargo install --version 0.2.100 wasm-bindgen-cli
|
|
cargo install --version 0.13.1 wasm-pack
|
|
- name: Build hf_xet_thin_wasm
|
|
working-directory: hf_xet_thin_wasm
|
|
run: |
|
|
./build_wasm.sh
|
|
- name: Build hf_xet_wasm
|
|
working-directory: hf_xet_wasm
|
|
run: |
|
|
./build_wasm.sh
|