pub_soft/xet-core
3
0
Fork 0
mirror of https://github.com/huggingface/xet-core.git synced 2026-06-04 13:30:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Reduce workspace dependencies (batches 1-3) (#746)

Browse Source 
## Summary

- **Remove unused dependencies**: warp (zero imports), paste (zero
invocations), tower-service (zero imports), and heed misplacement in
xet_core_structures
- **Move mockall to dev-dependencies** in xet_client by gating
`#[automock]` with `#[cfg_attr(test, automock)]`
- **Feature-gate simulation module** behind `simulation` cargo feature
in xet_client, making axum, heed, humantime, futures-util,
human-bandwidth, and tower-http optional
- **Replace duration-str with humantime** (~2 deps vs ~78 transitive
deps) across xet_runtime, xet_client simulation, and simulation crate

## Impact

| Metric | Before | After | Change |
|---|---|---|---|
| hf-xet production deps | 371 | 321 | **-50** |
| Workspace total | 575 | 569 | -6 |

## Test plan

- [x] `cargo check --workspace` passes
- [x] `cargo check -p hf-xet` passes (without simulation feature — key
validation)
- [x] `cargo test --workspace` — all tests pass (4 pre-existing auth
test failures in git_xet unrelated to this PR)
- [x] `cargo tree -p hf-xet -e normal --prefix none | sort -u | wc -l`
confirms 321 deps

🤖 Generated with [Claude Code](https://claude.com/claude-code)

<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> **Medium Risk**
> Medium risk because it changes dependency graph and Cargo feature
gating (notably `xet-client` simulation modules and CI test features),
which can affect build/test behavior across targets despite minimal
runtime logic changes.
> 
> **Overview**
> Reduces workspace dependency surface by removing `duration-str`
(replaced with `humantime`) and trimming other transitive-heavy crates;
updates lockfiles accordingly across the workspace, `hf_xet`, and WASM
builds.
> 
> Introduces/propagates a `simulation` Cargo feature: `xet-client`’s
simulation server-related deps become optional and are only
compiled/exported when `feature = "simulation"` is enabled; `git_xet`
adds a `simulation` feature that forwards to dependent crates, and CI
now runs tests with `strict simulation git-xet-for-integration-test`.
> 
> Minor repo hygiene updates include ignoring `.claude/` in `.gitignore`
and wiring the `simulation` crate to depend on `xet-client` with
`features = ["simulation"]` (plus swapping its duration parsing helper
to `humantime`).
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
6abc194398. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Assaf Vayner
2026-03-27 09:54:36 -07:00
committed by GitHub
parent 69962587b5
commit 9c0cb6e4c8
33 changed files with 1006 additions and 3138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/ci.yml vendored
View File

@@ -61,7 +61,7 @@ jobs:
git lfs install
- name: Build and Test
run: |
cargo test --verbose --no-fail-fast --features "strict git-xet-for-integration-test"
cargo test --verbose --no-fail-fast --features "strict simulation git-xet-for-integration-test"
- name: Build and Test hf_xet
run: |
cd hf_xet && cargo test --verbose --no-fail-fast
@@ -79,7 +79,7 @@ jobs:
- uses: ./.github/actions/cache-rust-build
- name: Build and Test
run: |
cargo test --verbose --no-fail-fast --features "strict git-xet-for-integration-test"
cargo test --verbose --no-fail-fast --features "strict simulation git-xet-for-integration-test"
- name: Build and Test hf_xet
run: |
cd hf_xet && cargo test --verbose --no-fail-fast
@@ -97,7 +97,7 @@ jobs:
- uses: ./.github/actions/cache-rust-build
- name: Build and Test
run: |
cargo test --verbose --no-fail-fast --features "strict git-xet-for-integration-test"
cargo test --verbose --no-fail-fast --features "strict simulation git-xet-for-integration-test"
- name: Build and Test hf_xet
run: |
cd hf_xet && cargo test --verbose --no-fail-fast

1
.gitignore vendored
View File

@@ -1,4 +1,5 @@
**/.idea
.claude/
# Generated by Cargo
# will have compiled files and executables
debug/

363
Cargo.lock generated
View File

@@ -69,9 +69,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstream"
version = "0.6.21"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -84,15 +84,15 @@ dependencies = [
[[package]]
name = "anstyle"
version = "1.0.13"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
[[package]]
name = "anstyle-parse"
version = "0.2.7"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
dependencies = [
"utf8parse",
]
@@ -644,9 +644,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.56"
version = "1.2.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -753,9 +753,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.60"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a"
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
dependencies = [
"clap_builder",
"clap_derive",
@@ -763,21 +763,21 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.60"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876"
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
dependencies = [
"anstream",
"anstyle",
"clap_lex 1.0.0",
"clap_lex 1.1.0",
"strsim",
]
[[package]]
name = "clap_derive"
version = "4.5.55"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5"
checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
dependencies = [
"heck",
"proc-macro2",
@@ -796,9 +796,9 @@ dependencies = [
[[package]]
name = "clap_lex"
version = "1.0.0"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "cmake"
@@ -811,9 +811,9 @@ dependencies = [
[[package]]
name = "colorchoice"
version = "1.0.4"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]]
name = "colored"
@@ -1302,20 +1302,6 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "duration-str"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12494809f9915b6132014cc259c4e204ab53ab6c6dd2225672703b5359267d82"
dependencies = [
"chrono",
"rust_decimal",
"serde",
"thiserror 2.0.18",
"time",
"winnow",
]
[[package]]
name = "ecdsa"
version = "0.16.9"
@@ -1523,12 +1509,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "futures"
version = "0.3.32"
@@ -1771,7 +1751,7 @@ version = "0.2.1"
dependencies = [
"anyhow",
"async-trait",
"clap 4.5.60",
"clap 4.6.0",
"git-url-parse",
"git2",
"hf-xet",
@@ -2026,7 +2006,7 @@ dependencies = [
"async-std",
"async-trait",
"bytes",
"clap 4.5.60",
"clap 4.6.0",
"futures",
"http",
"more-asserts",
@@ -2479,9 +2459,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "iri-string"
version = "0.7.10"
version = "0.7.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb"
dependencies = [
"memchr",
"serde",
@@ -2526,7 +2506,7 @@ dependencies = [
"cesu8",
"cfg-if 1.0.4",
"combine",
"jni-sys",
"jni-sys 0.3.1",
"log",
"thiserror 1.0.69",
"walkdir",
@@ -2535,9 +2515,31 @@ dependencies = [
[[package]]
name = "jni-sys"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
dependencies = [
"jni-sys 0.4.1",
]
[[package]]
name = "jni-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
dependencies = [
"jni-sys-macros",
]
[[package]]
name = "jni-sys-macros"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
dependencies = [
"quote",
"syn 2.0.117",
]
[[package]]
name = "jobserver"
@@ -2798,16 +2800,6 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "matrixmultiply"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
dependencies = [
"autocfg",
"rawpointer",
]
[[package]]
name = "md5"
version = "0.7.0"
@@ -2835,16 +2827,6 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@@ -2904,23 +2886,6 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e"
[[package]]
name = "nalgebra"
version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b"
dependencies = [
"approx",
"matrixmultiply",
"num-complex",
"num-rational",
"num-traits",
"rand 0.8.5",
"rand_distr 0.4.3",
"simba",
"typenum",
]
[[package]]
name = "native-tls"
version = "0.2.18"
@@ -3005,15 +2970,6 @@ dependencies = [
"zeroize",
]
[[package]]
name = "num-complex"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.2.0"
@@ -3040,17 +2996,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -3101,9 +3046,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.21.3"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "once_cell_polyfill"
@@ -3131,9 +3076,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "openssl"
version = "0.10.75"
version = "0.10.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328"
checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
dependencies = [
"bitflags 2.11.0",
"cfg-if 1.0.4",
@@ -3178,9 +3123,9 @@ dependencies = [
[[package]]
name = "openssl-sys"
version = "0.9.111"
version = "0.9.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321"
checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
dependencies = [
"cc",
"libc",
@@ -3311,12 +3256,6 @@ dependencies = [
"subtle",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pastey"
version = "0.1.1"
@@ -3863,19 +3802,6 @@ version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "rand"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
dependencies = [
"fuchsia-cprng",
"libc",
"rand_core 0.3.1",
"rdrand",
"winapi",
]
[[package]]
name = "rand"
version = "0.8.5"
@@ -3917,21 +3843,6 @@ dependencies = [
"rand_core 0.9.5",
]
[[package]]
name = "rand_core"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
dependencies = [
"rand_core 0.4.2",
]
[[package]]
name = "rand_core"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]]
name = "rand_core"
version = "0.6.4"
@@ -3950,16 +3861,6 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_distr"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
dependencies = [
"num-traits",
"rand 0.8.5",
]
[[package]]
name = "rand_distr"
version = "0.5.1"
@@ -3970,12 +3871,6 @@ dependencies = [
"rand 0.9.2",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rayon"
version = "1.11.0"
@@ -3996,15 +3891,6 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "rdrand"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -4054,15 +3940,6 @@ version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "remove_dir_all"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
dependencies = [
"winapi",
]
[[package]]
name = "reqwest"
version = "0.13.2"
@@ -4292,16 +4169,6 @@ dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "rust_decimal"
version = "1.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0"
dependencies = [
"arrayvec",
"num-traits",
]
[[package]]
name = "rustc-hash"
version = "2.1.1"
@@ -4423,15 +4290,6 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d"
[[package]]
name = "safe_arch"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"
dependencies = [
"bytemuck",
]
[[package]]
name = "salsa20"
version = "0.10.2"
@@ -4468,12 +4326,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -4737,19 +4589,6 @@ dependencies = [
"rand_core 0.6.4",
]
[[package]]
name = "simba"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95"
dependencies = [
"approx",
"num-complex",
"num-traits",
"paste",
"wide",
]
[[package]]
name = "simd-adler32"
version = "0.3.8"
@@ -4769,9 +4608,9 @@ dependencies = [
"anyhow",
"bytes",
"chrono",
"clap 4.5.60",
"duration-str",
"clap 4.6.0",
"http",
"humantime",
"rand 0.9.2",
"reqwest",
"reqwest-middleware",
@@ -4896,9 +4735,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e"
dependencies = [
"approx",
"nalgebra",
"num-traits",
"rand 0.8.5",
]
[[package]]
@@ -5041,16 +4878,6 @@ version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
[[package]]
name = "tempdir"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
dependencies = [
"rand 0.4.6",
"remove_dir_all",
]
[[package]]
name = "tempfile"
version = "3.27.0"
@@ -5178,9 +5005,9 @@ dependencies = [
[[package]]
name = "tinyvec"
version = "1.10.0"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
dependencies = [
"tinyvec_macros",
]
@@ -5452,9 +5279,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.3.22"
version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
dependencies = [
"matchers",
"nu-ansi-term",
@@ -5526,12 +5353,6 @@ dependencies = [
"web-time",
]
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-ident"
version = "1.0.24"
@@ -5662,35 +5483,6 @@ dependencies = [
"try-lock",
]
[[package]]
name = "warp"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d06d9202adc1f15d709c4f4a2069be5428aa912cc025d6f268ac441ab066b0"
dependencies = [
"bytes",
"futures-util",
"headers",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"log",
"mime",
"mime_guess",
"percent-encoding",
"pin-project",
"scoped-tls",
"serde",
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-util",
"tower-service",
"tracing",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -5895,16 +5687,6 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wide"
version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"
dependencies = [
"bytemuck",
"safe_arch",
]
[[package]]
name = "winapi"
version = "0.3.9"
@@ -6279,15 +6061,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
dependencies = [
"memchr",
]
[[package]]
name = "wiremock"
version = "0.6.5"
@@ -6415,24 +6188,23 @@ dependencies = [
"axum",
"base64 0.22.1",
"bytes",
"clap 4.5.60",
"clap 4.6.0",
"crc32fast",
"ctor",
"derivative",
"duration-str",
"futures",
"futures-util",
"heed",
"http",
"httpmock",
"human-bandwidth",
"humantime",
"hyper",
"lazy_static",
"mockall",
"more-asserts",
"once_cell",
"rand 0.9.2",
"rand_distr 0.5.1",
"rand_distr",
"reqwest",
"reqwest-middleware",
"reqwest-retry",
@@ -6440,7 +6212,6 @@ dependencies = [
"serde_json",
"serde_repr",
"statrs",
"tempdir",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -6451,7 +6222,6 @@ dependencies = [
"tracing-test",
"url",
"urlencoding",
"warp",
"web-time",
"wiremock",
"xet-core-structures",
@@ -6468,7 +6238,7 @@ dependencies = [
"blake3",
"bytemuck",
"bytes",
"clap 4.5.60",
"clap 4.6.0",
"countio",
"csv",
"futures",
@@ -6476,7 +6246,6 @@ dependencies = [
"getrandom 0.4.2",
"half",
"heapify",
"heed",
"itertools 0.14.0",
"lazy_static",
"lz4_flex",
@@ -6506,7 +6275,7 @@ dependencies = [
"async-trait",
"bytes",
"chrono",
"clap 4.5.60",
"clap 4.6.0",
"criterion",
"ctor",
"dirs",
@@ -6550,10 +6319,10 @@ dependencies = [
"const-str",
"ctor",
"dirs",
"duration-str",
"futures",
"futures-util",
"git-version",
"humantime",
"konst",
"lazy_static",
"libc",

9
Cargo.toml
View File

@@ -54,8 +54,8 @@ csv = "1"
ctor = "0.6"
derivative = "2.2"
dirs = "6.0"
duration-str = "0.19"
futures = "0.3"
humantime = "2.1"
futures-util = "0.3"
gearhash = "0.1"
getrandom = { version = "0.4", features = ["wasm_js"] }
@@ -75,9 +75,7 @@ libc = "0.2"
lz4_flex = "0.13"
mockall = "0.14"
more-asserts = "0.3"
once_cell = "1.21"
oneshot = "0.1"
paste = "1.0"
pin-project = "1"
pyo3 = { version = "0.26", features = ["abi3-py37", "multiple-pymethods"] }
prometheus = "0.14"
@@ -102,14 +100,13 @@ shell-words = "1.1"
shellexpand = "3.1"
smol = "2"
static_assertions = "1.1"
statrs = "0.18"
statrs = { version = "0.18", default-features = false }
sysinfo = "0.38"
tempfile = "3.25"
thiserror = "2.0"
tokio = { version = "1.49" }
tokio-retry = "0.3"
tokio-util = { version = "0.7" }
tower-service = "0.3"
tracing = "0.1"
tracing-appender = "0.2"
tracing-log = "0.2"
@@ -119,7 +116,6 @@ url = "2.5"
urlencoding = "2.1"
uuid = "1"
walkdir = "2"
warp = { version = "0.4", features = ["server"] }
web-time = "1.1"
whoami = "2"
@@ -139,6 +135,5 @@ rand_core = "0.6"
rand_distr = "0.5"
russh = "0.55"
serial_test = "3"
tempdir = "0.3"
tracing-test = { version = "0.2", features = ["no-env-filter"] }
wiremock = "0.6"

1
git_xet/Cargo.toml
View File

@@ -39,6 +39,7 @@ serial_test = { workspace = true }
smoke-test = []
git-xet-for-integration-test = []
git2-vendored-openssl = ["git2/vendored-openssl"]
simulation = ["xet-data/simulation", "xet-client/simulation", "xet-pkg/simulation"]
# tell cargo-machete a renamed crate
[package.metadata.cargo-machete.renamed]

429
hf_xet/Cargo.lock generated
View File

@@ -187,13 +187,10 @@ checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8"
dependencies = [
"axum-core",
"bytes",
"form_urlencoded",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"itoa",
"matchit",
"memchr",
@@ -201,15 +198,10 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"serde_core",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -228,7 +220,6 @@ dependencies = [
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -246,16 +237,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "bandwidth"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a464cd54c99441ba44d3d09f6f980f8c29d068645022852ab66cbaad42ef6a0"
dependencies = [
"rustversion",
"serde",
]
[[package]]
name = "base64"
version = "0.21.7"
@@ -744,12 +725,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "downcast"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
[[package]]
name = "doxygen-rs"
version = "0.4.2"
@@ -780,20 +755,6 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "duration-str"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12494809f9915b6132014cc259c4e204ab53ab6c6dd2225672703b5359267d82"
dependencies = [
"chrono",
"rust_decimal",
"serde",
"thiserror 2.0.18",
"time",
"winnow",
]
[[package]]
name = "either"
version = "1.15.0"
@@ -906,12 +867,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fragile"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619"
[[package]]
name = "fs_extra"
version = "1.3.0"
@@ -1151,30 +1106,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "headers"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb"
dependencies = [
"base64 0.22.1",
"bytes",
"headers-core",
"http",
"httpdate",
"mime",
"sha1",
]
[[package]]
name = "headers-core"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4"
dependencies = [
"http",
]
[[package]]
name = "heapify"
version = "0.2.0"
@@ -1316,15 +1247,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "human-bandwidth"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a5afe042873d564e1fccc5d50983e1e6341ffcae8fb7603c6c542de7129a785"
dependencies = [
"bandwidth",
]
[[package]]
name = "humantime"
version = "2.3.0"
@@ -1612,9 +1534,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "iri-string"
version = "0.7.10"
version = "0.7.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb"
dependencies = [
"memchr",
"serde",
@@ -1657,9 +1579,9 @@ dependencies = [
[[package]]
name = "itoa"
version = "1.0.17"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jni"
@@ -1670,7 +1592,7 @@ dependencies = [
"cesu8",
"cfg-if 1.0.4",
"combine",
"jni-sys",
"jni-sys 0.3.1",
"log",
"thiserror 1.0.69",
"walkdir",
@@ -1679,9 +1601,31 @@ dependencies = [
[[package]]
name = "jni-sys"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
dependencies = [
"jni-sys 0.4.1",
]
[[package]]
name = "jni-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
dependencies = [
"jni-sys-macros",
]
[[package]]
name = "jni-sys-macros"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
dependencies = [
"quote",
"syn 2.0.117",
]
[[package]]
name = "jobserver"
@@ -1738,12 +1682,6 @@ version = "0.2.183"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
[[package]]
name = "libm"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libredox"
version = "0.1.14"
@@ -1821,16 +1759,6 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "matrixmultiply"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
dependencies = [
"autocfg",
"rawpointer",
]
[[package]]
name = "memchr"
version = "2.8.0"
@@ -1861,16 +1789,6 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@@ -1898,55 +1816,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "mockall"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f58d964098a5f9c6b63d0798e5372fd04708193510a7af313c22e9f29b7b620b"
dependencies = [
"cfg-if 1.0.4",
"downcast",
"fragile",
"mockall_derive",
"predicates",
"predicates-tree",
]
[[package]]
name = "mockall_derive"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca41ce716dda6a9be188b385aa78ee5260fc25cd3802cb2a8afdc6afbe6b6dbf"
dependencies = [
"cfg-if 1.0.4",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "more-asserts"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e"
[[package]]
name = "nalgebra"
version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b"
dependencies = [
"approx",
"matrixmultiply",
"num-complex",
"num-rational",
"num-traits",
"rand 0.8.5",
"rand_distr",
"simba",
"typenum",
]
[[package]]
name = "native-tls"
version = "0.2.18"
@@ -2003,25 +1878,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.2.0"
@@ -2038,26 +1894,6 @@ dependencies = [
"itoa",
]
[[package]]
name = "num-integer"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -2065,7 +1901,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
"libm",
]
[[package]]
@@ -2225,12 +2060,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -2372,32 +2201,6 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "predicates"
version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe"
dependencies = [
"anstyle",
"predicates-core",
]
[[package]]
name = "predicates-core"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144"
[[package]]
name = "predicates-tree"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2"
dependencies = [
"predicates-core",
"termtree",
]
[[package]]
name = "prettyplease"
version = "0.2.37"
@@ -2739,22 +2542,6 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_distr"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
dependencies = [
"num-traits",
"rand 0.8.5",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -2914,16 +2701,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rust_decimal"
version = "1.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0"
dependencies = [
"arrayvec",
"num-traits",
]
[[package]]
name = "rustc-demangle"
version = "0.1.27"
@@ -3014,9 +2791,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.9"
version = "0.103.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
dependencies = [
"aws-lc-rs",
"ring",
@@ -3042,15 +2819,6 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d"
[[package]]
name = "safe_arch"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"
dependencies = [
"bytemuck",
]
[[package]]
name = "same-file"
version = "1.0.6"
@@ -3069,12 +2837,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -3153,17 +2915,6 @@ dependencies = [
"zmij",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_repr"
version = "0.1.20"
@@ -3175,29 +2926,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [
"form_urlencoded",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha1"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if 1.0.4",
"cpufeatures",
"digest",
]
[[package]]
name = "sha2"
version = "0.10.9"
@@ -3265,19 +2993,6 @@ dependencies = [
"libc",
]
[[package]]
name = "simba"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95"
dependencies = [
"approx",
"num-complex",
"num-traits",
"paste",
"wide",
]
[[package]]
name = "simd-adler32"
version = "0.3.8"
@@ -3340,9 +3055,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e"
dependencies = [
"approx",
"nalgebra",
"num-traits",
"rand 0.8.5",
]
[[package]]
@@ -3365,9 +3078,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symbolic-common"
version = "12.17.2"
version = "12.17.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "751a2823d606b5d0a7616499e4130a516ebd01a44f39811be2b9600936509c23"
checksum = "52ca086c1eb5c7ee74b151ba83c6487d5d33f8c08ad991b86f3f58f6629e68d5"
dependencies = [
"debugid",
"memmap2",
@@ -3377,9 +3090,9 @@ dependencies = [
[[package]]
name = "symbolic-demangle"
version = "12.17.2"
version = "12.17.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79b237cfbe320601dd24b4ac817a5b68bb28f5508e33f08d42be0682cadc8ac9"
checksum = "baa911a28a62823aaf2cc2e074212492a3ee69d0d926cc8f5b12b4a108ff5c0c"
dependencies = [
"cpp_demangle",
"rustc-demangle",
@@ -3491,12 +3204,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "termtree"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -3779,7 +3486,6 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -3894,12 +3600,6 @@ dependencies = [
"web-time",
]
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-ident"
version = "1.0.24"
@@ -4002,35 +3702,6 @@ dependencies = [
"try-lock",
]
[[package]]
name = "warp"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d06d9202adc1f15d709c4f4a2069be5428aa912cc025d6f268ac441ab066b0"
dependencies = [
"bytes",
"futures-util",
"headers",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"log",
"mime",
"mime_guess",
"percent-encoding",
"pin-project",
"scoped-tls",
"serde",
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-util",
"tower-service",
"tracing",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -4235,16 +3906,6 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wide"
version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"
dependencies = [
"bytemuck",
"safe_arch",
]
[[package]]
name = "winapi"
version = "0.3.9"
@@ -4619,15 +4280,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
dependencies = [
"memchr",
]
[[package]]
name = "wit-bindgen"
version = "0.51.0"
@@ -4728,23 +4380,17 @@ version = "1.4.0"
dependencies = [
"anyhow",
"async-trait",
"axum",
"base64 0.22.1",
"bytes",
"clap",
"crc32fast",
"derivative",
"duration-str",
"futures",
"futures-util",
"heed",
"http",
"human-bandwidth",
"hyper",
"lazy_static",
"mockall",
"more-asserts",
"once_cell",
"rand 0.9.2",
"reqwest",
"reqwest-middleware",
@@ -4757,12 +4403,10 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"tokio-retry",
"tower-http",
"tracing",
"tracing-subscriber",
"url",
"urlencoding",
"warp",
"web-time",
"xet-core-structures",
"xet-runtime",
@@ -4786,7 +4430,6 @@ dependencies = [
"getrandom 0.4.2",
"half",
"heapify",
"heed",
"itertools 0.14.0",
"lazy_static",
"lz4_flex",
@@ -4852,10 +4495,10 @@ dependencies = [
"const-str",
"ctor",
"dirs",
"duration-str",
"futures",
"futures-util",
"git-version",
"humantime",
"konst",
"lazy_static",
"libc",

4
simulation/Cargo.toml
View File

@@ -17,11 +17,11 @@ path = "src/bin/run_upload_simulations.rs"
[dependencies]
xet-runtime = { path = "../xet_runtime" }
xet-client = { path = "../xet_client" }
xet-client = { path = "../xet_client", features = ["simulation"] }
anyhow = { workspace = true }
chrono = { workspace = true }
duration-str = { workspace = true }
humantime = { workspace = true }
clap = { workspace = true }
rand = { workspace = true }
serde = { workspace = true }

588
simulation/chunk_cache_bench/Cargo.lock generated
View File

@@ -116,9 +116,9 @@ checksum = "d67af77d68a931ecd5cbd8a3b5987d63a1d1d1278f7f6a60ae33db485cdebb69"
[[package]]
name = "arc-swap"
version = "1.8.2"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
dependencies = [
"rustversion",
]
@@ -171,9 +171,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-lc-rs"
version = "1.16.1"
version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf"
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -181,9 +181,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
version = "0.38.0"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e"
checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a"
dependencies = [
"cc",
"cmake",
@@ -191,58 +191,6 @@ dependencies = [
"fs_extra",
]
[[package]]
name = "axum"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8"
dependencies = [
"axum-core",
"bytes",
"form_urlencoded",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"serde_core",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower 0.5.3",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
dependencies = [
"bytes",
"futures-core",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "backon"
version = "0.4.4"
@@ -255,16 +203,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "bandwidth"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a464cd54c99441ba44d3d09f6f980f8c29d068645022852ab66cbaad42ef6a0"
dependencies = [
"rustversion",
"serde",
]
[[package]]
name = "base64"
version = "0.21.7"
@@ -315,9 +253,6 @@ name = "bitflags"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
dependencies = [
"serde_core",
]
[[package]]
name = "blake3"
@@ -790,15 +725,6 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
@@ -983,21 +909,6 @@ dependencies = [
"const-random",
]
[[package]]
name = "downcast"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
[[package]]
name = "doxygen-rs"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
dependencies = [
"phf 0.11.3",
]
[[package]]
name = "dtor"
version = "0.1.1"
@@ -1019,20 +930,6 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "duration-str"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12494809f9915b6132014cc259c4e204ab53ab6c6dd2225672703b5359267d82"
dependencies = [
"chrono",
"rust_decimal",
"serde",
"thiserror 2.0.18",
"time",
"winnow",
]
[[package]]
name = "either"
version = "1.15.0"
@@ -1171,12 +1068,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fragile"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619"
[[package]]
name = "fs-err"
version = "2.11.0"
@@ -1424,30 +1315,6 @@ version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "headers"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb"
dependencies = [
"base64 0.22.1",
"bytes",
"headers-core",
"http",
"httpdate",
"mime",
"sha1",
]
[[package]]
name = "headers-core"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4"
dependencies = [
"http",
]
[[package]]
name = "heapify"
version = "0.2.0"
@@ -1460,44 +1327,6 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "heed"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a56c94661ddfb51aa9cdfbf102cfcc340aa69267f95ebccc4af08d7c530d393"
dependencies = [
"bitflags 2.11.0",
"byteorder",
"heed-traits",
"heed-types",
"libc",
"lmdb-master-sys",
"once_cell",
"page_size",
"serde",
"synchronoise",
"url",
]
[[package]]
name = "heed-traits"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
[[package]]
name = "heed-types"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d"
dependencies = [
"bincode",
"byteorder",
"heed-traits",
"serde",
"serde_json",
]
[[package]]
name = "hermit-abi"
version = "0.1.19"
@@ -1582,15 +1411,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "human-bandwidth"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a5afe042873d564e1fccc5d50983e1e6341ffcae8fb7603c6c542de7129a785"
dependencies = [
"bandwidth",
]
[[package]]
name = "humantime"
version = "2.3.0"
@@ -1851,9 +1671,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "iri-string"
version = "0.7.10"
version = "0.7.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb"
dependencies = [
"memchr",
"serde",
@@ -1896,9 +1716,9 @@ dependencies = [
[[package]]
name = "itoa"
version = "1.0.17"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jni"
@@ -1909,7 +1729,7 @@ dependencies = [
"cesu8",
"cfg-if",
"combine",
"jni-sys",
"jni-sys 0.3.1",
"log",
"thiserror 1.0.69",
"walkdir",
@@ -1918,9 +1738,31 @@ dependencies = [
[[package]]
name = "jni-sys"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
dependencies = [
"jni-sys 0.4.1",
]
[[package]]
name = "jni-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
dependencies = [
"jni-sys-macros",
]
[[package]]
name = "jni-sys-macros"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
dependencies = [
"quote",
"syn 2.0.117",
]
[[package]]
name = "jobserver"
@@ -2037,17 +1879,6 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
[[package]]
name = "lmdb-master-sys"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "864808e0b19fb6dd3b70ba94ee671b82fce17554cf80aeb0a155c65bb08027df"
dependencies = [
"cc",
"doxygen-rs",
"libc",
]
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -2071,9 +1902,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lz4_flex"
version = "0.12.1"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746"
checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a"
dependencies = [
"twox-hash 2.1.2",
]
@@ -2087,22 +1918,6 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "matrixmultiply"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
dependencies = [
"autocfg",
"rawpointer",
]
[[package]]
name = "md-5"
version = "0.10.6"
@@ -2134,16 +1949,6 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "miniz_oxide"
version = "0.8.9"
@@ -2165,55 +1970,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "mockall"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f58d964098a5f9c6b63d0798e5372fd04708193510a7af313c22e9f29b7b620b"
dependencies = [
"cfg-if",
"downcast",
"fragile",
"mockall_derive",
"predicates",
"predicates-tree",
]
[[package]]
name = "mockall_derive"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca41ce716dda6a9be188b385aa78ee5260fc25cd3802cb2a8afdc6afbe6b6dbf"
dependencies = [
"cfg-if",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "more-asserts"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e"
[[package]]
name = "nalgebra"
version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b"
dependencies = [
"approx",
"matrixmultiply",
"num-complex",
"num-rational",
"num-traits",
"rand 0.8.5",
"rand_distr",
"simba",
"typenum",
]
[[package]]
name = "nanorand"
version = "0.7.0"
@@ -2284,15 +2046,6 @@ dependencies = [
"zeroize",
]
[[package]]
name = "num-complex"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.2.0"
@@ -2319,17 +2072,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -2527,16 +2269,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "page_size"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "parking_lot"
version = "0.12.5"
@@ -2560,12 +2292,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pbkdf2"
version = "0.12.2"
@@ -2601,58 +2327,16 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "phf"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
dependencies = [
"phf_macros",
"phf_shared 0.11.3",
]
[[package]]
name = "phf"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
dependencies = [
"phf_shared 0.13.1",
"phf_shared",
"serde",
]
[[package]]
name = "phf_generator"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared 0.11.3",
"rand 0.8.5",
]
[[package]]
name = "phf_macros"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
dependencies = [
"phf_generator",
"phf_shared 0.11.3",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "phf_shared"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
dependencies = [
"siphasher",
]
[[package]]
name = "phf_shared"
version = "0.13.1"
@@ -2839,32 +2523,6 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "predicates"
version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe"
dependencies = [
"anstyle",
"predicates-core",
]
[[package]]
name = "predicates-core"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144"
[[package]]
name = "predicates-tree"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2"
dependencies = [
"predicates-core",
"termtree",
]
[[package]]
name = "prettyplease"
version = "0.2.37"
@@ -3061,22 +2719,6 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_distr"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
dependencies = [
"num-traits",
"rand 0.8.5",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rayon"
version = "1.11.0"
@@ -3418,16 +3060,6 @@ dependencies = [
"ordered-multimap",
]
[[package]]
name = "rust_decimal"
version = "1.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0"
dependencies = [
"arrayvec",
"num-traits",
]
[[package]]
name = "rustc-hash"
version = "2.1.1"
@@ -3493,7 +3125,7 @@ dependencies = [
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki 0.103.9",
"rustls-webpki 0.103.10",
"subtle",
"zeroize",
]
@@ -3556,7 +3188,7 @@ dependencies = [
"rustls 0.23.37",
"rustls-native-certs 0.8.3",
"rustls-platform-verifier-android",
"rustls-webpki 0.103.9",
"rustls-webpki 0.103.10",
"security-framework 3.7.0",
"security-framework-sys",
"webpki-root-certs",
@@ -3582,9 +3214,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.9"
version = "0.103.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
dependencies = [
"aws-lc-rs",
"ring",
@@ -3621,15 +3253,6 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d"
[[package]]
name = "safe_arch"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"
dependencies = [
"bytemuck",
]
[[package]]
name = "salsa20"
version = "0.10.2"
@@ -3734,12 +3357,6 @@ dependencies = [
"parking_lot",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -3842,17 +3459,6 @@ dependencies = [
"zmij",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_repr"
version = "0.1.20"
@@ -3959,19 +3565,6 @@ dependencies = [
"rand_core 0.6.4",
]
[[package]]
name = "simba"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95"
dependencies = [
"approx",
"num-complex",
"num-traits",
"paste",
"wide",
]
[[package]]
name = "simd-adler32"
version = "0.3.8"
@@ -4066,9 +3659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e"
dependencies = [
"approx",
"nalgebra",
"num-traits",
"rand 0.8.5",
]
[[package]]
@@ -4134,15 +3725,6 @@ dependencies = [
"futures-core",
]
[[package]]
name = "synchronoise"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"
dependencies = [
"crossbeam-queue",
]
[[package]]
name = "synstructure"
version = "0.13.2"
@@ -4191,9 +3773,9 @@ dependencies = [
[[package]]
name = "tar"
version = "0.4.44"
version = "0.4.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973"
dependencies = [
"filetime",
"libc",
@@ -4224,20 +3806,14 @@ dependencies = [
[[package]]
name = "terminal_size"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0"
checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874"
dependencies = [
"rustix 1.1.4",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
name = "termtree"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
[[package]]
name = "textwrap"
version = "0.16.2"
@@ -4421,7 +3997,7 @@ dependencies = [
"log",
"parking_lot",
"percent-encoding",
"phf 0.13.1",
"phf",
"pin-project-lite",
"postgres-protocol",
"postgres-types",
@@ -4554,7 +4130,6 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -4708,12 +4283,6 @@ version = "1.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-bidi"
version = "0.3.18"
@@ -4841,35 +4410,6 @@ dependencies = [
"try-lock",
]
[[package]]
name = "warp"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d06d9202adc1f15d709c4f4a2069be5428aa912cc025d6f268ac441ab066b0"
dependencies = [
"bytes",
"futures-util",
"headers",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"log",
"mime",
"mime_guess",
"percent-encoding",
"pin-project",
"scoped-tls",
"serde",
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-util",
"tower-service",
"tracing",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -5108,16 +4648,6 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wide"
version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"
dependencies = [
"bytemuck",
"safe_arch",
]
[[package]]
name = "winapi"
version = "0.3.9"
@@ -5692,23 +5222,16 @@ version = "1.4.0"
dependencies = [
"anyhow",
"async-trait",
"axum",
"base64 0.22.1",
"bytes",
"clap 4.6.0",
"crc32fast",
"derivative",
"duration-str",
"futures",
"futures-util",
"heed",
"http",
"human-bandwidth",
"hyper",
"lazy_static",
"mockall",
"more-asserts",
"once_cell",
"rand 0.9.2",
"reqwest 0.13.2",
"reqwest-middleware",
@@ -5721,12 +5244,10 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"tokio-retry",
"tower-http",
"tracing",
"tracing-subscriber",
"url",
"urlencoding",
"warp",
"web-time",
"xet-core-structures",
"xet-runtime",
@@ -5736,7 +5257,6 @@ dependencies = [
name = "xet-core-structures"
version = "1.4.0"
dependencies = [
"anyhow",
"async-trait",
"base64 0.22.1",
"bincode",
@@ -5751,7 +5271,6 @@ dependencies = [
"getrandom 0.4.2",
"half",
"heapify",
"heed",
"itertools 0.14.0",
"lazy_static",
"lz4_flex",
@@ -5775,6 +5294,7 @@ dependencies = [
name = "xet-runtime"
version = "1.4.0"
dependencies = [
"anyhow",
"async-trait",
"bytes",
"chrono",
@@ -5782,10 +5302,10 @@ dependencies = [
"const-str",
"ctor",
"dirs",
"duration-str",
"futures",
"futures-util",
"git-version",
"humantime",
"konst",
"lazy_static",
"libc",
@@ -5833,18 +5353,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.42"
version = "0.8.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.42"
version = "0.8.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
dependencies = [
"proc-macro2",
"quote",

4
simulation/src/bin/run_upload_scenario.rs
View File

@@ -127,9 +127,9 @@ async fn run_added_uploads(mut scenario: SimulationScenario, duration_sec: u64)
// ── Utilities ────────────────────────────────────────────────────────────────
/// Parses a duration string (duration_str crate: e.g. "10ms", "1s").
/// Parses a duration string (humantime crate: e.g. "10ms", "1s").
fn parse_duration(s: &str) -> ScenarioResult<Duration> {
duration_str::parse(s).map_err(|e| ScenarioError::Scenario(format!("invalid duration {:?}: {}", s, e)))
humantime::parse_duration(s).map_err(|e| ScenarioError::Scenario(format!("invalid duration {:?}: {}", s, e)))
}
fn setup_logging(out_dir: &Path) {

568
wasm/hf_xet_thin_wasm/Cargo.lock generated
View File

@@ -22,9 +22,9 @@ dependencies = [
[[package]]
name = "anstream"
version = "0.6.21"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -37,15 +37,15 @@ dependencies = [
[[package]]
name = "anstyle"
version = "1.0.13"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
[[package]]
name = "anstyle-parse"
version = "0.2.7"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
dependencies = [
"utf8parse",
]
@@ -122,9 +122,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-lc-rs"
version = "1.16.1"
version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf"
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -132,9 +132,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
version = "0.38.0"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e"
checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a"
dependencies = [
"cc",
"cmake",
@@ -142,68 +142,6 @@ dependencies = [
"fs_extra",
]
[[package]]
name = "axum"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8"
dependencies = [
"axum-core",
"bytes",
"form_urlencoded",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"serde_core",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
dependencies = [
"bytes",
"futures-core",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "bandwidth"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a464cd54c99441ba44d3d09f6f980f8c29d068645022852ab66cbaad42ef6a0"
dependencies = [
"rustversion",
"serde",
]
[[package]]
name = "base64"
version = "0.22.1"
@@ -288,9 +226,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
[[package]]
name = "cc"
version = "1.2.56"
version = "1.2.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -337,9 +275,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.60"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a"
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
dependencies = [
"clap_builder",
"clap_derive",
@@ -347,9 +285,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.60"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876"
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
dependencies = [
"anstream",
"anstyle",
@@ -359,9 +297,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.55"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5"
checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
dependencies = [
"heck",
"proc-macro2",
@@ -371,9 +309,9 @@ dependencies = [
[[package]]
name = "clap_lex"
version = "1.0.0"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "cmake"
@@ -386,9 +324,9 @@ dependencies = [
[[package]]
name = "colorchoice"
version = "1.0.4"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]]
name = "colored"
@@ -622,12 +560,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "downcast"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
[[package]]
name = "doxygen-rs"
version = "0.4.2"
@@ -658,20 +590,6 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "duration-str"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12494809f9915b6132014cc259c4e204ab53ab6c6dd2225672703b5359267d82"
dependencies = [
"chrono",
"rust_decimal",
"serde",
"thiserror 2.0.18",
"time",
"winnow",
]
[[package]]
name = "either"
version = "1.15.0"
@@ -727,12 +645,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fragile"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619"
[[package]]
name = "fs_extra"
version = "1.3.0"
@@ -908,25 +820,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "h2"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
dependencies = [
"atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
"http",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "half"
version = "2.7.1"
@@ -953,30 +846,6 @@ version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "headers"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb"
dependencies = [
"base64",
"bytes",
"headers-core",
"http",
"httpdate",
"mime",
"sha1",
]
[[package]]
name = "headers-core"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4"
dependencies = [
"http",
]
[[package]]
name = "heapify"
version = "0.2.0"
@@ -1079,19 +948,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
[[package]]
name = "httpdate"
version = "1.0.3"
name = "humantime"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "human-bandwidth"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a5afe042873d564e1fccc5d50983e1e6341ffcae8fb7603c6c542de7129a785"
dependencies = [
"bandwidth",
]
checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
[[package]]
name = "hyper"
@@ -1103,11 +963,9 @@ dependencies = [
"bytes",
"futures-channel",
"futures-core",
"h2",
"http",
"http-body",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"pin-utils",
@@ -1309,9 +1167,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "iri-string"
version = "0.7.10"
version = "0.7.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb"
dependencies = [
"memchr",
"serde",
@@ -1334,9 +1192,9 @@ dependencies = [
[[package]]
name = "itoa"
version = "1.0.17"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jni"
@@ -1347,7 +1205,7 @@ dependencies = [
"cesu8",
"cfg-if 1.0.4",
"combine",
"jni-sys",
"jni-sys 0.3.1",
"log",
"thiserror 1.0.69",
"walkdir",
@@ -1356,9 +1214,31 @@ dependencies = [
[[package]]
name = "jni-sys"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
dependencies = [
"jni-sys 0.4.1",
]
[[package]]
name = "jni-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
dependencies = [
"jni-sys-macros",
]
[[package]]
name = "jni-sys-macros"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
dependencies = [
"quote",
"syn 2.0.117",
]
[[package]]
name = "jobserver"
@@ -1415,12 +1295,6 @@ version = "0.2.183"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
[[package]]
name = "libm"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libredox"
version = "0.1.14"
@@ -1492,44 +1366,12 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "matrixmultiply"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
dependencies = [
"autocfg",
"rawpointer",
]
[[package]]
name = "memchr"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "mime"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "mio"
version = "1.1.1"
@@ -1541,55 +1383,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "mockall"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f58d964098a5f9c6b63d0798e5372fd04708193510a7af313c22e9f29b7b620b"
dependencies = [
"cfg-if 1.0.4",
"downcast",
"fragile",
"mockall_derive",
"predicates",
"predicates-tree",
]
[[package]]
name = "mockall_derive"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca41ce716dda6a9be188b385aa78ee5260fc25cd3802cb2a8afdc6afbe6b6dbf"
dependencies = [
"cfg-if 1.0.4",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "more-asserts"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e"
[[package]]
name = "nalgebra"
version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b"
dependencies = [
"approx",
"matrixmultiply",
"num-complex",
"num-rational",
"num-traits",
"rand 0.8.5",
"rand_distr",
"simba",
"typenum",
]
[[package]]
name = "ntapi"
version = "0.4.3"
@@ -1608,51 +1407,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
[[package]]
name = "num-integer"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -1660,7 +1420,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
"libm",
]
[[package]]
@@ -1684,9 +1443,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.21.3"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "once_cell_polyfill"
@@ -1754,12 +1513,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -1864,32 +1617,6 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "predicates"
version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe"
dependencies = [
"anstyle",
"predicates-core",
]
[[package]]
name = "predicates-core"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144"
[[package]]
name = "predicates-tree"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2"
dependencies = [
"predicates-core",
"termtree",
]
[[package]]
name = "prettyplease"
version = "0.2.37"
@@ -2080,22 +1807,6 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_distr"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
dependencies = [
"num-traits",
"rand 0.8.5",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -2243,16 +1954,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rust_decimal"
version = "1.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0"
dependencies = [
"arrayvec",
"num-traits",
]
[[package]]
name = "rustc-hash"
version = "2.1.1"
@@ -2337,9 +2038,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.9"
version = "0.103.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
dependencies = [
"aws-lc-rs",
"ring",
@@ -2365,15 +2066,6 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d"
[[package]]
name = "safe_arch"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"
dependencies = [
"bytemuck",
]
[[package]]
name = "same-file"
version = "1.0.6"
@@ -2392,12 +2084,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -2487,17 +2173,6 @@ dependencies = [
"zmij",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_repr"
version = "0.1.20"
@@ -2509,29 +2184,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [
"form_urlencoded",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha1"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if 1.0.4",
"cpufeatures",
"digest",
]
[[package]]
name = "sha2"
version = "0.10.9"
@@ -2579,19 +2231,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "simba"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95"
dependencies = [
"approx",
"num-complex",
"num-traits",
"paste",
"wide",
]
[[package]]
name = "siphasher"
version = "1.0.2"
@@ -2639,9 +2278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e"
dependencies = [
"approx",
"nalgebra",
"num-traits",
"rand 0.8.5",
]
[[package]]
@@ -2755,12 +2392,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "termtree"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -2853,9 +2484,9 @@ dependencies = [
[[package]]
name = "tinyvec"
version = "1.10.0"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
dependencies = [
"tinyvec_macros",
]
@@ -2939,7 +2570,6 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -2978,7 +2608,6 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -3040,9 +2669,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.3.22"
version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
dependencies = [
"matchers",
"nu-ansi-term",
@@ -3093,12 +2722,6 @@ dependencies = [
"web-time",
]
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-ident"
version = "1.0.24"
@@ -3189,35 +2812,6 @@ dependencies = [
"try-lock",
]
[[package]]
name = "warp"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d06d9202adc1f15d709c4f4a2069be5428aa912cc025d6f268ac441ab066b0"
dependencies = [
"bytes",
"futures-util",
"headers",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"log",
"mime",
"mime_guess",
"percent-encoding",
"pin-project",
"scoped-tls",
"serde",
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-util",
"tower-service",
"tracing",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -3432,16 +3026,6 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wide"
version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"
dependencies = [
"bytemuck",
"safe_arch",
]
[[package]]
name = "winapi"
version = "0.3.9"
@@ -3816,15 +3400,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
dependencies = [
"memchr",
]
[[package]]
name = "wit-bindgen"
version = "0.51.0"
@@ -3925,23 +3500,17 @@ version = "1.4.0"
dependencies = [
"anyhow",
"async-trait",
"axum",
"base64",
"bytes",
"clap",
"crc32fast",
"derivative",
"duration-str",
"futures",
"futures-util",
"heed",
"http",
"human-bandwidth",
"hyper",
"lazy_static",
"mockall",
"more-asserts",
"once_cell",
"rand 0.9.2",
"reqwest",
"reqwest-middleware",
@@ -3954,12 +3523,10 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"tokio-retry",
"tower-http",
"tracing",
"tracing-subscriber",
"url",
"urlencoding",
"warp",
"web-time",
"xet-core-structures",
"xet-runtime",
@@ -3983,7 +3550,6 @@ dependencies = [
"getrandom 0.4.2",
"half",
"heapify",
"heed",
"itertools",
"lazy_static",
"lz4_flex",
@@ -4048,10 +3614,10 @@ dependencies = [
"const-str",
"ctor",
"dirs",
"duration-str",
"futures",
"futures-util",
"git-version",
"humantime",
"konst",
"lazy_static",
"libc",
@@ -4099,18 +3665,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.42"
version = "0.8.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.42"
version = "0.8.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
dependencies = [
"proc-macro2",
"quote",

724
wasm/hf_xet_wasm/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

17
xet_client/Cargo.toml
View File

@@ -24,9 +24,7 @@ derivative = { workspace = true }
futures = { workspace = true }
http = { workspace = true }
lazy_static = { workspace = true }
mockall = { workspace = true }
more-asserts = { workspace = true }
once_cell = { workspace = true }
rand = { workspace = true }
reqwest = { workspace = true }
reqwest-middleware = { workspace = true }
@@ -48,15 +46,14 @@ tokio = { workspace = true, features = ["sync", "macros", "io-util", "rt", "time
web-time = { workspace = true }
[target.'cfg(not(target_family = "wasm"))'.dependencies]
axum = { workspace = true }
duration-str = { workspace = true }
futures-util = { workspace = true }
axum = { workspace = true, optional = true }
humantime = { workspace = true, optional = true }
futures-util = { workspace = true, optional = true }
heed = { workspace = true }
human-bandwidth = { workspace = true }
human-bandwidth = { workspace = true, optional = true }
hyper = { workspace = true }
tokio = { workspace = true }
tower-http = { version = "0.6", features = ["cors"] }
warp = { workspace = true }
tower-http = { version = "0.6", features = ["cors"], optional = true }
[features]
strict = []
@@ -67,10 +64,12 @@ rustls-tls = ["reqwest/rustls"]
native-tls = ["reqwest/native-tls"]
native-tls-vendored = ["reqwest/native-tls-vendored"]
analysis = []
simulation = ["dep:axum", "dep:humantime", "dep:futures-util", "dep:human-bandwidth", "dep:tower-http"]
[[bin]]
name = "local_cas_server"
path = "src/cas_client/simulation/local_server/main.rs"
required-features = ["simulation"]
[[bin]]
name = "cache_analysis"
@@ -80,10 +79,10 @@ required-features = ["analysis"]
[dev-dependencies]
approx = { workspace = true }
ctor = { workspace = true }
mockall = { workspace = true }
httpmock = { workspace = true }
rand_distr = { workspace = true }
serde_json = { workspace = true }
tempdir = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread"] }
tracing-test = { workspace = true }
wiremock = { workspace = true }

6
xet_client/src/cas_client/mod.rs
View File

@@ -3,9 +3,11 @@ pub use interface::{Client, URLProvider};
pub use remote_client::RemoteClient;
pub use simulation::{ClientTestingUtils, DirectAccessClient, MemoryClient, RandomFileContents, RandomXorb};
#[cfg(not(target_family = "wasm"))]
pub use simulation::{DeletionControlableClient, LocalClient};
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
pub use simulation::{
DeletionControlableClient, LocalClient, LocalServer, LocalServerConfig, LocalTestServer, LocalTestServerBuilder,
RemoteSimulationClient, SimulationControlClient,
LocalServer, LocalServerConfig, LocalTestServer, LocalTestServerBuilder, RemoteSimulationClient,
SimulationControlClient,
};
use tracing::Level;

1
xet_client/src/cas_client/remote_client.rs
View File

@@ -118,6 +118,7 @@ impl RemoteClient {
&self.endpoint
}
#[cfg(feature = "simulation")]
pub(crate) fn http_client(&self) -> Arc<ClientWithMiddleware> {
self.http_client.clone()
}

32
xet_client/src/cas_client/simulation/local_server/handlers.rs
View File

@@ -18,14 +18,13 @@ use std::sync::Arc;
use axum::Json;
use axum::body::Body;
use axum::extract::{Path, State};
use axum::http::header::HOST;
use axum::http::{HeaderMap, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use base64::Engine;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use bytes::Bytes;
use futures_util::StreamExt;
use http::header::RANGE;
use http::header::{HOST, RANGE};
use http::{HeaderMap, HeaderValue, StatusCode, Uri};
use xet_core_structures::merklehash::MerkleHash;
use super::super::super::{DeletionControlableClient, DirectAccessClient};
@@ -338,11 +337,7 @@ fn transform_v2_xorb_urls(response: &mut QueryReconstructionResponseV2, base_url
/// Response: Map of file ID -> reconstruction info
///
/// The URLs in fetch_info are transformed from local file paths to HTTP URLs.
pub async fn batch_get_reconstruction(
State(state): State<ServerState>,
uri: axum::http::Uri,
headers: HeaderMap,
) -> Response {
pub async fn batch_get_reconstruction(State(state): State<ServerState>, uri: Uri, headers: HeaderMap) -> Response {
let connection_guard = state.latency_simulation.register_connection().await;
if let Some(simulated_error) = connection_guard.simulate_error() {
return simulated_error;
@@ -391,7 +386,7 @@ pub async fn batch_get_reconstruction(
/// concatenated in order, allowing a single request to serve multi-range blocks.
///
/// Returns raw (compressed) bytes that the client will decompress.
pub async fn fetch_term(State(state): State<ServerState>, uri: axum::http::Uri, headers: HeaderMap) -> Response {
pub async fn fetch_term(State(state): State<ServerState>, uri: Uri, headers: HeaderMap) -> Response {
let connection_guard = state.latency_simulation.register_connection().await;
if let Some(simulated_error) = connection_guard.simulate_error() {
return simulated_error;
@@ -679,7 +674,7 @@ async fn collect_body(body: Body) -> Result<Bytes, String> {
/// Parses a duration range string in the format "(min, max)" where min and max are duration strings.
///
/// Supports duration formats like "10ms", "1s", "500us", etc. (via duration_str crate).
/// Supports duration formats like "10ms", "1s", "500us", etc. (via humantime crate).
///
/// Examples:
/// - "(10ms, 100ms)" -> Duration range from 10ms to 100ms
@@ -701,8 +696,8 @@ fn parse_duration_range(value: &str) -> Result<std::ops::Range<std::time::Durati
return Err(format!("Expected format '(min, max)' with two duration values, got: {value}"));
}
let min = duration_str::parse(parts[0]).map_err(|e| format!("Invalid min duration '{}': {e}", parts[0]))?;
let max = duration_str::parse(parts[1]).map_err(|e| format!("Invalid max duration '{}': {e}", parts[1]))?;
let min = humantime::parse_duration(parts[0]).map_err(|e| format!("Invalid min duration '{}': {e}", parts[0]))?;
let max = humantime::parse_duration(parts[1]).map_err(|e| format!("Invalid max duration '{}': {e}", parts[1]))?;
if min > max {
return Err(format!("Min duration ({:?}) cannot be greater than max ({:?})", min, max));
@@ -711,12 +706,9 @@ fn parse_duration_range(value: &str) -> Result<std::ops::Range<std::time::Durati
Ok(min..max)
}
pub async fn set_config(State(state): State<ServerState>, uri: axum::http::Uri, body: Body) -> Response {
pub async fn set_config(State(state): State<ServerState>, uri: Uri, body: Body) -> Response {
// Try to parse as JSON body first
let body_bytes = match axum::body::to_bytes(body, 1_048_576).await {
Ok(bytes) => bytes,
Err(_) => Bytes::new(),
};
let body_bytes = axum::body::to_bytes(body, 1_048_576).await.unwrap_or_else(|_| Bytes::new());
if !body_bytes.is_empty() {
match serde_json::from_slice::<ServerLatencyProfile>(&body_bytes) {
@@ -796,8 +788,10 @@ fn parse_random_delay_value(value: &str) -> Result<(u64, u64), String> {
if parts.len() != 2 {
return Err(format!("Expected format '(min, max)' with two duration values, got: {value}"));
}
let min_dur = duration_str::parse(parts[0]).map_err(|e| format!("Invalid min duration '{}': {e}", parts[0]))?;
let max_dur = duration_str::parse(parts[1]).map_err(|e| format!("Invalid max duration '{}': {e}", parts[1]))?;
let min_dur =
humantime::parse_duration(parts[0]).map_err(|e| format!("Invalid min duration '{}': {e}", parts[0]))?;
let max_dur =
humantime::parse_duration(parts[1]).map_err(|e| format!("Invalid max duration '{}': {e}", parts[1]))?;
if min_dur > max_dur {
return Err("Min duration cannot be greater than max".to_string());
}

63
xet_client/src/cas_client/simulation/mod.rs
View File

@@ -11,47 +11,54 @@
//! - [`client_unit_testing`]: Common unit tests for Client implementations (not available in WASM)
//! - [`network_simulation`]: Bandwidth-limit proxy and network profiles (not available in WASM)
// --- Always-available lightweight types ---
pub mod client_testing_utils;
#[cfg(all(test, not(target_family = "wasm")))]
pub mod client_unit_testing;
#[cfg(not(target_family = "wasm"))]
mod deletion_controls;
#[cfg(all(test, not(target_family = "wasm")))]
pub mod deletion_unit_testing;
mod direct_access_client;
#[cfg(not(target_family = "wasm"))]
mod local_client;
#[cfg(not(target_family = "wasm"))]
pub mod local_server;
mod memory_client;
#[cfg(not(target_family = "wasm"))]
pub mod network_simulation;
mod random_xorb;
#[cfg(not(target_family = "wasm"))]
mod simulation_client;
#[cfg(not(target_family = "wasm"))]
mod simulation_server;
#[cfg(unix)]
#[cfg(not(target_family = "wasm"))]
pub mod socket_proxy;
pub(crate) mod xorb_utils;
pub use client_testing_utils::{ClientTestingUtils, RandomFileContents};
pub use direct_access_client::DirectAccessClient;
pub use memory_client::MemoryClient;
pub use random_xorb::RandomXorb;
// --- Non-WASM types that don't need heavy deps ---
#[cfg(not(target_family = "wasm"))]
mod deletion_controls;
#[cfg(not(target_family = "wasm"))]
mod local_client;
#[cfg(not(target_family = "wasm"))]
pub use deletion_controls::DeletionControlableClient;
pub use direct_access_client::DirectAccessClient;
#[cfg(not(target_family = "wasm"))]
pub use local_client::LocalClient;
#[cfg(not(target_family = "wasm"))]
// --- Test-only modules (always available for non-WASM) ---
#[cfg(all(test, not(target_family = "wasm")))]
pub mod client_unit_testing;
#[cfg(all(test, not(target_family = "wasm")))]
pub mod deletion_unit_testing;
// --- Heavy deps: require "simulation" feature (axum, tower-http, etc.) ---
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
pub mod local_server;
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
pub mod network_simulation;
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
mod simulation_client;
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
mod simulation_server;
#[cfg(all(feature = "simulation", unix, not(target_family = "wasm")))]
pub mod socket_proxy;
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
pub use local_server::{LocalServer, LocalServerConfig, SimulationControlClient};
pub use memory_client::MemoryClient;
#[cfg(not(target_family = "wasm"))]
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
pub use network_simulation::{NetworkConfig, NetworkProfile, NetworkProfileOptions, NetworkSimulationProxy};
pub use random_xorb::RandomXorb;
#[cfg(not(target_family = "wasm"))]
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
pub use simulation_client::RemoteSimulationClient;
#[cfg(not(target_family = "wasm"))]
#[cfg(all(feature = "simulation", not(target_family = "wasm")))]
pub use simulation_server::{LocalTestServer, LocalTestServerBuilder};
#[cfg(unix)]
#[cfg(not(target_family = "wasm"))]
#[cfg(all(feature = "simulation", unix, not(target_family = "wasm")))]
pub use socket_proxy::UnixSocketProxy;

6
xet_client/src/chunk_cache/cache_manager.rs
View File

@@ -1,16 +1,14 @@
use std::cell::RefCell;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex, Weak};
use once_cell::sync::Lazy;
use std::sync::{Arc, LazyLock, Mutex, Weak};
use super::error::ChunkCacheError;
use super::{CacheConfig, ChunkCache, DiskCache};
// single instance of CACHE_MANAGER not exposed to outside users that
// dedupes cache instances based on configurations
static CACHE_MANAGER: Lazy<CacheManager> = Lazy::new(CacheManager::new);
static CACHE_MANAGER: LazyLock<CacheManager> = LazyLock::new(CacheManager::new);
/// get_cache attempts to return a cache given the provided config parameter
pub fn get_cache(config: &CacheConfig) -> Result<Arc<dyn ChunkCache>, ChunkCacheError> {

34
xet_client/src/chunk_cache/disk.rs
View File

@@ -816,7 +816,7 @@ mod tests {
use rand::SeedableRng;
use rand::rngs::StdRng;
use tempdir::TempDir;
use tempfile::TempDir;
use xet_runtime::utils::output_bytes;
use super::super::{CacheConfig, ChunkCache};
@@ -831,7 +831,7 @@ mod tests {
#[tokio::test]
async fn test_get_cache_empty() {
let mut rng = StdRng::seed_from_u64(RANDOM_SEED);
let cache_root = TempDir::new("empty").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: DEFAULT_CHUNK_CACHE_CAPACITY,
@@ -850,7 +850,7 @@ mod tests {
#[tokio::test]
async fn test_put_get_simple() {
let mut rng = StdRng::seed_from_u64(RANDOM_SEED);
let cache_root = TempDir::new("put_get_simple").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: DEFAULT_CHUNK_CACHE_CAPACITY,
@@ -882,7 +882,7 @@ mod tests {
#[tokio::test]
async fn test_put_get_subrange() {
let mut rng = StdRng::seed_from_u64(RANDOM_SEED);
let cache_root = TempDir::new("put_get_subrange").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: DEFAULT_CHUNK_CACHE_CAPACITY,
@@ -929,7 +929,7 @@ mod tests {
async fn test_puts_eviction() {
const MIN_NUM_KEYS: u32 = 12;
const CAP: u64 = (RANGE_LEN * (MIN_NUM_KEYS - 1)) as u64;
let cache_root = TempDir::new("puts_eviction").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: CAP,
@@ -953,7 +953,7 @@ mod tests {
#[tokio::test]
async fn test_same_puts_noop() {
let cache_root = TempDir::new("same_puts_noop").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: DEFAULT_CHUNK_CACHE_CAPACITY,
@@ -970,7 +970,7 @@ mod tests {
async fn test_overlap_range_data_mismatch_fail() {
let setup = || async move {
let mut it = RandomEntryIterator::std_from_seed(RANDOM_SEED);
let cache_root = TempDir::new("overlap_range_data_mismatch_fail").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: DEFAULT_CHUNK_CACHE_CAPACITY,
@@ -1021,7 +1021,7 @@ mod tests {
#[tokio::test]
async fn test_initialize_non_empty() {
let cache_root = TempDir::new("initialize_non_empty").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: DEFAULT_CHUNK_CACHE_CAPACITY,
@@ -1054,7 +1054,7 @@ mod tests {
#[tokio::test]
async fn test_initialize_too_large_file() {
const LARGE_FILE: u64 = 1000;
let cache_root = TempDir::new("initialize_too_large_file").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: DEFAULT_CHUNK_CACHE_CAPACITY,
@@ -1078,7 +1078,7 @@ mod tests {
#[tokio::test]
async fn test_initialize_stops_loading_early_with_too_many_files() {
const LARGE_FILE: u64 = 1000;
let cache_root = TempDir::new("initialize_stops_loading_early_with_too_many_files").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: LARGE_FILE * 10,
@@ -1111,7 +1111,7 @@ mod tests {
#[tokio::test]
async fn test_unknown_eviction() {
let cache_root = TempDir::new("initialize_non_empty").unwrap();
let cache_root = TempDir::new().unwrap();
let capacity = 12 * RANGE_LEN as u64;
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
@@ -1152,7 +1152,7 @@ mod tests {
#[tokio::test]
async fn put_subrange() {
let cache_root = TempDir::new("put_subrange").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: DEFAULT_CHUNK_CACHE_CAPACITY,
@@ -1205,7 +1205,7 @@ mod tests {
#[tokio::test]
async fn test_evictions_with_multiple_range_per_key() {
const NUM: u32 = 12;
let cache_root = TempDir::new("multiple_range_per_key").unwrap();
let cache_root = TempDir::new().unwrap();
let capacity = (NUM * RANGE_LEN) as u64;
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
@@ -1258,7 +1258,7 @@ mod tests {
#[cfg(test)]
mod concurrency_tests {
use tempdir::TempDir;
use tempfile::TempDir;
use super::super::{CacheConfig, ChunkCache};
use super::DiskCache;
@@ -1271,7 +1271,7 @@ mod concurrency_tests {
#[tokio::test]
async fn test_run_concurrently() {
let cache_root = TempDir::new("run_concurrently").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
@@ -1307,7 +1307,7 @@ mod concurrency_tests {
#[tokio::test]
#[cfg_attr(feature = "smoke-test", ignore)]
async fn test_run_concurrently_with_evictions() {
let cache_root = TempDir::new("run_concurrently_with_evictions").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: RANGE_LEN as u64 * NUM_ITEMS_PER_TASK as u64,
@@ -1341,7 +1341,7 @@ mod concurrency_tests {
#[tokio::test(flavor = "multi_thread")]
async fn test_run_concurrently_thundering_herd() {
let cache_root = TempDir::new("run_concurrently_thundering_herd").unwrap();
let cache_root = TempDir::new().unwrap();
let config = CacheConfig {
cache_directory: cache_root.path().to_path_buf(),
cache_size: RANGE_LEN as u64 * NUM_ITEMS_PER_TASK as u64,

3
xet_client/src/chunk_cache/mod.rs
View File

@@ -9,6 +9,7 @@ pub use cache_manager::get_cache;
pub use disk::DiskCache;
pub use disk::test_utils::*;
use error::ChunkCacheError;
#[cfg(test)]
use mockall::automock;
use xet_runtime::core::xet_config;
@@ -34,7 +35,7 @@ pub struct CacheRange {
///
/// implementors are allowed to evict data, a get after a put is not required to
/// be a cache hit.
#[automock]
#[cfg_attr(test, automock)]
#[async_trait]
pub trait ChunkCache: Sync + Send {
/// get should return an Ok() variant if significant error occurred, check the error

1
xet_client/tests/test_shard_upload_timeout.rs
View File

@@ -1,3 +1,4 @@
#![cfg(feature = "simulation")]
//! Integration tests for the shard upload no-read-timeout client (XET-885).
//!
//! Verifies that shard uploads succeed even when the server takes a long time to process,

1
xet_core_structures/Cargo.toml
View File

@@ -50,7 +50,6 @@ tracing = { workspace = true }
[target.'cfg(not(target_family = "wasm"))'.dependencies]
bincode = { workspace = true }
bytemuck = { workspace = true }
heed = { workspace = true }
tokio = { workspace = true, features = ["time", "rt", "macros", "sync", "test-util", "io-util", "rt-multi-thread"] }
tokio-util = { workspace = true, features = ["io"] }
uuid = { workspace = true, features = ["v4"] }

1
xet_data/Cargo.toml
View File

@@ -90,3 +90,4 @@ strict = []
smoke-test = []
expensive_tests = []
python = ["dep:pyo3"]
simulation = ["xet-client/simulation"]

839
xet_data/src/file_reconstruction/file_reconstructor.rs
View File

@@ -401,7 +401,6 @@ mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio_util::sync::CancellationToken;
use xet_client::cas_client::{ClientTestingUtils, DirectAccessClient, LocalClient, RandomFileContents};
use xet_client::cas_types::FileRange;
use xet_runtime::core::XetRuntime;
@@ -1315,227 +1314,6 @@ mod tests {
assert_eq!(&result[start as usize..end as usize], &file_contents.data[start as usize..end as usize]);
}
// ==================== V1 Fallback Tests ====================
//
// These tests use LocalTestServer with V2 disabled to verify that
// reconstruction works correctly when the client falls back from V2 to V1.
/// Helper to reconstruct through a LocalTestServer (RemoteClient HTTP path).
async fn reconstruct_via_server(
server: &xet_client::cas_client::LocalTestServer,
file_hash: MerkleHash,
byte_range: Option<FileRange>,
config: &ReconstructionConfig,
) -> Result<Vec<u8>> {
let buffer = Arc::new(std::sync::Mutex::new(Cursor::new(Vec::new())));
let writer = StaticCursorWriter(buffer.clone());
let client: Arc<dyn Client> = server.remote_client().clone();
let mut reconstructor = FileReconstructor::new(&client, file_hash).with_config(config);
if let Some(range) = byte_range {
reconstructor = reconstructor.with_byte_range(range);
}
reconstructor.reconstruct_to_writer(writer).await?;
let data = buffer.lock().unwrap().get_ref().clone();
Ok(data)
}
#[tokio::test]
async fn test_v1_fallback_full_reconstruction() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 3)), (2, (0, 2))], TEST_CHUNK_SIZE)
.await
.unwrap();
// Disable V2 so the remote client falls back to V1 + conversion.
server.disable_v2_reconstruction(404);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, None, &config)
.await
.unwrap();
assert_eq!(result, file_contents.data.as_ref());
}
#[tokio::test]
async fn test_v1_fallback_partial_range() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 5)), (2, (0, 3))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let file_len = file_contents.data.len() as u64;
let range = FileRange::new(file_len / 4, file_len * 3 / 4);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, Some(range), &config)
.await
.unwrap();
assert_eq!(result, &file_contents.data[range.start as usize..range.end as usize]);
}
#[tokio::test]
async fn test_v1_fallback_non_contiguous_chunks() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 2)), (1, (4, 6))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, None, &config)
.await
.unwrap();
assert_eq!(result, file_contents.data.as_ref());
}
#[tokio::test]
async fn test_v1_fallback_multiple_xorbs() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 2)), (2, (0, 3)), (3, (0, 2)), (1, (2, 4))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, None, &config)
.await
.unwrap();
assert_eq!(result, file_contents.data.as_ref());
}
/// V1 fallback with three disjoint ranges from the same xorb.
#[tokio::test]
async fn test_v1_fallback_triple_disjoint_ranges() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 2)), (1, (4, 6)), (1, (8, 10))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, None, &config)
.await
.unwrap();
assert_eq!(result, file_contents.data.as_ref());
}
// ==================== Max Ranges Tests ====================
//
// These tests use LocalTestServer with max_ranges_per_fetch=2 to verify that
// multi-range fetch splitting works correctly through the full HTTP path.
/// Helper to set up a server with max_ranges_per_fetch and reconstruct.
async fn reconstruct_via_server_with_max_ranges(
term_spec: &[(u64, (u64, u64))],
max_ranges: usize,
byte_range: Option<FileRange>,
) -> (Vec<u8>, RandomFileContents) {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(term_spec, TEST_CHUNK_SIZE)
.await
.unwrap();
server.set_max_ranges_per_fetch(max_ranges);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, byte_range, &config)
.await
.unwrap();
(result, file_contents)
}
#[tokio::test]
async fn test_max_ranges_simple() {
let (result, file_contents) =
reconstruct_via_server_with_max_ranges(&[(1, (0, 3)), (2, (0, 2))], 2, None).await;
assert_eq!(result, file_contents.data.as_ref());
}
/// A single xorb with two disjoint ranges, split at max_ranges=1.
/// Each range becomes its own fetch entry.
#[tokio::test]
async fn test_max_ranges_1_disjoint() {
let (result, file_contents) =
reconstruct_via_server_with_max_ranges(&[(1, (0, 2)), (1, (4, 6))], 1, None).await;
assert_eq!(result, file_contents.data.as_ref());
}
/// Three disjoint ranges from the same xorb with max_ranges=2.
/// First two ranges are grouped, third gets its own fetch entry.
#[tokio::test]
async fn test_max_ranges_2_triple_disjoint() {
let (result, file_contents) =
reconstruct_via_server_with_max_ranges(&[(1, (0, 2)), (1, (4, 6)), (1, (8, 10))], 2, None).await;
assert_eq!(result, file_contents.data.as_ref());
}
/// Multiple xorbs, each with disjoint ranges, with max_ranges=2.
/// Tests that splitting is applied per-xorb correctly.
#[tokio::test]
async fn test_max_ranges_2_multi_xorb_disjoint() {
let term_spec = &[
(1, (0, 2)),
(2, (0, 2)),
(1, (4, 6)),
(2, (4, 6)),
(1, (8, 10)),
(2, (8, 10)),
];
let (result, file_contents) = reconstruct_via_server_with_max_ranges(term_spec, 2, None).await;
assert_eq!(result, file_contents.data.as_ref());
}
/// Complex interleaved pattern with max_ranges=2 and a partial byte range.
#[tokio::test]
async fn test_max_ranges_2_partial_range() {
let term_spec = &[
(1, (0, 3)),
(2, (0, 2)),
(1, (3, 5)),
(3, (1, 4)),
(2, (4, 6)),
(1, (0, 2)),
];
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(term_spec, TEST_CHUNK_SIZE)
.await
.unwrap();
server.set_max_ranges_per_fetch(2);
let file_len = file_contents.data.len() as u64;
let range = FileRange::new(file_len / 4, file_len * 3 / 4);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, Some(range), &config)
.await
.unwrap();
assert_eq!(result, &file_contents.data[range.start as usize..range.end as usize]);
}
// ==================== Multi-Disjoint Range Tests (LocalClient) ====================
//
// These tests exercise complex disjoint range patterns through the LocalClient path
@@ -1659,228 +1437,453 @@ mod tests {
// ==================== Cancellation Flag Tests ====================
#[tokio::test]
async fn test_cancellation_token_before_start() {
let (client, file_contents) = setup_test_file(&[(1, (0, 3))]).await;
let config = test_config();
mod cancellation_tests {
use tokio_util::sync::CancellationToken;
let token = CancellationToken::new();
token.cancel();
let buffer = Arc::new(std::sync::Mutex::new(Cursor::new(Vec::new())));
let writer = StaticCursorWriter(buffer.clone());
use super::*;
let bytes_written = FileReconstructor::new(&(client.clone() as Arc<dyn Client>), file_contents.file_hash)
.with_config(&config)
.with_cancellation_token(token)
.reconstruct_to_writer(writer)
.await
.unwrap();
#[tokio::test]
async fn test_cancellation_token_before_start() {
let (client, file_contents) = setup_test_file(&[(1, (0, 3))]).await;
let config = test_config();
assert_eq!(bytes_written, 0);
}
let token = CancellationToken::new();
token.cancel();
let buffer = Arc::new(std::sync::Mutex::new(Cursor::new(Vec::new())));
let writer = StaticCursorWriter(buffer.clone());
/// A writer that cancels a token after a certain number of writes,
/// used to deterministically test mid-reconstruction cancellation.
struct CancellingWriter {
buffer: Arc<std::sync::Mutex<Vec<u8>>>,
cancel_token: CancellationToken,
write_count: AtomicUsize,
cancel_after_writes: usize,
}
let bytes_written = FileReconstructor::new(&(client.clone() as Arc<dyn Client>), file_contents.file_hash)
.with_config(&config)
.with_cancellation_token(token)
.reconstruct_to_writer(writer)
.await
.unwrap();
impl Write for CancellingWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let n = self.buffer.lock().unwrap().write(buf)?;
let count = self.write_count.fetch_add(1, Ordering::Relaxed) + 1;
if count >= self.cancel_after_writes {
self.cancel_token.cancel();
assert_eq!(bytes_written, 0);
}
/// A writer that cancels a token after a certain number of writes,
/// used to deterministically test mid-reconstruction cancellation.
struct CancellingWriter {
buffer: Arc<std::sync::Mutex<Vec<u8>>>,
cancel_token: CancellationToken,
write_count: AtomicUsize,
cancel_after_writes: usize,
}
impl Write for CancellingWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let n = self.buffer.lock().unwrap().write(buf)?;
let count = self.write_count.fetch_add(1, Ordering::Relaxed) + 1;
if count >= self.cancel_after_writes {
self.cancel_token.cancel();
}
Ok(n)
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
Ok(n)
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
#[tokio::test]
async fn test_cancellation_token_during_reconstruction() {
let term_spec: Vec<(u64, (u64, u64))> = (1..=10).map(|i| (i, (0, 5))).collect();
let (client, file_contents) = setup_test_file(&term_spec).await;
let config = test_config();
let token = CancellationToken::new();
let buffer = Arc::new(std::sync::Mutex::new(Vec::new()));
let writer = CancellingWriter {
buffer: buffer.clone(),
cancel_token: token.clone(),
write_count: AtomicUsize::new(0),
cancel_after_writes: 1,
};
// Use a tiny semaphore to force sequential term processing.
let tiny_semaphore = AdjustableSemaphore::new(1, (1, 1));
let bytes_written = FileReconstructor::new(&(client.clone() as Arc<dyn Client>), file_contents.file_hash)
.with_config(&config)
.with_cancellation_token(token)
.with_buffer_semaphore(tiny_semaphore)
.reconstruct_to_writer(writer)
.await
.unwrap();
// Verify cancellation returned Ok(0) and only partial data was written.
assert_eq!(bytes_written, 0);
let written = buffer.lock().unwrap().len();
assert!(written < file_contents.data.len());
}
#[tokio::test]
async fn test_cancellation_token_not_set_completes_normally() {
let (client, file_contents) = setup_test_file(&[(1, (0, 3)), (2, (0, 2))]).await;
let config = test_config();
let token = CancellationToken::new();
let buffer = Arc::new(std::sync::Mutex::new(Cursor::new(Vec::new())));
let writer = StaticCursorWriter(buffer.clone());
let bytes_written = FileReconstructor::new(&(client.clone() as Arc<dyn Client>), file_contents.file_hash)
.with_config(&config)
.with_cancellation_token(token)
.reconstruct_to_writer(writer)
.await
.unwrap();
assert_eq!(bytes_written, file_contents.data.len() as u64);
assert_eq!(buffer.lock().unwrap().get_ref().clone(), file_contents.data);
}
}
#[tokio::test]
async fn test_cancellation_token_during_reconstruction() {
let term_spec: Vec<(u64, (u64, u64))> = (1..=10).map(|i| (i, (0, 5))).collect();
let (client, file_contents) = setup_test_file(&term_spec).await;
let config = test_config();
// ==================== Multirange Fetching Tests (LocalClient) ====================
let token = CancellationToken::new();
let buffer = Arc::new(std::sync::Mutex::new(Vec::new()));
mod multirange_tests {
use super::*;
let writer = CancellingWriter {
buffer: buffer.clone(),
cancel_token: token.clone(),
write_count: AtomicUsize::new(0),
cancel_after_writes: 1,
};
fn with_multirange_config(enable: bool) -> Arc<XetRuntime> {
let mut config = xet_runtime::config::XetConfig::new();
config.client.enable_multirange_fetching = enable;
XetRuntime::new_with_config(config).unwrap()
}
// Use a tiny semaphore to force sequential term processing.
let tiny_semaphore = AdjustableSemaphore::new(1, (1, 1));
/// Exercises multiple disjoint-range scenarios through LocalClient with both
/// enable_multirange_fetching=true and =false.
#[test]
fn test_multirange_local_client() {
for enable in [false, true] {
let rt = with_multirange_config(enable);
rt.bridge_sync(async move {
let scenarios: Vec<Vec<(u64, (u64, u64))>> = vec![
vec![(1, (0, 2)), (1, (4, 6)), (1, (8, 10))],
vec![
(1, (0, 2)),
(2, (0, 2)),
(1, (4, 6)),
(2, (4, 6)),
(1, (8, 10)),
(2, (8, 10)),
],
vec![
(1, (0, 2)),
(2, (0, 3)),
(3, (2, 5)),
(1, (5, 8)),
(2, (6, 8)),
(3, (0, 2)),
],
];
let config = test_config();
for term_spec in &scenarios {
let (client, fc) = setup_test_file(term_spec).await;
reconstruct_and_verify_full(&client, &fc, config.clone()).await;
let bytes_written = FileReconstructor::new(&(client.clone() as Arc<dyn Client>), file_contents.file_hash)
.with_config(&config)
.with_cancellation_token(token)
.with_buffer_semaphore(tiny_semaphore)
.reconstruct_to_writer(writer)
.await
.unwrap();
let file_len = fc.data.len() as u64;
let range = FileRange::new(file_len / 4, file_len * 3 / 4);
reconstruct_and_verify_range(&client, &fc, range, config.clone()).await;
}
})
.unwrap();
}
}
// Verify cancellation returned Ok(0) and only partial data was written.
assert_eq!(bytes_written, 0);
let written = buffer.lock().unwrap().len();
assert!(written < file_contents.data.len());
/// LocalClient with max_ranges_per_fetch constraint, both enable settings.
#[test]
fn test_multirange_max_ranges() {
for enable in [false, true] {
let rt = with_multirange_config(enable);
rt.bridge_sync(async {
let client = LocalClient::temporary().await.unwrap();
client.set_max_ranges_per_fetch(2);
let term_spec = &[(1, (0, 2)), (1, (4, 6)), (1, (8, 10)), (1, (12, 14))];
let fc = client.upload_random_file(term_spec, TEST_CHUNK_SIZE).await.unwrap();
let config = test_config();
let result = reconstruct_to_vec(&client, fc.file_hash, None, &config, None).await.unwrap();
assert_eq!(result, fc.data.as_ref());
})
.unwrap();
}
}
}
#[tokio::test]
async fn test_cancellation_token_not_set_completes_normally() {
let (client, file_contents) = setup_test_file(&[(1, (0, 3)), (2, (0, 2))]).await;
let config = test_config();
// ==================== Server-dependent tests (require simulation feature) ====================
#[cfg(feature = "simulation")]
mod server_tests {
use super::*;
let token = CancellationToken::new();
let buffer = Arc::new(std::sync::Mutex::new(Cursor::new(Vec::new())));
let writer = StaticCursorWriter(buffer.clone());
// ==================== V1 Fallback Tests ====================
//
// These tests use LocalTestServer with V2 disabled to verify that
// reconstruction works correctly when the client falls back from V2 to V1.
let bytes_written = FileReconstructor::new(&(client.clone() as Arc<dyn Client>), file_contents.file_hash)
.with_config(&config)
.with_cancellation_token(token)
.reconstruct_to_writer(writer)
.await
.unwrap();
/// Helper to reconstruct through a LocalTestServer (RemoteClient HTTP path).
async fn reconstruct_via_server(
server: &xet_client::cas_client::LocalTestServer,
file_hash: MerkleHash,
byte_range: Option<FileRange>,
config: &ReconstructionConfig,
) -> Result<Vec<u8>> {
let buffer = Arc::new(std::sync::Mutex::new(Cursor::new(Vec::new())));
let writer = StaticCursorWriter(buffer.clone());
assert_eq!(bytes_written, file_contents.data.len() as u64);
assert_eq!(buffer.lock().unwrap().get_ref().clone(), file_contents.data);
}
let client: Arc<dyn Client> = server.remote_client().clone();
let mut reconstructor = FileReconstructor::new(&client, file_hash).with_config(config);
// ==================== Multirange Fetching Tests ====================
//
// These tests verify that reconstruction works correctly with both values
// of `enable_multirange_fetching`. When true, V2 multi-range fetch entries
// are used as-is (multirange HTTP requests). When false (default), each
// range is split into its own XorbBlock and fetched via a separate
// single-range request in parallel.
//
// Uses XetRuntime::new_with_config() to override the config per-test,
// following the pattern from test_dynamic_buffer_scaling_noop_increment_preserves_total_permits.
if let Some(range) = byte_range {
reconstructor = reconstructor.with_byte_range(range);
}
fn with_multirange_config(enable: bool) -> Arc<XetRuntime> {
let mut config = xet_runtime::config::XetConfig::new();
config.client.enable_multirange_fetching = enable;
XetRuntime::new_with_config(config).unwrap()
}
reconstructor.reconstruct_to_writer(writer).await?;
/// Exercises multiple disjoint-range scenarios through LocalClient with both
/// enable_multirange_fetching=true and =false.
#[test]
fn test_multirange_local_client() {
for enable in [false, true] {
let rt = with_multirange_config(enable);
rt.bridge_sync(async move {
let scenarios: Vec<Vec<(u64, (u64, u64))>> = vec![
vec![(1, (0, 2)), (1, (4, 6)), (1, (8, 10))],
vec![
(1, (0, 2)),
(2, (0, 2)),
(1, (4, 6)),
(2, (4, 6)),
(1, (8, 10)),
(2, (8, 10)),
],
vec![
(1, (0, 2)),
(2, (0, 3)),
(3, (2, 5)),
(1, (5, 8)),
(2, (6, 8)),
(3, (0, 2)),
],
];
let config = test_config();
for term_spec in &scenarios {
let (client, fc) = setup_test_file(term_spec).await;
reconstruct_and_verify_full(&client, &fc, config.clone()).await;
let data = buffer.lock().unwrap().get_ref().clone();
Ok(data)
}
#[tokio::test]
async fn test_v1_fallback_full_reconstruction() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 3)), (2, (0, 2))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, None, &config)
.await
.unwrap();
assert_eq!(result, file_contents.data.as_ref());
}
#[tokio::test]
async fn test_v1_fallback_partial_range() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 5)), (2, (0, 3))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let file_len = file_contents.data.len() as u64;
let range = FileRange::new(file_len / 4, file_len * 3 / 4);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, Some(range), &config)
.await
.unwrap();
assert_eq!(result, &file_contents.data[range.start as usize..range.end as usize]);
}
#[tokio::test]
async fn test_v1_fallback_non_contiguous_chunks() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 2)), (1, (4, 6))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, None, &config)
.await
.unwrap();
assert_eq!(result, file_contents.data.as_ref());
}
#[tokio::test]
async fn test_v1_fallback_multiple_xorbs() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 2)), (2, (0, 3)), (3, (0, 2)), (1, (2, 4))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, None, &config)
.await
.unwrap();
assert_eq!(result, file_contents.data.as_ref());
}
/// V1 fallback with three disjoint ranges from the same xorb.
#[tokio::test]
async fn test_v1_fallback_triple_disjoint_ranges() {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(&[(1, (0, 2)), (1, (4, 6)), (1, (8, 10))], TEST_CHUNK_SIZE)
.await
.unwrap();
server.disable_v2_reconstruction(404);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, None, &config)
.await
.unwrap();
assert_eq!(result, file_contents.data.as_ref());
}
// ==================== Max Ranges Tests (via server) ====================
/// Helper to set up a server with max_ranges_per_fetch and reconstruct.
async fn reconstruct_via_server_with_max_ranges(
term_spec: &[(u64, (u64, u64))],
max_ranges: usize,
byte_range: Option<FileRange>,
) -> (Vec<u8>, RandomFileContents) {
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(term_spec, TEST_CHUNK_SIZE)
.await
.unwrap();
server.set_max_ranges_per_fetch(max_ranges);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, byte_range, &config)
.await
.unwrap();
(result, file_contents)
}
#[tokio::test]
async fn test_max_ranges_simple() {
let (result, file_contents) =
reconstruct_via_server_with_max_ranges(&[(1, (0, 3)), (2, (0, 2))], 2, None).await;
assert_eq!(result, file_contents.data.as_ref());
}
#[tokio::test]
async fn test_max_ranges_1_disjoint() {
let (result, file_contents) =
reconstruct_via_server_with_max_ranges(&[(1, (0, 2)), (1, (4, 6))], 1, None).await;
assert_eq!(result, file_contents.data.as_ref());
}
#[tokio::test]
async fn test_max_ranges_2_triple_disjoint() {
let (result, file_contents) =
reconstruct_via_server_with_max_ranges(&[(1, (0, 2)), (1, (4, 6)), (1, (8, 10))], 2, None).await;
assert_eq!(result, file_contents.data.as_ref());
}
#[tokio::test]
async fn test_max_ranges_2_multi_xorb_disjoint() {
let term_spec = &[
(1, (0, 2)),
(2, (0, 2)),
(1, (4, 6)),
(2, (4, 6)),
(1, (8, 10)),
(2, (8, 10)),
];
let (result, file_contents) = reconstruct_via_server_with_max_ranges(term_spec, 2, None).await;
assert_eq!(result, file_contents.data.as_ref());
}
#[tokio::test]
async fn test_max_ranges_2_partial_range() {
let term_spec = &[
(1, (0, 3)),
(2, (0, 2)),
(1, (3, 5)),
(3, (1, 4)),
(2, (4, 6)),
(1, (0, 2)),
];
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let file_contents = server
.remote_client()
.upload_random_file(term_spec, TEST_CHUNK_SIZE)
.await
.unwrap();
server.set_max_ranges_per_fetch(2);
let file_len = file_contents.data.len() as u64;
let range = FileRange::new(file_len / 4, file_len * 3 / 4);
let config = test_config();
let result = reconstruct_via_server(&server, file_contents.file_hash, Some(range), &config)
.await
.unwrap();
assert_eq!(result, &file_contents.data[range.start as usize..range.end as usize]);
}
// ==================== Multirange via Server ====================
fn with_multirange_config(enable: bool) -> Arc<XetRuntime> {
let mut config = xet_runtime::config::XetConfig::new();
config.client.enable_multirange_fetching = enable;
XetRuntime::new_with_config(config).unwrap()
}
/// Exercises HTTP server path with full, max-ranges-split, and partial-range
/// reconstruction, both enable_multirange_fetching values.
#[test]
fn test_multirange_via_server() {
for enable in [false, true] {
let rt = with_multirange_config(enable);
rt.bridge_sync(async {
let config = test_config();
// Full reconstruction with disjoint ranges
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let fc = server
.remote_client()
.upload_random_file(&[(1, (0, 2)), (1, (4, 6)), (1, (8, 10))], TEST_CHUNK_SIZE)
.await
.unwrap();
let result = reconstruct_via_server(&server, fc.file_hash, None, &config).await.unwrap();
assert_eq!(result, fc.data.as_ref());
// Multi-xorb with max_ranges_per_fetch=2
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let fc = server
.remote_client()
.upload_random_file(
&[(1, (0, 2)), (2, (0, 2)), (1, (4, 6)), (2, (4, 6)), (1, (8, 10))],
TEST_CHUNK_SIZE,
)
.await
.unwrap();
server.set_max_ranges_per_fetch(2);
let result = reconstruct_via_server(&server, fc.file_hash, None, &config).await.unwrap();
assert_eq!(result, fc.data.as_ref());
// Partial byte range
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let fc = server
.remote_client()
.upload_random_file(&[(1, (0, 3)), (2, (0, 2)), (1, (3, 5)), (2, (4, 6))], TEST_CHUNK_SIZE)
.await
.unwrap();
let file_len = fc.data.len() as u64;
let range = FileRange::new(file_len / 4, file_len * 3 / 4);
reconstruct_and_verify_range(&client, &fc, range, config.clone()).await;
}
})
.unwrap();
let result = reconstruct_via_server(&server, fc.file_hash, Some(range), &config)
.await
.unwrap();
assert_eq!(result, &fc.data[range.start as usize..range.end as usize]);
})
.unwrap();
}
}
}
/// LocalClient with max_ranges_per_fetch constraint, both enable settings.
#[test]
fn test_multirange_max_ranges() {
for enable in [false, true] {
let rt = with_multirange_config(enable);
rt.bridge_sync(async {
let client = LocalClient::temporary().await.unwrap();
client.set_max_ranges_per_fetch(2);
let term_spec = &[(1, (0, 2)), (1, (4, 6)), (1, (8, 10)), (1, (12, 14))];
let fc = client.upload_random_file(term_spec, TEST_CHUNK_SIZE).await.unwrap();
let config = test_config();
let result = reconstruct_to_vec(&client, fc.file_hash, None, &config, None).await.unwrap();
assert_eq!(result, fc.data.as_ref());
})
.unwrap();
}
}
/// Exercises HTTP server path with full, max-ranges-split, and partial-range
/// reconstruction, both enable_multirange_fetching values.
#[test]
fn test_multirange_via_server() {
for enable in [false, true] {
let rt = with_multirange_config(enable);
rt.bridge_sync(async {
let config = test_config();
// Full reconstruction with disjoint ranges
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let fc = server
.remote_client()
.upload_random_file(&[(1, (0, 2)), (1, (4, 6)), (1, (8, 10))], TEST_CHUNK_SIZE)
.await
.unwrap();
let result = reconstruct_via_server(&server, fc.file_hash, None, &config).await.unwrap();
assert_eq!(result, fc.data.as_ref());
// Multi-xorb with max_ranges_per_fetch=2
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let fc = server
.remote_client()
.upload_random_file(
&[(1, (0, 2)), (2, (0, 2)), (1, (4, 6)), (2, (4, 6)), (1, (8, 10))],
TEST_CHUNK_SIZE,
)
.await
.unwrap();
server.set_max_ranges_per_fetch(2);
let result = reconstruct_via_server(&server, fc.file_hash, None, &config).await.unwrap();
assert_eq!(result, fc.data.as_ref());
// Partial byte range
let server = xet_client::cas_client::LocalTestServerBuilder::new().start().await;
let fc = server
.remote_client()
.upload_random_file(&[(1, (0, 3)), (2, (0, 2)), (1, (3, 5)), (2, (4, 6))], TEST_CHUNK_SIZE)
.await
.unwrap();
let file_len = fc.data.len() as u64;
let range = FileRange::new(file_len / 4, file_len * 3 / 4);
let result = reconstruct_via_server(&server, fc.file_hash, Some(range), &config)
.await
.unwrap();
assert_eq!(result, &fc.data[range.start as usize..range.end as usize]);
})
.unwrap();
}
}
} // mod server_tests
}

10
xet_data/src/processing/remote_client_interface.rs
View File

@@ -19,14 +19,12 @@ pub(crate) async fn create_remote_client(
Ok(xet_client::cas_client::LocalClient::new(xorb_path).await?)
}
#[cfg(target_family = "wasm")]
unimplemented!("Local file system access is not supported in WASM builds")
} else if session.is_memory() {
#[cfg(not(target_family = "wasm"))]
{
Ok(xet_client::cas_client::MemoryClient::new())
let _ = local_path;
unimplemented!("Local file system access is not available in WASM")
}
#[cfg(target_family = "wasm")]
unimplemented!("In-memory client is not supported in WASM builds")
} else if session.is_memory() {
Ok(xet_client::cas_client::MemoryClient::new())
} else {
Ok(RemoteClient::new(
&session.endpoint,

80
xet_data/src/processing/test_utils.rs
View File

@@ -6,7 +6,9 @@ use std::sync::Arc;
use itertools::multizip;
use rand::prelude::*;
use tempfile::TempDir;
use xet_client::cas_client::{Client, LocalClient, LocalTestServer, LocalTestServerBuilder};
use xet_client::cas_client::{Client, LocalClient};
#[cfg(feature = "simulation")]
use xet_client::cas_client::{LocalTestServer, LocalTestServerBuilder};
use super::configurations::TranslatorConfig;
use super::data_client::clean_file;
@@ -24,8 +26,11 @@ use super::{FileDownloadSession, FileUploadSession, XetFileInfo};
#[derive(Debug, Clone, Copy)]
pub enum HydrationMode {
DirectClient,
#[cfg(feature = "simulation")]
ServerV2,
#[cfg(feature = "simulation")]
ServerV1Fallback,
#[cfg(feature = "simulation")]
ServerMaxRanges2,
}
@@ -33,14 +38,21 @@ impl HydrationMode {
pub fn all() -> &'static [HydrationMode] {
&[
HydrationMode::DirectClient,
#[cfg(feature = "simulation")]
HydrationMode::ServerV2,
#[cfg(feature = "simulation")]
HydrationMode::ServerV1Fallback,
#[cfg(feature = "simulation")]
HydrationMode::ServerMaxRanges2,
]
}
pub fn uses_server(&self) -> bool {
!matches!(self, HydrationMode::DirectClient)
match self {
HydrationMode::DirectClient => false,
#[cfg(feature = "simulation")]
_ => true,
}
}
}
@@ -48,8 +60,11 @@ impl std::fmt::Display for HydrationMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
HydrationMode::DirectClient => write!(f, "direct_client"),
#[cfg(feature = "simulation")]
HydrationMode::ServerV2 => write!(f, "server_v2"),
#[cfg(feature = "simulation")]
HydrationMode::ServerV1Fallback => write!(f, "server_v1_fallback"),
#[cfg(feature = "simulation")]
HydrationMode::ServerMaxRanges2 => write!(f, "server_max_ranges_2"),
}
}
@@ -174,6 +189,7 @@ pub struct HydrateDehydrateTest {
pub dest_dir: PathBuf,
use_test_server: bool,
/// Kept alive so the test server stays running for the duration of the test.
#[cfg(feature = "simulation")]
test_server: Option<LocalTestServer>,
}
@@ -210,6 +226,7 @@ impl HydrateDehydrateTest {
dest_dir,
_temp_dir,
use_test_server,
#[cfg(feature = "simulation")]
test_server: None,
}
}
@@ -224,13 +241,16 @@ impl HydrateDehydrateTest {
pub async fn apply_hydration_mode(&mut self, mode: HydrationMode) {
match mode {
HydrationMode::DirectClient => {},
#[cfg(feature = "simulation")]
HydrationMode::ServerV2 => {
self.ensure_server_created().await;
},
#[cfg(feature = "simulation")]
HydrationMode::ServerV1Fallback => {
self.ensure_server_created().await;
self.test_server.as_ref().unwrap().client().disable_v2_reconstruction(404);
},
#[cfg(feature = "simulation")]
HydrationMode::ServerMaxRanges2 => {
self.ensure_server_created().await;
self.test_server.as_ref().unwrap().client().set_max_ranges_per_fetch(2);
@@ -240,6 +260,7 @@ impl HydrateDehydrateTest {
/// Ensures the test server is running, creating it if necessary.
/// Call this before configuring the server (e.g., disabling V2 or setting max ranges).
#[cfg(feature = "simulation")]
pub async fn ensure_server_created(&mut self) {
if self.use_test_server && self.test_server.is_none() {
let local_client = LocalClient::new(self.cas_dir.join("xet/xorbs")).await.unwrap();
@@ -248,6 +269,7 @@ impl HydrateDehydrateTest {
}
/// Returns a reference to the test server, if one has been created.
#[cfg(feature = "simulation")]
pub fn test_server(&self) -> Option<&LocalTestServer> {
self.test_server.as_ref()
}
@@ -255,11 +277,18 @@ impl HydrateDehydrateTest {
/// Lazily initializes the test server (if needed) and returns a CAS client.
async fn get_or_create_client(&mut self) -> Arc<dyn Client> {
if self.use_test_server {
if self.test_server.is_none() {
let local_client = LocalClient::new(self.cas_dir.join("xet/xorbs")).await.unwrap();
self.test_server = Some(LocalTestServerBuilder::new().with_client(local_client).start().await);
#[cfg(feature = "simulation")]
{
if self.test_server.is_none() {
let local_client = LocalClient::new(self.cas_dir.join("xet/xorbs")).await.unwrap();
self.test_server = Some(LocalTestServerBuilder::new().with_client(local_client).start().await);
}
self.test_server.as_ref().unwrap().remote_client().clone() as Arc<dyn Client>
}
#[cfg(not(feature = "simulation"))]
{
panic!("test server requires the 'simulation' feature");
}
self.test_server.as_ref().unwrap().remote_client().clone() as Arc<dyn Client>
} else {
LocalClient::new(self.cas_dir.join("xet/xorbs")).await.unwrap() as Arc<dyn Client>
}
@@ -393,3 +422,42 @@ impl HydrateDehydrateTest {
verify_directories_match(&self.src_dir, &self.dest_dir);
}
}
/// Provides a test environment with a config suitable for `FileUploadSession` / `FileDownloadSession`.
///
/// When the `simulation` feature is enabled the environment spins up a `LocalTestServer` and
/// returns a server-backed config; otherwise it falls back to `LocalClient` via `local_config`.
pub struct TestEnvironment {
_temp_dir: TempDir,
pub base_dir: PathBuf,
pub config: Arc<super::configurations::TranslatorConfig>,
#[cfg(feature = "simulation")]
_server: Option<LocalTestServer>,
}
impl TestEnvironment {
pub async fn new() -> Self {
let temp_dir = TempDir::new().unwrap();
let base_dir = temp_dir.path().to_path_buf();
#[cfg(feature = "simulation")]
let (config, server) = {
let server = LocalTestServerBuilder::new().start().await;
let config = Arc::new(
super::configurations::TranslatorConfig::test_server_config(server.http_endpoint(), &base_dir).unwrap(),
);
(config, Some(server))
};
#[cfg(not(feature = "simulation"))]
let config = Arc::new(super::configurations::TranslatorConfig::local_config(&base_dir).unwrap());
Self {
_temp_dir: temp_dir,
base_dir,
config,
#[cfg(feature = "simulation")]
_server: server,
}
}
}

35
xet_data/tests/test_full_file_download.rs
View File

@@ -10,9 +10,7 @@ mod tests {
use std::fs;
use std::sync::Arc;
use tempfile::TempDir;
use xet_client::cas_client::LocalTestServerBuilder;
use xet_data::processing::configurations::TranslatorConfig;
use xet_data::processing::test_utils::TestEnvironment;
use xet_data::processing::{FileDownloadSession, FileUploadSession, Sha256Policy, XetFileInfo};
async fn upload_bytes(upload_session: &Arc<FileUploadSession>, name: &str, data: &[u8]) -> XetFileInfo {
@@ -24,13 +22,11 @@ mod tests {
xfi
}
/// Uploads files of various sizes through a LocalTestServer and downloads them
/// without an explicit byte range, verifying no 416 errors occur.
/// Uploads files of various sizes and downloads them without an explicit byte
/// range, verifying no 416 errors occur.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_full_file_download_no_416() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let env = TestEnvironment::new().await;
let test_cases: &[(&str, &[u8])] = &[
("one_byte", &[0x42]),
@@ -39,15 +35,22 @@ mod tests {
("larger", &vec![0xCD; 64 * 1024]),
];
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
// Upload all files first in a single session.
let upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
let xfis: Vec<_> = {
let mut v = Vec::new();
for (name, data) in test_cases {
v.push((*name, *data, upload_bytes(&upload_session, name, data).await));
}
v
};
upload_session.finalize().await.unwrap();
for (name, data) in test_cases {
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, name, data).await;
upload_session.finalize().await.unwrap();
let out_path = base_dir.path().join(format!("out_{name}"));
let (_id, n_bytes) = download_session.download_file(&xfi, &out_path).await.unwrap();
// Now download and verify each file.
let download_session = FileDownloadSession::new(env.config.clone()).await.unwrap();
for (name, data, xfi) in &xfis {
let out_path = env.base_dir.join(format!("out_{name}"));
let (_id, n_bytes) = download_session.download_file(xfi, &out_path).await.unwrap();
assert_eq!(n_bytes, data.len() as u64, "size mismatch for {name}");
assert_eq!(fs::read(&out_path).unwrap(), *data, "content mismatch for {name}");

24
xet_data/tests/test_range_downloads.rs
View File

@@ -1,4 +1,4 @@
//! Integration tests for range-based downloads using a LocalTestServer.
//! Integration tests for range-based downloads.
//!
//! Exercises all range variants (`start..end`, `start..`, `..end`, `..`) across
//! the three download paths: file, writer, and streaming.
@@ -8,9 +8,7 @@ mod tests {
use std::fs;
use std::sync::Arc;
use tempfile::TempDir;
use xet_client::cas_client::{LocalTestServer, LocalTestServerBuilder};
use xet_data::processing::configurations::TranslatorConfig;
use xet_data::processing::test_utils::TestEnvironment;
use xet_data::processing::{FileDownloadSession, FileUploadSession, Sha256Policy, XetFileInfo};
async fn upload_bytes(upload_session: &Arc<FileUploadSession>, name: &str, data: &[u8]) -> XetFileInfo {
@@ -23,27 +21,23 @@ mod tests {
}
struct TestHarness {
_server: LocalTestServer,
_base_dir: TempDir,
_env: TestEnvironment,
session: Arc<FileDownloadSession>,
xfi: XetFileInfo,
data: Vec<u8>,
}
async fn setup() -> TestHarness {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let env = TestEnvironment::new().await;
let data: Vec<u8> = (0..=255u8).cycle().take(8192).collect();
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "range_test", &data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config).await.unwrap();
let download_session = FileDownloadSession::new(env.config.clone()).await.unwrap();
TestHarness {
_server: server,
_base_dir: base_dir,
_env: env,
session: download_session,
xfi,
data,
@@ -160,7 +154,7 @@ mod tests {
let h = setup().await;
let xfi_no_size = XetFileInfo::new_hash_only(h.xfi.hash().to_string());
let out_path = h._base_dir.path().join("unknown_size.bin");
let out_path = h._env.base_dir.join("unknown_size.bin");
let (_id, n_bytes) = h.session.download_file(&xfi_no_size, &out_path).await.unwrap();
assert_eq!(n_bytes, h.data.len() as u64);
@@ -186,7 +180,7 @@ mod tests {
let h = setup().await;
let wrong_size = XetFileInfo::new(h.xfi.hash().to_string(), 42);
let out_path = h._base_dir.path().join("mismatch.bin");
let out_path = h._env.base_dir.join("mismatch.bin");
let err = h.session.download_file(&wrong_size, &out_path).await.unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("mismatch"), "Expected size mismatch error, got: {msg}");

32
xet_data/tests/test_session_resume.rs
View File

@@ -1,11 +1,7 @@
use std::time::Duration;
// Run tests that determine deduplication, especially across different test subjects.
use tempfile::TempDir;
use xet_client::cas_client::LocalTestServerBuilder;
use xet_data::deduplication::constants::{MAX_XORB_BYTES, MAX_XORB_CHUNKS, TARGET_CHUNK_SIZE};
use xet_data::processing::configurations::TranslatorConfig;
use xet_data::processing::{FileUploadSession, Sha256Policy};
use xet_data::deduplication::constants::{MAX_XORB_CHUNKS, TARGET_CHUNK_SIZE};
use xet_runtime::{test_set_config, test_set_constants};
// Runs this test suite with small chunks and xorbs so that we can make sure that all the different edge
@@ -32,14 +28,14 @@ test_set_config! {
// Test the deduplication framework.
#[cfg(test)]
mod tests {
use std::sync::Arc;
use more_asserts::*;
use rand::prelude::*;
use xet_data::deduplication::constants::MAX_CHUNK_SIZE;
use xet_data::processing::test_utils::{HydrateDehydrateTest, create_random_file, create_random_files};
use xet_data::deduplication::constants::{MAX_CHUNK_SIZE, MAX_XORB_BYTES};
use xet_data::processing::test_utils::{
HydrateDehydrateTest, TestEnvironment, create_random_file, create_random_files,
};
use xet_data::processing::{FileUploadSession, Sha256Policy};
use super::*;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_simple_resume() {
// Ensure the deduplication numbers are approximately accurate.
@@ -53,12 +49,10 @@ mod tests {
let mut rng = StdRng::seed_from_u64(0);
rng.fill(&mut data[..]);
let server = LocalTestServerBuilder::new().start().await;
let shard_base = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), shard_base.path()).unwrap());
let env = TestEnvironment::new().await;
{
let file_upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let file_upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
// Feed it half the data, and checkpoint.
let (_id, mut cleaner) = file_upload_session
@@ -79,7 +73,7 @@ mod tests {
// Now try again to test the resume.
{
let file_upload_session = FileUploadSession::new(config).await.unwrap();
let file_upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
// Feed it half the data, and checkpoint.
let (_id, mut cleaner) = file_upload_session
@@ -112,15 +106,13 @@ mod tests {
let mut rng = StdRng::seed_from_u64(0);
rng.fill(&mut data[..]);
let server = LocalTestServerBuilder::new().start().await;
let shard_base = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), shard_base.path()).unwrap());
let env = TestEnvironment::new().await;
let max_deviance = (*MAX_XORB_BYTES + *MAX_CHUNK_SIZE) as u64;
let mut prev_rn = 0;
for rn in resume_n {
let file_upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let file_upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
// Feed it half the data, and checkpoint.
let (_id, mut cleaner) = file_upload_session
@@ -144,7 +136,7 @@ mod tests {
// Now try again to test the resume.
{
let file_upload_session = FileUploadSession::new(config).await.unwrap();
let file_upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
// Feed it half the data, and checkpoint.
let (_id, mut cleaner) = file_upload_session

250
xet_data/tests/test_unordered_download.rs
View File

@@ -4,9 +4,7 @@ mod tests {
use std::sync::Arc;
use std::time::Duration;
use tempfile::TempDir;
use xet_client::cas_client::LocalTestServerBuilder;
use xet_data::processing::configurations::TranslatorConfig;
use xet_data::processing::test_utils::TestEnvironment;
use xet_data::processing::{FileDownloadSession, FileUploadSession, Sha256Policy, XetFileInfo};
async fn upload_bytes(upload_session: &Arc<FileUploadSession>, name: &str, data: &[u8]) -> XetFileInfo {
@@ -26,11 +24,10 @@ mod tests {
buf
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_async_various_sizes() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
/// Helper: uploads all test cases in one session, finalizes, then returns (env, download_session, xfis).
async fn setup_various_sizes()
-> (TestEnvironment, Arc<FileDownloadSession>, Vec<(&'static str, Vec<u8>, XetFileInfo)>) {
let env = TestEnvironment::new().await;
let test_cases: Vec<(&str, Vec<u8>)> = vec![
("one_byte", vec![0x42]),
@@ -39,14 +36,50 @@ mod tests {
("larger", vec![0xCD; 64 * 1024]),
];
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
let mut xfis = Vec::new();
for (name, data) in test_cases {
let xfi = upload_bytes(&upload_session, name, &data).await;
xfis.push((name, data, xfi));
}
upload_session.finalize().await.unwrap();
for (name, data) in &test_cases {
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, name, data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(env.config.clone()).await.unwrap();
(env, download_session, xfis)
}
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
/// Helper: uploads a single payload, finalizes, returns (env, download_session, xfi).
async fn setup_single(name: &str, data: &[u8]) -> (TestEnvironment, Arc<FileDownloadSession>, XetFileInfo) {
let env = TestEnvironment::new().await;
let upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, name, data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(env.config.clone()).await.unwrap();
(env, download_session, xfi)
}
/// Helper: uploads two payloads, finalizes, returns (env, download_session, xfi_a, xfi_b).
async fn setup_two(
name_a: &str,
data_a: &[u8],
name_b: &str,
data_b: &[u8],
) -> (TestEnvironment, Arc<FileDownloadSession>, XetFileInfo, XetFileInfo) {
let env = TestEnvironment::new().await;
let upload_session = FileUploadSession::new(env.config.clone()).await.unwrap();
let xfi_a = upload_bytes(&upload_session, name_a, data_a).await;
let xfi_b = upload_bytes(&upload_session, name_b, data_b).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(env.config.clone()).await.unwrap();
(env, download_session, xfi_a, xfi_b)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_async_various_sizes() {
let (_env, download_session, xfis) = setup_various_sizes().await;
for (name, data, xfi) in &xfis {
let (_id, mut stream) = download_session.download_unordered_stream(xfi, None).await.unwrap();
let mut chunks = Vec::new();
while let Some((offset, chunk)) = stream.next().await.unwrap() {
@@ -61,25 +94,10 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_blocking_various_sizes() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let (_env, download_session, xfis) = setup_various_sizes().await;
let test_cases: Vec<(&str, Vec<u8>)> = vec![
("one_byte", vec![0x42]),
("small", b"hello world".to_vec()),
("medium", vec![0xAB; 4096]),
("larger", vec![0xCD; 64 * 1024]),
];
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
for (name, data) in &test_cases {
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, name, data).await;
upload_session.finalize().await.unwrap();
let (_id, stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
for (name, data, xfi) in &xfis {
let (_id, stream) = download_session.download_unordered_stream(xfi, None).await.unwrap();
let expected_data = data.clone();
let collected = tokio::task::spawn_blocking(move || {
@@ -99,17 +117,8 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_reassemble_to_file() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let (env, download_session, xfi) = setup_single("file_to_disk", &vec![0xEF; 64 * 1024]).await;
let original_data = vec![0xEF; 64 * 1024];
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "file_to_disk", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
let mut chunks = Vec::new();
@@ -117,25 +126,18 @@ mod tests {
chunks.push((offset, chunk));
}
let original_data = vec![0xEF; 64 * 1024];
let assembled = reassemble(chunks, original_data.len());
let out_path = base_dir.path().join("reassembled.bin");
let out_path = env.base_dir.join("reassembled.bin");
fs::write(&out_path, &assembled).unwrap();
assert_eq!(fs::read(&out_path).unwrap(), original_data);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_total_bytes_expected() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data = b"total bytes tracking test";
let (_env, download_session, xfi) = setup_single("tracking", original_data).await;
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "tracking", original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
assert_eq!(stream.total_bytes_expected(), original_data.len() as u64);
@@ -150,17 +152,9 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_is_complete_loop_drains_all_data() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data: Vec<u8> = (0..131072u32).map(|i| (i % 251) as u8).collect();
let (_env, download_session, xfi) = setup_single("is_complete_loop", &original_data).await;
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "is_complete_loop", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
let mut chunks = Vec::new();
@@ -177,17 +171,9 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_next_returns_none_after_complete() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data = b"extra none calls";
let (_env, download_session, xfi) = setup_single("none_test", original_data).await;
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "none_test", original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
while stream.next().await.unwrap().is_some() {}
@@ -198,17 +184,9 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_cancel_before_start() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data = b"cancel before start";
let (_env, download_session, xfi) = setup_single("cancel_pre", original_data).await;
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "cancel_pre", original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
stream.cancel();
@@ -218,17 +196,9 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_cancel_after_partial_read() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data = b"cancel after partial read test data";
let (_env, download_session, xfi) = setup_single("cancel_mid", original_data).await;
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "cancel_mid", original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
let _ = stream.next().await.unwrap();
@@ -246,19 +216,9 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_multiple_concurrent() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let data_a = b"Unordered stream A for concurrent download";
let data_b = b"Unordered stream B for concurrent download - different content";
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi_a = upload_bytes(&upload_session, "concurrent_a", data_a).await;
let xfi_b = upload_bytes(&upload_session, "concurrent_b", data_b).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_env, download_session, xfi_a, xfi_b) = setup_two("concurrent_a", data_a, "concurrent_b", data_b).await;
let (_id_a, mut stream_a) = download_session.download_unordered_stream(&xfi_a, None).await.unwrap();
let (_id_b, mut stream_b) = download_session.download_unordered_stream(&xfi_b, None).await.unwrap();
@@ -290,23 +250,14 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_drop_without_reading() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data = b"drop without reading cleanup test";
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "drop_test", original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (env, download_session, xfi) = setup_single("drop_test", original_data).await;
let (_id, stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
drop(stream);
tokio::task::yield_now().await;
let out_path = base_dir.path().join("after_drop.txt");
let out_path = env.base_dir.join("after_drop.txt");
let (_id2, n_bytes) = download_session.download_file(&xfi, &out_path).await.unwrap();
assert_eq!(n_bytes, original_data.len() as u64);
assert_eq!(fs::read(&out_path).unwrap(), original_data);
@@ -314,19 +265,10 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unordered_stream_matches_sequential_download() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data: Vec<u8> = (0..=255u8).cycle().take(32 * 1024).collect();
let (env, download_session, xfi) = setup_single("compare_test", &original_data).await;
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "compare_test", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let out_path = base_dir.path().join("sequential.bin");
let out_path = env.base_dir.join("sequential.bin");
let (_id, _) = download_session.download_file(&xfi, &out_path).await.unwrap();
let sequential_result = fs::read(&out_path).unwrap();
@@ -343,17 +285,8 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn stress_test_repeated_blocking_downloads() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data: Vec<u8> = (0..65536u32).map(|i| (i % 251) as u8).collect();
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "stress_blocking", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_env, download_session, xfi) = setup_single("stress_blocking", &original_data).await;
for _ in 0..30 {
let (_id, stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
@@ -374,17 +307,8 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn stress_test_repeated_async_downloads() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data: Vec<u8> = (0..65536u32).map(|i| (i % 251) as u8).collect();
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "stress_async", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_env, download_session, xfi) = setup_single("stress_async", &original_data).await;
for _ in 0..30 {
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
@@ -399,17 +323,8 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn stress_test_concurrent_blocking_downloads() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data: Vec<u8> = (0..65536u32).map(|i| (i % 251) as u8).collect();
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "stress_concurrent", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_env, download_session, xfi) = setup_single("stress_concurrent", &original_data).await;
let mut handles = Vec::new();
for _ in 0..8 {
@@ -433,19 +348,10 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn stress_test_large_file_download() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data: Vec<u8> = (0..262144u32)
.map(|i| ((i.wrapping_mul(7919) ^ (i >> 3)) % 256) as u8)
.collect();
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "stress_large", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_env, download_session, xfi) = setup_single("stress_large", &original_data).await;
{
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();
@@ -476,17 +382,8 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn stress_test_mixed_concurrent_async_and_blocking() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data: Vec<u8> = (0..65536u32).map(|i| (i % 251) as u8).collect();
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "stress_mixed", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_env, download_session, xfi) = setup_single("stress_mixed", &original_data).await;
let mut handles: Vec<tokio::task::JoinHandle<Vec<u8>>> = Vec::new();
@@ -523,17 +420,8 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn stress_test_rapid_create_and_drop() {
let server = LocalTestServerBuilder::new().start().await;
let base_dir = TempDir::new().unwrap();
let config = Arc::new(TranslatorConfig::test_server_config(server.http_endpoint(), base_dir.path()).unwrap());
let original_data: Vec<u8> = (0..32768u32).map(|i| (i % 199) as u8).collect();
let upload_session = FileUploadSession::new(config.clone()).await.unwrap();
let xfi = upload_bytes(&upload_session, "stress_drop", &original_data).await;
upload_session.finalize().await.unwrap();
let download_session = FileDownloadSession::new(config.clone()).await.unwrap();
let (_env, download_session, xfi) = setup_single("stress_drop", &original_data).await;
for _ in 0..20 {
let (_id, mut stream) = download_session.download_unordered_stream(&xfi, None).await.unwrap();

1
xet_pkg/Cargo.toml
View File

@@ -32,6 +32,7 @@ pyo3 = { workspace = true, optional = true }
[features]
smoke-test = []
python = ["xet-runtime/python", "dep:pyo3"]
simulation = ["xet-client/simulation"]
[dev-dependencies]
anyhow = { workspace = true }

1
xet_pkg/tests/test_legacy_data_client.rs
View File

@@ -1,3 +1,4 @@
#![cfg(feature = "simulation")]
use std::fs;
use std::sync::Arc;

2
xet_runtime/Cargo.toml
View File

@@ -17,7 +17,7 @@ colored = { workspace = true }
const-str = "1.1"
ctor = { workspace = true }
dirs = { workspace = true }
duration-str = { workspace = true }
humantime = { workspace = true }
git-version = { workspace = true }
konst = "0.4"
lazy_static = { workspace = true }

6
xet_runtime/src/utils/configuration_utils.rs
View File

@@ -130,11 +130,11 @@ impl<T: ParsableConfigValue> ParsableConfigValue for Option<T> {
/// Implement proper parsing for Duration types as well.
///
/// Now the following suffixes are supported [y, mon, d, h, m, s, ms];
/// see the duration_str crate for the full list.
/// Now the following suffixes are supported: s, ms, us, ns, m, h, d, etc.;
/// see the humantime crate for the full list.
impl ParsableConfigValue for std::time::Duration {
fn parse_user_value(value: &str) -> Option<Self> {
duration_str::parse(value).ok()
humantime::parse_duration(value).ok()
}
fn to_config_string(&self) -> String {