mirror of
https://github.com/huggingface/xet-core.git
synced 2026-06-04 13:30:29 +08:00
5868f64ab9
CI for hf-hub is running cargo audit and found many issues through
hf-xet transitive deps. this PR attempts to solve some of them (not
necessarily all of them).
Main changes:
- dropped derivative and reqwest-retry
- replaced bincode with postcard, only used in testing
- upgrade xet-core rand usage
- added audit CI step and ignoring some issues that we can't easily fix.
<!-- CURSOR_SUMMARY -->
---
> [!NOTE]
> **Medium Risk**
> Medium risk because it removes `reqwest-retry`/`derivative` and
replaces part of the retry classification logic with an in-house
equivalent, which could subtly change HTTP retry behavior; the remaining
changes are dependency/version bumps and test-only serialization swaps.
>
> **Overview**
> Adds a new CI `cargo audit` job and introduces `.cargo/audit.toml` to
ignore a small set of **dev-only** RustSec advisories with documented
rationale.
>
> Reduces audit surface by dropping `derivative` (manual `Debug` impl
for `AuthConfig`) and removing `reqwest-retry`, replacing its
status-code classification with a local `Retryable` enum +
`default_on_request_success` helper in `RetryWrapper`.
>
> Updates workspace deps (notably `rand` to `0.10` and `rand_distr` to
`0.6`) and adjusts call sites to the newer `rand` APIs (`RngExt`
imports, minor test/bench tweaks). Test-only binary serialization
switches from `bincode` to `postcard` (and updates affected tests), with
corresponding lockfile updates across crates.
>
> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit
26377f4a1c. Bugbot is set up for automated
code reviews on this repo. Configure
[here](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- /CURSOR_SUMMARY -->
107 lines
2.7 KiB
TOML
107 lines
2.7 KiB
TOML
[package]
|
|
name = "xet-core-structures"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
homepage.workspace = true
|
|
repository.workspace = true
|
|
description = "Core data structures including MerkleHash, metadata shards, and Xorb objects."
|
|
readme = "README.md"
|
|
keywords = ["huggingface"]
|
|
categories = ["artificial-intelligence", "data-structures"]
|
|
|
|
[lib]
|
|
name = "xet_core_structures"
|
|
path = "src/lib.rs"
|
|
|
|
[[bench]]
|
|
name = "compression_bench"
|
|
harness = false
|
|
bench = true
|
|
|
|
[[bench]]
|
|
name = "bg_split_regroup_bench"
|
|
harness = false
|
|
bench = true
|
|
|
|
[dependencies]
|
|
xet-runtime = { version = "1.5.1", path = "../xet_runtime" }
|
|
|
|
async-trait = { workspace = true }
|
|
base64 = { workspace = true }
|
|
blake3 = { workspace = true }
|
|
bytes = { workspace = true }
|
|
clap = { workspace = true }
|
|
countio = { workspace = true }
|
|
csv = { workspace = true }
|
|
futures = { workspace = true }
|
|
futures-util = { workspace = true }
|
|
heapify = { workspace = true }
|
|
itertools = { workspace = true }
|
|
lazy_static = { workspace = true }
|
|
lz4_flex = { workspace = true }
|
|
more-asserts = { workspace = true }
|
|
rand = { workspace = true }
|
|
regex = { workspace = true }
|
|
safe-transmute = { workspace = true }
|
|
serde = { workspace = true }
|
|
static_assertions = { workspace = true }
|
|
tempfile = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
tracing = { workspace = true }
|
|
|
|
[target.'cfg(not(target_family = "wasm"))'.dependencies]
|
|
bytemuck = { workspace = true }
|
|
tokio = { workspace = true, features = [
|
|
"time",
|
|
"rt",
|
|
"macros",
|
|
"sync",
|
|
"test-util",
|
|
"io-util",
|
|
"rt-multi-thread",
|
|
] }
|
|
tokio-util = { workspace = true, features = ["io"] }
|
|
uuid = { workspace = true, features = ["v4"] }
|
|
|
|
[target.'cfg(target_family = "wasm")'.dependencies]
|
|
getrandom = { workspace = true, features = ["wasm_js"] }
|
|
tokio = { workspace = true, features = [
|
|
"sync",
|
|
"macros",
|
|
"io-util",
|
|
"rt",
|
|
"time",
|
|
] }
|
|
uuid = { workspace = true, features = ["v4", "js"] }
|
|
web-time = { workspace = true }
|
|
|
|
[dev-dependencies]
|
|
postcard = { workspace = true }
|
|
futures-util = { workspace = true }
|
|
half = { workspace = true }
|
|
rand = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
serial_test = { workspace = true }
|
|
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
|
|
|
|
[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
|
|
tempfile = { workspace = true }
|
|
|
|
[[bin]]
|
|
name = "shard_benchmark"
|
|
path = "src/metadata_shard/shard_benchmark.rs"
|
|
|
|
[[bin]]
|
|
path = "src/xorb_object/byte_grouping/compression_stats/collect_compression_stats.rs"
|
|
name = "collect_compression_stats"
|
|
|
|
[[bin]]
|
|
path = "src/xorb_object/byte_grouping/bg4_prediction_benchmark.rs"
|
|
name = "bg4_prediction_benchmark"
|
|
|
|
[features]
|
|
strict = []
|
|
smoke-test = []
|
|
simulation = ["xet-runtime/simulation"]
|