mirror of
https://github.com/huggingface/xet-core.git
synced 2026-06-04 13:30:29 +08:00
45d38a13a9
This PR is a massive rearrangement of the code base into 5 packages intended for release on cargo. The directories and corresponding packages are: 1. xet_runtime/ — compiles into the xet-runtime package. Contains the runtime, config, and logging management. 2. xet_core_structures/ — compiles into the xet-core-structures package. Contains core data structures for hashing, shards, and xorbs as well as internal data structures that depend on these. 3. xet_client/ — compiles into the xet-client package, contains client code for remotely connecting to the Hugging Face servers. 4. xet_data/ — compiles into the xet-data package, contains the data processing pipeline: chunking/deduplication, file reconstruction, clean/smudge operations, and progress tracking. 5. xet_pkg/ — compiles into the hf-xet package, provides the top-level session-based API for file upload and download with user-facing error categorization. This is the primary package downstream dependencies would use. This also contains a single summary error type, XetError, that translates cleanly into python error types. In addition, the other tools are: - git_xet/ — the git_xet CLI binary crate (location preserved). - hf_xet/ -- the hf_xet python package (location preserved). - simulation/ — the simulation crate for upload scenario benchmarking. - wasm/ -- the wasm objects. The full description — and information for an AI agent to use to update downstream dependencies — is at api_changes/update_260309_package_restructure.md. Summary of moves: - xet_runtime: became xet_runtime::core inside xet_runtime/. - utils: became xet_runtime::utils inside xet_runtime/. - xet_config: became xet_runtime::config inside xet_runtime/. - xet_logging: became xet_runtime::logging inside xet_runtime/. - error_printer: became xet_runtime::error_printer inside xet_runtime/. - file_utils: became xet_runtime::file_utils inside xet_runtime/. - merklehash: became xet_core_structures::merklehash inside xet_core_structures/. - mdb_shard: became xet_core_structures::metadata_shard inside xet_core_structures/. - xorb_object: became xet_core_structures::xorb_object inside xet_core_structures/. - cas_client: became xet_client::cas_client inside xet_client/. - hub_client: became xet_client::hub_client inside xet_client/. - cas_types: became xet_client::cas_types inside xet_client/. - chunk_cache: became xet_client::chunk_cache inside xet_client/. - data: became xet_data::processing inside xet_data/. - deduplication: became xet_data::deduplication inside xet_data/. - file_reconstruction: became xet_data::file_reconstruction inside xet_data/. - progress_tracking: became xet_data::progress_tracking inside xet_data/. - xet_session: became xet::xet_session inside xet_pkg/. - Wasm packages (hf_xet_wasm, hf_xet_thin_wasm): moved from top-level into wasm/; internal imports updated, public APIs unchanged.
139 lines
2.7 KiB
TOML
139 lines
2.7 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
# Consolidated packages
|
|
"xet_runtime",
|
|
"xet_core_structures",
|
|
"xet_client",
|
|
"xet_data",
|
|
"xet_pkg",
|
|
|
|
# Top-level crates (not published as packages)
|
|
"git_xet",
|
|
"simulation",
|
|
]
|
|
exclude = ["simulation/chunk_cache_bench", "hf_xet", "wasm/hf_xet_wasm", "wasm/hf_xet_thin_wasm"]
|
|
|
|
[workspace.package]
|
|
version = "1.4.0"
|
|
edition = "2024"
|
|
license = "Apache-2.0"
|
|
repository = "https://github.com/huggingface/xet-core"
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = true
|
|
debug = 1
|
|
|
|
[profile.opt-test]
|
|
inherits = "dev"
|
|
opt-level = 3
|
|
debug = 1
|
|
|
|
[workspace.dependencies]
|
|
anyhow = "1"
|
|
axum = "0.8"
|
|
async-trait = "0.1"
|
|
base64 = "0.22"
|
|
bincode = "1.3"
|
|
bitflags = { version = "2.10", features = ["serde"] }
|
|
blake3 = "1.8"
|
|
bytemuck = "1"
|
|
bytes = "1.11"
|
|
chrono = "0.4"
|
|
clap = { version = "4", features = ["derive"] }
|
|
colored = "3"
|
|
console-subscriber = "0.5"
|
|
countio = { version = "0.3", features = ["futures"] }
|
|
crc32fast = "1.5"
|
|
csv = "1"
|
|
ctor = "0.6"
|
|
derivative = "2.2"
|
|
dirs = "6.0"
|
|
human-bandwidth = "0.1"
|
|
duration-str = "0.19"
|
|
futures = "0.3"
|
|
futures-util = "0.3"
|
|
gearhash = "0.1"
|
|
getrandom = { version = "0.4", features = ["wasm_js"] }
|
|
git-url-parse = "0.4"
|
|
git-version = "0.3"
|
|
git2 = "0.20"
|
|
half = "2.7"
|
|
heapify = "0.2"
|
|
heed = "0.22"
|
|
http = "1"
|
|
hyper = "1.8"
|
|
hyper-util = "0.1"
|
|
itertools = "0.14"
|
|
lazy_static = "1.5"
|
|
libc = "0.2"
|
|
lz4_flex = "0.12"
|
|
mockall = "0.14"
|
|
more-asserts = "0.3"
|
|
once_cell = "1.21"
|
|
oneshot = "0.1"
|
|
paste = "1.0"
|
|
pin-project = "1"
|
|
prometheus = "0.14"
|
|
rand = "0.9"
|
|
rand_chacha = "0.9"
|
|
regex = "1"
|
|
reqwest = { version = "0.13.1", features = [
|
|
"json",
|
|
"stream",
|
|
"system-proxy",
|
|
"socks",
|
|
], default-features = false }
|
|
reqwest-middleware = "0.5"
|
|
reqwest-retry = "0.9"
|
|
rust-netrc = "0.1"
|
|
safe-transmute = "0.11"
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_repr = "0.1"
|
|
sha2 = "0.10"
|
|
shell-words = "1.1"
|
|
shellexpand = "3.1"
|
|
static_assertions = "1.1"
|
|
statrs = "0.18"
|
|
sysinfo = "0.38"
|
|
tempfile = "3.25"
|
|
thiserror = "2.0"
|
|
tokio = { version = "1.49" }
|
|
tokio-retry = "0.3"
|
|
tokio-util = { version = "0.7" }
|
|
tower-service = "0.3"
|
|
tracing = "0.1"
|
|
tracing-appender = "0.2"
|
|
tracing-log = "0.2"
|
|
tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
|
|
ulid = "1.2"
|
|
url = "2.5"
|
|
urlencoding = "2.1"
|
|
uuid = "1"
|
|
walkdir = "2"
|
|
warp = { version = "0.4", features = ["server"] }
|
|
web-time = "1.1"
|
|
whoami = "2"
|
|
|
|
# windows
|
|
winapi = { version = "0.3", features = [
|
|
"winerror",
|
|
"winnt",
|
|
"handleapi",
|
|
"processthreadsapi",
|
|
"securitybaseapi",
|
|
] }
|
|
|
|
# dev-deps
|
|
approx = "0.5"
|
|
httpmock = "0.8"
|
|
rand_core = "0.6"
|
|
rand_distr = "0.5"
|
|
russh = "0.55"
|
|
serial_test = "3"
|
|
tempdir = "0.3"
|
|
tracing-test = { version = "0.2", features = ["no-env-filter"] }
|
|
wiremock = "0.6"
|