mirror of
https://github.com/huggingface/xet-core.git
synced 2026-06-04 13:30:29 +08:00
e701aeddac
`XetSession` always created its own tokio runtime via
`XetRuntime::new_with_config`, and calling `external_run_async_task`
panics when already inside a tokio context. This blocked embedding the
session in async Rust frameworks.
Core strategy:
- `RuntimeMode` enum —
`Owned` (session created its own thread pool via
`XetSessionBuilder::build` or `XetSessionBuilder::build_async` when
outside tokio context. Both `_blocking` and async methods are supported.
Async methods use an internal `bridge_to_owned` bridge that routes
futures onto the owned thread pool, so they work from any executor
(tokio, smol, async-std))
vs
`External` (session wraps a caller-supplied tokio handle via
`XetSessionBuilder::with_tokio_handle` or
`XetSessionBuilder::build_async` when inside qualified tokio context.
Only async methods may be called; `_blocking` methods return
`SessionError::WrongRuntimeMode`. No second thread pool is created).
- `XetRuntime::bridge_to_owned` — a new bridge that routes a future onto
the owned tokio thread pool from any executor (smol, async-std,
futures::executor, non-qualified tokio runtime) by delivering the result
via a `tokio::sync::oneshot` channel that can be polled by any async
executor.
- Async public API — `UploadCommit` and `DownloadGroup` methods
(`upload_from_path`, `upload_bytes`, `upload_file`, `commit`, `finish`)
are now async fn. Factory methods `XetSession::new_upload_commit` and
`new_download_group` are async.
Example:
```
let session = XetSessionBuilder::new().build_async().await?;
// Upload
let commit = session.new_upload_commit().await?;
let handle = commit.upload_from_path("file.bin".into()).await?;
let results = commit.commit().await?;
// Download
let group = session.new_download_group().await?;
let info = XetFileInfo {
hash: ...,
file_size: ...,
};
let dl_handle = group.download_file_to_path(info, "out/file.bin".into())?;
let finish_results = group.finish().await?;
```
- Sync wrappers — New `UploadCommitSync` / `DownloadGroupSync` in
`xet_session/sync/` expose a fully blocking API for sync Rust and Python
(PyO3) callers. Returned by `new_upload_commit_blocking()` and
`new_download_group_blocking()`.
Example:
```
let session = XetSessionBuilder::new().build()?;
// Upload
let commit = session.new_upload_commit_blocking()?;
let handle = commit.upload_from_path("file.bin".into())?;
let results = commit.commit()?;
let m = results.values().next().unwrap().as_ref().as_ref().unwrap();
// Download
let group = session.new_download_group_blocking()?;
let info = XetFileInfo {
hash: ...,
file_size: ...,
};
let dl_handle = group.download_file_to_path(info, "out/file.bin".into())?;
let finish_results = group.finish()?;
```
Additional fixes: `download_file_to_path` and `upload_from_path` now
canonicalize paths with `std::path::absolute` before enqueuing; task
status is only overwritten when still `Running`, preventing a race with
concurrent abort().
Fix XET-891
---------
Co-authored-by: Hoyt Koepke <hoytak@huggingface.co>
141 lines
2.7 KiB
TOML
141 lines
2.7 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
# Consolidated packages
|
|
"xet_runtime",
|
|
"xet_core_structures",
|
|
"xet_client",
|
|
"xet_data",
|
|
"xet_pkg",
|
|
|
|
# Top-level crates (not published as packages)
|
|
"git_xet",
|
|
"simulation",
|
|
]
|
|
exclude = ["simulation/chunk_cache_bench", "hf_xet", "wasm/hf_xet_wasm", "wasm/hf_xet_thin_wasm"]
|
|
|
|
[workspace.package]
|
|
version = "1.4.0"
|
|
edition = "2024"
|
|
license = "Apache-2.0"
|
|
repository = "https://github.com/huggingface/xet-core"
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = true
|
|
debug = 1
|
|
|
|
[profile.opt-test]
|
|
inherits = "dev"
|
|
opt-level = 3
|
|
debug = 1
|
|
|
|
[workspace.dependencies]
|
|
anyhow = "1"
|
|
async-std = "1"
|
|
async-trait = "0.1"
|
|
axum = "0.8"
|
|
base64 = "0.22"
|
|
bincode = "1.3"
|
|
bitflags = { version = "2.10", features = ["serde"] }
|
|
blake3 = "1.8"
|
|
bytemuck = "1"
|
|
bytes = "1.11"
|
|
chrono = "0.4"
|
|
clap = { version = "4", features = ["derive"] }
|
|
colored = "3"
|
|
console-subscriber = "0.5"
|
|
countio = { version = "0.3", features = ["futures"] }
|
|
crc32fast = "1.5"
|
|
csv = "1"
|
|
ctor = "0.6"
|
|
derivative = "2.2"
|
|
dirs = "6.0"
|
|
duration-str = "0.19"
|
|
futures = "0.3"
|
|
futures-util = "0.3"
|
|
gearhash = "0.1"
|
|
getrandom = { version = "0.4", features = ["wasm_js"] }
|
|
git-url-parse = "0.4"
|
|
git-version = "0.3"
|
|
git2 = "0.20"
|
|
half = "2.7"
|
|
heapify = "0.2"
|
|
heed = "0.22"
|
|
http = "1"
|
|
human-bandwidth = "0.1"
|
|
hyper = "1.8"
|
|
hyper-util = "0.1"
|
|
itertools = "0.14"
|
|
lazy_static = "1.5"
|
|
libc = "0.2"
|
|
lz4_flex = "0.12"
|
|
mockall = "0.14"
|
|
more-asserts = "0.3"
|
|
once_cell = "1.21"
|
|
oneshot = "0.1"
|
|
paste = "1.0"
|
|
pin-project = "1"
|
|
prometheus = "0.14"
|
|
rand = "0.9"
|
|
rand_chacha = "0.9"
|
|
regex = "1"
|
|
reqwest = { version = "0.13.1", features = [
|
|
"json",
|
|
"stream",
|
|
"system-proxy",
|
|
"socks",
|
|
], default-features = false }
|
|
reqwest-middleware = "0.5"
|
|
reqwest-retry = "0.9"
|
|
rust-netrc = "0.1"
|
|
safe-transmute = "0.11"
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_repr = "0.1"
|
|
sha2 = "0.10"
|
|
shell-words = "1.1"
|
|
shellexpand = "3.1"
|
|
smol = "2"
|
|
static_assertions = "1.1"
|
|
statrs = "0.18"
|
|
sysinfo = "0.38"
|
|
tempfile = "3.25"
|
|
thiserror = "2.0"
|
|
tokio = { version = "1.49" }
|
|
tokio-retry = "0.3"
|
|
tokio-util = { version = "0.7" }
|
|
tower-service = "0.3"
|
|
tracing = "0.1"
|
|
tracing-appender = "0.2"
|
|
tracing-log = "0.2"
|
|
tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
|
|
ulid = "1.2"
|
|
url = "2.5"
|
|
urlencoding = "2.1"
|
|
uuid = "1"
|
|
walkdir = "2"
|
|
warp = { version = "0.4", features = ["server"] }
|
|
web-time = "1.1"
|
|
whoami = "2"
|
|
|
|
# windows
|
|
winapi = { version = "0.3", features = [
|
|
"winerror",
|
|
"winnt",
|
|
"handleapi",
|
|
"processthreadsapi",
|
|
"securitybaseapi",
|
|
] }
|
|
|
|
# dev-deps
|
|
approx = "0.5"
|
|
httpmock = "0.8"
|
|
rand_core = "0.6"
|
|
rand_distr = "0.5"
|
|
russh = "0.55"
|
|
serial_test = "3"
|
|
tempdir = "0.3"
|
|
tracing-test = { version = "0.2", features = ["no-env-filter"] }
|
|
wiremock = "0.6"
|