pub_soft/xet-core
3
0
Fork 0
mirror of https://github.com/huggingface/xet-core.git synced 2026-06-04 13:30:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

clean some more crates (#66)

Browse Source 
* small mods to crates

* mdb_shard

* more crates

* mdb
This commit is contained in:
Assaf Vayner
2024-10-25 13:14:54 -07:00
committed by GitHub
parent 5ca7c3882b
commit 8364c9337e
12 changed files with 41 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
Cargo.lock generated
View File

@@ -64,15 +64,6 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "ansi_term"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
dependencies = [
"winapi",
]
[[package]]
name = "anstream"
version = "0.6.15"
@@ -418,12 +409,6 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.21.7"
@@ -465,15 +450,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "binary-heap-plus"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4551d8382e911ecc0d0f0ffb602777988669be09447d536ff4388d1def11296"
dependencies = [
"compare",
]
[[package]]
name = "bincode"
version = "1.3.3"
@@ -523,16 +499,6 @@ dependencies = [
"constant_time_eq",
]
[[package]]
name = "block-buffer"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
dependencies = [
"block-padding 0.2.1",
"generic-array",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
@@ -542,12 +508,6 @@ dependencies = [
"generic-array",
]
[[package]]
name = "block-padding"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
[[package]]
name = "block-padding"
version = "0.3.3"
@@ -631,16 +591,12 @@ name = "cas_object"
version = "0.1.0"
dependencies = [
"anyhow",
"bincode",
"blake3",
"bytes",
"cas_types",
"http 1.1.0",
"lz4_flex",
"merkledb",
"merklehash",
"rand 0.8.5",
"tempfile",
"tracing",
"xet_error",
]
@@ -649,7 +605,6 @@ dependencies = [
name = "cas_types"
version = "0.1.0"
dependencies = [
"anyhow",
"merklehash",
"serde",
"serde_repr",
@@ -787,13 +742,9 @@ version = "2.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
"ansi_term",
"atty",
"bitflags 1.3.2",
"strsim 0.8.0",
"textwrap 0.11.0",
"unicode-width",
"vec_map",
]
[[package]]
@@ -906,12 +857,6 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "compare"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "120133d4db2ec47efe2e26502ee984747630c67f51974fca0b6c1340cf2368d3"
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@@ -1230,16 +1175,6 @@ dependencies = [
"xet_error",
]
[[package]]
name = "deadqueue"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16a2561fd313df162315935989dceb8c99db4ee1933358270a57a3cfb8c957f3"
dependencies = [
"crossbeam-queue",
"tokio",
]
[[package]]
name = "der"
version = "0.7.9"
@@ -1271,22 +1206,13 @@ dependencies = [
"syn 2.0.85",
]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"block-buffer",
"const-oid",
"crypto-common",
"subtle",
@@ -1830,15 +1756,6 @@ dependencies = [
"fxhash",
]
[[package]]
name = "heck"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "heck"
version = "0.4.1"
@@ -1923,7 +1840,7 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
dependencies = [
"digest 0.10.7",
"digest",
]
[[package]]
@@ -2208,7 +2125,7 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
dependencies = [
"block-padding 0.3.3",
"block-padding",
"generic-array",
]
@@ -2304,15 +2221,6 @@ dependencies = [
"simple_asn1",
]
[[package]]
name = "keccak"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654"
dependencies = [
"cpufeatures",
]
[[package]]
name = "kv-log-macro"
version = "1.0.7"
@@ -2495,7 +2403,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if 1.0.0",
"digest 0.10.7",
"digest",
]
[[package]]
@@ -2503,19 +2411,14 @@ name = "mdb_shard"
version = "0.14.5"
dependencies = [
"anyhow",
"async-scoped",
"async-trait",
"binary-heap-plus",
"clap 3.2.25",
"lazy_static",
"merklehash",
"more-asserts",
"rand 0.8.5",
"regex",
"serde",
"static_assertions",
"tempdir",
"tempfile",
"tokio",
"tracing",
"utils",
@@ -2549,9 +2452,7 @@ dependencies = [
"blake3",
"clap 3.2.25",
"criterion 0.3.6",
"futures",
"gearhash",
"itertools 0.10.5",
"lazy_static",
"merklehash",
"parutils",
@@ -2559,10 +2460,8 @@ dependencies = [
"rand_chacha",
"rand_core 0.6.4",
"rayon",
"ron",
"rustc-hash 1.1.0",
"serde",
"structopt",
"tempfile",
"tokio",
"tracing",
@@ -2575,15 +2474,10 @@ name = "merklehash"
version = "0.14.5"
dependencies = [
"blake3",
"generic-array",
"heed",
"rand 0.8.5",
"rand_chacha",
"rand_core 0.6.4",
"safe-transmute",
"serde",
"sha3",
"structopt",
]
[[package]]
@@ -2784,12 +2678,6 @@ version = "11.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
[[package]]
name = "opaque-debug"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "opendal"
version = "0.48.0"
@@ -2976,12 +2864,10 @@ dependencies = [
"anyhow",
"async-scoped",
"async-trait",
"deadqueue",
"futures",
"len-trait",
"more-asserts",
"tokio",
"tracing",
]
[[package]]
@@ -2990,7 +2876,7 @@ version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
dependencies = [
"digest 0.10.7",
"digest",
"hmac",
]
@@ -3295,9 +3181,6 @@ version = "0.14.5"
dependencies = [
"atty",
"crossterm",
"more-asserts",
"tokio",
"tracing",
"utils",
]
@@ -3835,17 +3718,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "ron"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4"
dependencies = [
"base64 0.13.1",
"bitflags 1.3.2",
"serde",
]
[[package]]
name = "rsa"
version = "0.9.6"
@@ -3853,7 +3725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc"
dependencies = [
"const-oid",
"digest 0.10.7",
"digest",
"num-bigint-dig",
"num-integer",
"num-traits",
@@ -4278,7 +4150,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if 1.0.0",
"cpufeatures",
"digest 0.10.7",
"digest",
]
[[package]]
@@ -4295,19 +4167,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
dependencies = [
"cfg-if 1.0.0",
"cpufeatures",
"digest 0.10.7",
]
[[package]]
name = "sha3"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809"
dependencies = [
"block-buffer 0.9.0",
"digest 0.9.0",
"keccak",
"opaque-debug",
"digest",
]
[[package]]
@@ -4361,7 +4221,7 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
dependencies = [
"digest 0.10.7",
"digest",
"rand_core 0.6.4",
]
@@ -4478,12 +4338,6 @@ dependencies = [
"vte",
]
[[package]]
name = "strsim"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "strsim"
version = "0.10.0"
@@ -4496,30 +4350,6 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "structopt"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"
dependencies = [
"clap 2.34.0",
"lazy_static",
"structopt-derive",
]
[[package]]
name = "structopt-derive"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
dependencies = [
"heck 0.3.3",
"proc-macro-error",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "subtle"
version = "2.6.1"
@@ -5137,12 +4967,6 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
[[package]]
name = "unicode-segmentation"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
[[package]]
name = "unicode-width"
version = "0.1.14"
@@ -5220,12 +5044,6 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version_check"
version = "0.9.5"

2
cas_client/src/remote_client.rs
View File

@@ -369,7 +369,7 @@ async fn download_range(
#[cfg(test)]
mod tests {
use cas_object::test_utils::*;
use cas_object::test_utils::{build_cas_object, ChunkSize};
use tracing_test::traced_test;
use super::*;

7
cas_object/Cargo.toml
View File

@@ -5,18 +5,11 @@ edition = "2021"
[dependencies]
xet_error = { path = "../xet_error" }
cas_types = { path = "../cas_types" }
merkledb = { path = "../merkledb" }
merklehash = { path = "../merklehash" }
anyhow = "1.0.88"
bincode = "1.3.3"
http = "1.1.0"
tempfile = "3.12.0"
tracing = "0.1.40"
lz4_flex = "0.11.3"
bytes = "1.7.2"
rand = "0.8.5"
blake3 = "1.5.4"
[dev-dependencies]
tempfile = "3.12.0"

3
cas_types/Cargo.toml
View File

@@ -5,7 +5,6 @@ edition = "2021"
[dependencies]
merklehash = { path = "../merklehash" }
xet_error = { path = "../xet_error"}
anyhow = "1.0.86"
xet_error = { path = "../xet_error" }
serde = { version = "1.0.208", features = ["derive"] }
serde_repr = "0.1.19"

4
chunk_cache/Cargo.toml
View File

@@ -14,8 +14,8 @@ blake3 = "1.5.4"
base64 = "0.22.1"
tracing = "0.1.40"
sorted-vec = "0.8.3"
rand = "0.8"
tempdir = "0.3.7"
rand = "0.8.5"
[dev-dependencies]
tokio = { version = "1.36", features = ["full"] }
tempdir = "0.3.7"

14
file_utils/Cargo.toml
View File

@@ -4,16 +4,13 @@ version = "0.14.2"
edition = "2021"
[dependencies]
colored = "2.0.0"
tracing = "0.1.*"
libc = "0.2"
lazy_static = "1.4.0"
whoami = "1.4.1"
anyhow = "1"
tempfile = "3.2.0"
rand = "0.8.4"
rand = "0.8.5"
[target.'cfg(windows)'.dependencies]
winapi = { version = "0.3", features = [
"winerror",
"winnt",
@@ -21,3 +18,10 @@ winapi = { version = "0.3", features = [
"processthreadsapi",
"securitybaseapi",
] }
[target.'cfg(unix)'.dependencies]
whoami = "1.4.1"
colored = "2.0.0"
[dev-dependencies]
anyhow = "1"

13
mdb_shard/Cargo.toml
View File

@@ -7,22 +7,17 @@ edition = "2021"
[dependencies]
merklehash = { path = "../merklehash" }
xet_error = {path = "../xet_error"}
utils = {path = "../utils"}
more-asserts = "0.3.*"
xet_error = { path = "../xet_error" }
utils = { path = "../utils" }
tempdir = "0.3.7"
serde = {version="1.0.129", features = ["derive"]}
tokio = { version = "1.36", features = ["full"] }
lazy_static = "1.4.0"
regex = "1.5"
tracing = "0.1.*"
uuid = {version = "1.3.2", features = ["v4"]}
async-scoped = {version = "0.7", features = ["use-tokio"]}
binary-heap-plus = "0.5.0"
tempfile = "3.2.0"
uuid = { version = "1.3.2", features = ["v4"] }
clap = { version = "3.1.6", features = ["derive"] }
anyhow = "1"
rand = {version = "0.8.5", features = ["small_rng"]}
rand = { version = "0.8.5", features = ["small_rng"] }
async-trait = "0.1.9"
static_assertions = "1.1.0"

3
mdb_shard/src/shard_file_manager.rs
View File

@@ -502,7 +502,6 @@ mod tests {
use std::cmp::min;
use std::time::Duration;
use more_asserts::assert_lt;
use rand::prelude::*;
use tempdir::TempDir;
@@ -949,7 +948,7 @@ mod tests {
}
if let Some(n) = last_num_files {
assert_lt!(n_merged_shards, n);
assert!(n_merged_shards < n, "n_merged_shards({n_merged_shards}) < n({n})");
}
last_num_files = Some(n_merged_shards);

18
merkledb/Cargo.toml
View File

@@ -9,13 +9,13 @@ edition = "2021"
doctest = false
[dependencies]
merklehash = { path = "../merklehash"}
rand = "0.8.4"
merklehash = { path = "../merklehash" }
xet_error = { path = "../xet_error" }
parutils = { path = "../parutils" }
rand = "0.8.5"
rand_core = "0.6.3"
rand_chacha = "0.3.1"
serde = {version="1.0.129", features = ["derive"]}
structopt = "0.3.22"
ron = "0.6.4"
serde = { version = "1.0.129", features = ["derive"] }
gearhash = "0.1.3"
walkdir = "2.3.2"
rayon = "1.5.1"
@@ -23,21 +23,17 @@ bincode = "1.3.3"
rustc-hash = "1.1.0"
tempfile = "3.2.0"
bitflags = "1.3.2"
itertools = "0.10.1"
xet_error = {path = "../xet_error"}
tracing = "0.1.31"
async-trait = "0.1.9"
tokio = { version = "1.36", features = ["full"] }
futures = "0.3"
lazy_static = "1.4.0"
clap = { version = "3.1.6", features = ["derive"] }
blake3 = "1.5.1"
parutils = { path = "../parutils"}
[dev-dependencies]
criterion = { version = "0.3.5", features = ["html_reports"] }
lazy_static = "1.4.0"
async-scoped = {version = "0.7", features = ["use-tokio"]}
async-scoped = { version = "0.7", features = ["use-tokio"] }
tokio = { version = "1.36", features = ["full"] }
[[bench]]
name = "rolling_hash_benchmark"

7
merklehash/Cargo.toml
View File

@@ -6,13 +6,8 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
rand = "0.8.4"
rand_core = "0.6.3"
rand_chacha = "0.3.1"
structopt = "0.3.22"
sha3 = "0.9.1"
rand = "0.8.5"
blake3 = "1.5.1"
generic-array = "0.14.4"
safe-transmute = "0.11.2"
serde = { version = "1.0.129", features = ["derive"] }
heed = "0.11"

10
parutils/Cargo.toml
View File

@@ -7,14 +7,14 @@ edition = "2021"
[dependencies]
tokio = { version = "1.36", features = ["full"] }
anyhow = "1"
async-scoped = {version = "0.7", features = ["use-tokio"]}
async-scoped = { version = "0.7", features = ["use-tokio"] }
futures = "0.3.28"
async-trait = "0.1.53"
deadqueue = "0.2.4"
tracing = "0.1.31"
more-asserts = "0.3.*"
len-trait = "0.6.1"
[dev-dependencies]
anyhow = "1"
more-asserts = "0.3.*"
[features]
strict = []

7
progress_reporting/Cargo.toml
View File

@@ -13,10 +13,5 @@ path = "src/main.rs"
[dependencies]
crossterm = "0.25.0"
tokio = { version = "1.36", features = ["full"] }
utils = {path = "../utils"}
tracing = "0.1.*"
more-asserts = "0.3.*"
utils = { path = "../utils" }
atty = "0.2"