pub_soft/xet-core
3
0
Fork 0
mirror of https://github.com/huggingface/xet-core.git synced 2026-06-04 13:30:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove prometheus dependency and metrics (#769)

Browse Source 
## Summary
- Remove the `prometheus` crate dependency from the workspace and
`xet_data`
- Delete `prometheus_metrics.rs` which defined 3 IntCounter metrics (CAS
bytes produced, bytes cleaned, bytes smudged)
- Remove metric increment calls from `file_upload_session.rs` and
`file_download_session.rs`
- Fix Windows CI flake: redb "Database already open" error in
`test_single_large`

These metrics were collected but never exposed via any HTTP endpoint or
text encoder, making them effectively dead code.

## Test plan
- [x] `cargo +nightly fmt` — clean
- [x] `cargo clippy --all-targets` — no new warnings
- [x] `cargo test -p xet-data` — 17/17 pass
- [x] `cargo test -p xet-data --features simulation --test
test_clean_smudge` — 14/14 pass (including `test_single_large`)
- [x] WASM builds (`hf_xet_wasm`, `hf_xet_thin_wasm`) — both succeed

<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> **Low Risk**
> Low risk: this removes unused Prometheus metrics plumbing and related
dependencies without changing the core upload/download logic. Main risk
is loss of any downstream reliance on these counters at build time
(e.g., feature flags or imports).
> 
> **Overview**
> Removes the `prometheus` dependency from the workspace and `xet_data`,
and updates lockfiles accordingly (including WASM-related lockfiles).
> 
> Deletes `xet_data`’s `prometheus_metrics` module and strips the
associated counter increments from `FileUploadSession` and
`FileDownloadSession`, leaving the data processing behavior unchanged
aside from no longer recording these metrics.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
c6c866b7ca. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->
This commit is contained in:
Assaf Vayner
2026-04-01 14:56:58 -07:00
committed by GitHub
parent 3d377bdffb
commit 20198a9081
10 changed files with 5 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
Cargo.lock generated
View File

@@ -3475,21 +3475,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prometheus"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if 1.0.4",
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"protobuf",
"thiserror 2.0.18",
]
[[package]]
name = "prost"
version = "0.14.3"
@@ -3522,26 +3507,6 @@ dependencies = [
"prost",
]
[[package]]
name = "protobuf"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "pyo3"
version = "0.26.0"
@@ -6157,7 +6122,6 @@ dependencies = [
"itertools 0.14.0",
"lazy_static",
"more-asserts",
"prometheus",
"pyo3",
"rand 0.9.2",
"regex",

1
Cargo.toml
View File

@@ -78,7 +78,6 @@ more-asserts = "0.3"
oneshot = "0.1"
pin-project = "1"
pyo3 = { version = "0.26", features = ["abi3-py37", "multiple-pymethods"] }
prometheus = "0.14"
rand = "0.9"
rand_chacha = "0.9"
regex = "1"

42
hf_xet/Cargo.lock generated
View File

@@ -2036,7 +2036,7 @@ dependencies = [
"nix",
"once_cell",
"prost 0.12.6",
"protobuf 2.28.0",
"protobuf",
"protobuf-codegen-pure",
"smallvec",
"spin",
@@ -2073,21 +2073,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prometheus"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if 1.0.4",
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"protobuf 3.7.2",
"thiserror 2.0.18",
]
[[package]]
name = "prost"
version = "0.12.6"
@@ -2149,24 +2134,13 @@ version = "2.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
[[package]]
name = "protobuf"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-codegen"
version = "2.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6"
dependencies = [
"protobuf 2.28.0",
"protobuf",
]
[[package]]
@@ -2175,19 +2149,10 @@ version = "2.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865"
dependencies = [
"protobuf 2.28.0",
"protobuf",
"protobuf-codegen",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "pyo3"
version = "0.26.0"
@@ -4308,7 +4273,6 @@ dependencies = [
"itertools 0.14.0",
"lazy_static",
"more-asserts",
"prometheus",
"rand 0.9.2",
"serde",
"serde_json",

42
wasm/hf_xet_thin_wasm/Cargo.lock generated
View File

@@ -582,12 +582,6 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
@@ -1482,41 +1476,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prometheus"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if 1.0.4",
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"protobuf",
"thiserror 2.0.18",
]
[[package]]
name = "protobuf"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "quinn"
version = "0.11.9"
@@ -3421,7 +3380,6 @@ dependencies = [
"itertools",
"lazy_static",
"more-asserts",
"prometheus",
"rand 0.9.2",
"serde",
"serde_json",

42
wasm/hf_xet_wasm/Cargo.lock generated
View File

@@ -626,12 +626,6 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
@@ -1594,41 +1588,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prometheus"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if 1.0.4",
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"protobuf",
"thiserror 2.0.18",
]
[[package]]
name = "protobuf"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "quinn"
version = "0.11.9"
@@ -3593,7 +3552,6 @@ dependencies = [
"itertools",
"lazy_static",
"more-asserts",
"prometheus",
"rand 0.9.2",
"serde",
"serde_json",

1
xet_data/Cargo.toml
View File

@@ -26,7 +26,6 @@ http = { workspace = true }
itertools = { workspace = true }
lazy_static = { workspace = true }
more-asserts = { workspace = true }
prometheus = { workspace = true }
rand = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }

4
xet_data/src/processing/file_download_session.rs
View File

@@ -13,9 +13,9 @@ use xet_client::cas_types::FileRange;
use xet_client::chunk_cache::ChunkCache;
use xet_runtime::core::{XetRuntime, xet_config};
use super::XetFileInfo;
use super::configurations::TranslatorConfig;
use super::remote_client_interface::create_remote_client;
use super::{XetFileInfo, prometheus_metrics};
use crate::error::{DataError, Result};
use crate::file_reconstruction::{DownloadStream, FileReconstructor, UnorderedDownloadStream};
use crate::progress_tracking::{GroupProgress, ItemProgressUpdater, UniqueID};
@@ -143,7 +143,6 @@ impl FileDownloadSession {
actual: n_bytes,
});
}
prometheus_metrics::FILTER_BYTES_SMUDGED.inc_by(n_bytes);
Ok(n_bytes)
}
@@ -189,7 +188,6 @@ impl FileDownloadSession {
});
}
prometheus_metrics::FILTER_BYTES_SMUDGED.inc_by(n_bytes);
Ok((id, n_bytes))
}

6
xet_data/src/processing/file_upload_session.rs
View File

@@ -17,11 +17,11 @@ use xet_core_structures::metadata_shard::file_structs::MDBFileInfo;
use xet_core_structures::xorb_object::SerializedXorbObject;
use xet_runtime::core::{XetRuntime, xet_config};
use super::XetFileInfo;
use super::configurations::TranslatorConfig;
use super::file_cleaner::{Sha256Policy, SingleFileCleaner};
use super::remote_client_interface::create_remote_client;
use super::shard_interface::SessionShardInterface;
use super::{XetFileInfo, prometheus_metrics};
use crate::deduplication::constants::{
MAX_XORB_BYTES, MAX_XORB_CHUNKS, XORB_CUT_THRESHOLD_BYTES, XORB_CUT_THRESHOLD_CHUNKS,
};
@@ -503,10 +503,6 @@ impl FileUploadSession {
metrics.shard_bytes_uploaded = self.shard_interface.upload_and_register_session_shards().await?;
metrics.total_bytes_uploaded = metrics.shard_bytes_uploaded + metrics.xorb_bytes_uploaded;
// Update the global counters
prometheus_metrics::FILTER_CAS_BYTES_PRODUCED.inc_by(metrics.new_bytes);
prometheus_metrics::FILTER_BYTES_CLEANED.inc_by(metrics.total_bytes);
#[cfg(debug_assertions)]
{
self.completion_tracker.assert_complete();

1
xet_data/src/processing/mod.rs
View File

@@ -5,7 +5,6 @@ mod file_cleaner;
mod file_download_session;
mod file_upload_session;
pub mod migration_tool;
mod prometheus_metrics;
mod remote_client_interface;
mod sha256;
mod shard_interface;

13
xet_data/src/processing/prometheus_metrics.rs
View File

@@ -1,13 +0,0 @@
use lazy_static::lazy_static;
use prometheus::{IntCounter, register_int_counter};
// Some of the common tracking things
lazy_static! {
pub static ref FILTER_CAS_BYTES_PRODUCED: IntCounter =
register_int_counter!("filter_process_xorb_bytes_produced", "Number of CAS bytes produced during cleaning")
.unwrap();
pub static ref FILTER_BYTES_CLEANED: IntCounter =
register_int_counter!("filter_process_bytes_cleaned", "Number of bytes cleaned").unwrap();
pub static ref FILTER_BYTES_SMUDGED: IntCounter =
register_int_counter!("filter_process_bytes_smudged", "Number of bytes smudged").unwrap();
}