Composable Hash Functionality (#745)

Currently, computing aggregate chunk hashes across independently
processed ranges requires recomputing over the full concatenated chunk
list. This PR introduces ChunkHashRange, a composable representation
that can hash contiguous partial ranges and merge them while preserving
equivalence with the existing xorb_hash / file_hash behavior. This
allows an intermediate representation of the hash ranges that can be
merged in arbitrary order to get the final hash. It also uses O(log(n))
storage and all operations are done in linear time. Serialization and
Deserialization are fully supported.

The main use case for this is in doing partial file edits. Previously,
to edit the middle of a large file, the client would have to know all
the hashes for the full file, even if only a few in the middle were
changed. With a large file, this can still be 100s of MB; the chunk
metadata size is roughly 1/1000 of the data size. With this change, we
can now transmit the unmodified parts of a file in O(log(n)) storage but
still be able to build the entire function hash; now a sequence of 10M
chunks takes the equivalent storage of ~500 chunks or so.

Along the way, we also added in an optimization for the merge step to
avoid an allocation, yielding a 2x speedup.

---------

Co-authored-by: Hoyt Koepke <hoytak@xethub.com>

Cargo.lock

@@ -6485,6 +6485,7 @@ dependencies = [
  "regex",
  "safe-transmute",
  "serde",
+ "serde_json",
  "serial_test",
  "static_assertions",
  "tempfile",

api_changes/update_260320_chunk_hash_range_composable_aggregation.md

@@ -0,0 +1,25 @@
+# API Update: Composable Merkle hash subtree aggregation (2026-03-20)
+
+## Overview
+
+This PR adds a composable subtree representation for merklehash aggregation so that large chunk streams can be hashed incrementally and merged without materializing all chunks in memory.
+
+## API additions
+
+- New public module: `xet_core_structures::merklehash::merkle_hash_subtree`
+- New public type: `xet_core_structures::merklehash::MerkleHashSubtree` (re-exported from `merklehash::mod`)
+- New public functions in `merkle_hash_subtree`:
+  - `find_stable_start`
+  - `find_stable_end`
+
+## Behavior and compatibility
+
+- Existing `xorb_hash`, `file_hash`, and `file_hash_with_salt` outputs are unchanged.
+- This is an additive API update.  No existing public API was removed or renamed.
+- The new `MerkleHashSubtree` path is intended for streaming/range composition use cases where O(log n) retained state is preferred over O(n) chunk retention.
+- `MerkleHashSubtree` implements `Serialize`/`Deserialize` with hashes rendered as hex strings for cross-language JSON compatibility.
+
+## Notes for downstream users
+
+- Existing callers do not need to change anything.
+- New callers can construct partial ranges with `MerkleHashSubtree::from_chunks(...)`, merge with `subtree.merge_into(&other)` or `MerkleHashSubtree::merge(&[...])`, and request `final_hash()` only when both file boundaries are known.

xet_core_structures/Cargo.toml

@@ -65,6 +65,7 @@ web-time = { workspace = true }
 bincode = { workspace = true }
 futures-util = { workspace = true }
 rand = { workspace = true }
+serde_json = { workspace = true }
 serial_test = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 

xet_core_structures/src/merklehash/aggregated_hashes.rs

@@ -1,10 +1,19 @@
-use std::cell::RefCell;
-use std::fmt::Write;
-
 use super::{MerkleHash, compute_internal_node_hash};
 
 pub const AGGREGATED_HASHES_MEAN_TREE_BRANCHING_FACTOR: u64 = 4;
 
+/// Minimum group size: groups always have at least 2 nodes.
+pub(super) const MIN_GROUP_SIZE: usize = 2;
+
+/// Maximum group size: groups have at most 2*BF+1 = 9 nodes.
+pub(super) const MAX_GROUP_SIZE: usize = 2 * AGGREGATED_HASHES_MEAN_TREE_BRANCHING_FACTOR as usize + 1;
+
+/// Returns true if this hash would trigger a natural cut (hash % BF == 0).
+#[inline]
+pub(super) fn is_natural_cut(h: MerkleHash) -> bool {
+    h % AGGREGATED_HASHES_MEAN_TREE_BRANCHING_FACTOR == 0
+}
+
 /// Find the next cut point in a sequence of hashes at which to break.
 ///
 ///   
@@ -25,7 +34,7 @@ pub const AGGREGATED_HASHES_MEAN_TREE_BRANCHING_FACTOR: u64 = 4;
 ///    children: This ensures that the graph always has at most 1/2 the number of parents as children. and we don't have
 ///    too wide branches.
 #[inline]
-fn next_merge_cut(hashes: &[(MerkleHash, u64)]) -> usize {
+pub(super) fn next_merge_cut(hashes: &[(MerkleHash, u64)]) -> usize {
     if hashes.len() <= 2 {
         return hashes.len();
     }
@@ -43,26 +52,87 @@ fn next_merge_cut(hashes: &[(MerkleHash, u64)]) -> usize {
     end
 }
 
-/// Merge the hashes together, including the size information and returning the new (hash, size) pair.
+const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef";
+
+/// Write a u64 as 16 zero-padded lowercase hex chars directly into a fixed buffer.
 #[inline]
-fn merged_hash_of_sequence(hash: &[(MerkleHash, u64)]) -> (MerkleHash, u64) {
-    // Use a threadlocal buffer to avoid the overhead of reallocations.
-    thread_local! {
-        static BUFFER: RefCell<String> =
-        RefCell::new(String::with_capacity(1024));
+fn write_hex_u64(buf: &mut [u8], pos: &mut usize, val: u64) {
+    let p = *pos;
+    buf[p] = HEX_DIGITS[((val >> 60) & 0xF) as usize];
+    buf[p + 1] = HEX_DIGITS[((val >> 56) & 0xF) as usize];
+    buf[p + 2] = HEX_DIGITS[((val >> 52) & 0xF) as usize];
+    buf[p + 3] = HEX_DIGITS[((val >> 48) & 0xF) as usize];
+    buf[p + 4] = HEX_DIGITS[((val >> 44) & 0xF) as usize];
+    buf[p + 5] = HEX_DIGITS[((val >> 40) & 0xF) as usize];
+    buf[p + 6] = HEX_DIGITS[((val >> 36) & 0xF) as usize];
+    buf[p + 7] = HEX_DIGITS[((val >> 32) & 0xF) as usize];
+    buf[p + 8] = HEX_DIGITS[((val >> 28) & 0xF) as usize];
+    buf[p + 9] = HEX_DIGITS[((val >> 24) & 0xF) as usize];
+    buf[p + 10] = HEX_DIGITS[((val >> 20) & 0xF) as usize];
+    buf[p + 11] = HEX_DIGITS[((val >> 16) & 0xF) as usize];
+    buf[p + 12] = HEX_DIGITS[((val >> 12) & 0xF) as usize];
+    buf[p + 13] = HEX_DIGITS[((val >> 8) & 0xF) as usize];
+    buf[p + 14] = HEX_DIGITS[((val >> 4) & 0xF) as usize];
+    buf[p + 15] = HEX_DIGITS[(val & 0xF) as usize];
+    *pos = p + 16;
+}
+
+/// Write a u64 as decimal digits into a fixed buffer.
+#[inline]
+fn write_decimal_u64(buf: &mut [u8], pos: &mut usize, val: u64) {
+    if val == 0 {
+        buf[*pos] = b'0';
+        *pos += 1;
+        return;
     }
+    // Write digits in reverse into a small stack buffer, then copy forward.
+    let mut digits = [0u8; 20]; // u64 max is 20 digits
+    let mut dpos = 20;
+    let mut v = val;
+    while v > 0 {
+        dpos -= 1;
+        digits[dpos] = b'0' + (v % 10) as u8;
+        v /= 10;
+    }
+    let len = 20 - dpos;
+    buf[*pos..*pos + len].copy_from_slice(&digits[dpos..]);
+    *pos += len;
+}
 
-    BUFFER.with(|buffer| {
-        let mut buf = buffer.borrow_mut();
-        buf.clear();
-        let mut total_len = 0;
+/// Max bytes per entry: 64 hex + 3 " : " + 20 decimal digits + 1 newline = 88.
+/// Max group size: 2 * BRANCHING_FACTOR + 1 = 9.
+const MAX_MERGE_BUF_SIZE: usize = (2 * AGGREGATED_HASHES_MEAN_TREE_BRANCHING_FACTOR as usize + 1) * 88;
 
-        for (h, s) in hash.iter() {
-            writeln!(buf, "{h:x} : {s}").unwrap();
-            total_len += *s;
-        }
-        (compute_internal_node_hash(buf.as_bytes()), total_len)
-    })
+/// Write one (hash, size) entry into the merge buffer.
+#[inline]
+fn write_hash_entry(buf: &mut [u8], pos: &mut usize, total_len: &mut u64, h: &MerkleHash, s: u64) {
+    write_hex_u64(buf, pos, h[0].to_le());
+    write_hex_u64(buf, pos, h[1].to_le());
+    write_hex_u64(buf, pos, h[2].to_le());
+    write_hex_u64(buf, pos, h[3].to_le());
+    buf[*pos] = b' ';
+    buf[*pos + 1] = b':';
+    buf[*pos + 2] = b' ';
+    *pos += 3;
+    write_decimal_u64(buf, pos, s);
+    buf[*pos] = b'\n';
+    *pos += 1;
+    *total_len += s;
+}
+
+/// Merge the hashes together, including the size information and returning the new (hash, size) pair.
+///
+/// Formats each entry as `"{hash_hex} : {size}\n"` and computes the internal node hash.
+/// Uses direct byte writing to a stack buffer to avoid allocation and TLS overhead.
+#[inline]
+pub(super) fn merged_hash_of_sequence(hash: &[(MerkleHash, u64)]) -> (MerkleHash, u64) {
+    let mut buf = [0u8; MAX_MERGE_BUF_SIZE];
+    let mut pos = 0usize;
+    let mut total_len = 0u64;
+    for &(ref h, s) in hash.iter() {
+        write_hash_entry(&mut buf, &mut pos, &mut total_len, h, s);
+    }
+    (compute_internal_node_hash(&buf[..pos]), total_len)
 }
 
 /// The base calculation for the aggregated node hash.
@@ -70,7 +140,7 @@ fn merged_hash_of_sequence(hash: &[(MerkleHash, u64)]) -> (MerkleHash, u64) {
 /// Iteratively collapse the list of hashes using the criteria in next_merge_cut
 /// until only one hash remains; this is the aggregated hash.
 #[inline]
-fn aggregated_node_hash(chunks: &[(MerkleHash, u64)]) -> MerkleHash {
+pub(super) fn aggregated_node_hash(chunks: &[(MerkleHash, u64)]) -> MerkleHash {
     if chunks.is_empty() {
         return MerkleHash::default();
     }

xet_core_structures/src/merklehash/mod.rs

@@ -49,9 +49,11 @@ pub use data_hash::*;
 pub type MerkleHash = DataHash;
 
 mod aggregated_hashes;
+pub mod merkle_hash_subtree;
 pub mod passthrough_hasher;
 pub mod passthrough_hashmap;
 
 pub use aggregated_hashes::{file_hash, file_hash_with_salt, xorb_hash};
+pub use merkle_hash_subtree::MerkleHashSubtree;
 pub use passthrough_hasher::{U64DirectHasher, U64HashExtractable};
 pub use passthrough_hashmap::PassThroughHashMap;