diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 4edd3979..f9efa6a8 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1304,6 +1304,9 @@ struct HeapPtrs { MaybeInitializedPtr slot_array; }; +// Returns the maximum size of the SOO slot. +constexpr size_t MaxSooSlotSize() { return sizeof(HeapPtrs); } + // Manages the backing array pointers or the SOO slot. When raw_hash_set::is_soo // is true, the SOO slot is stored in `soo_data`. Otherwise, we use `heap`. union HeapOrSoo { @@ -1330,7 +1333,7 @@ union HeapOrSoo { } HeapPtrs heap; - unsigned char soo_data[sizeof(HeapPtrs)]; + unsigned char soo_data[MaxSooSlotSize()]; }; // CommonFields hold the fields in raw_hash_set that do not depend @@ -2076,6 +2079,36 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void InitializeSmallControlBytesAfterSoo( // new_ctrl after 3rd store = E0EEEEESE0EEEEEEEEEEEEE } +// Returns the optimal size for memcpy when transferring SOO slot. +// If any of transfer_uses_memcpy or soo_enabled is false, returns 0. +// Otherwise, returns the optimal size for memcpy SOO slot transfer +// to SooSlotIndex(). +// At the destination we are allowed to copy upto twice more bytes, +// because there is at least one more slot after SooSlotIndex(). +// The result must not exceed MaxSooSlotSize(). +// Some of the cases are merged to minimize the number of function +// instantiations. +constexpr size_t OptimalMemcpySizeForSooSlotTransfer(bool transfer_uses_memcpy, + bool soo_enabled, + size_t slot_size) { + if (!transfer_uses_memcpy || !soo_enabled) { + return 0; + } + if (slot_size == 1) { + return 1; + } + if (slot_size <= 3) { + return 4; + } + // We are merging 4 and 8 into one case because we expect them to be the + // hottest cases. Copying 8 bytes is as fast on common architectures. + if (slot_size <= 8) { + return 8; + } + static_assert(MaxSooSlotSize() <= 16, "unexpectedly large SOO slot size"); + return 16; +} + // Helper class to perform resize of the hash set. // // It contains special optimizations for small group resizes. @@ -2144,27 +2177,31 @@ class HashSetResizeHelper { // infoz.RecordRehash is called if old_capacity == 0. // // Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation. - template + template ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, Alloc alloc, size_t soo_slot_hash, size_t key_size, size_t value_size, const PolicyFunctions& policy) { ABSL_SWISSTABLE_ASSERT(c.capacity()); + const size_t slot_size = policy.slot_size; HashtablezInfoHandle infoz = c.infoz(); const bool should_sample = ShouldSampleHashtablezInfoOnResize(force_infoz_, old_capacity_, c); if (ABSL_PREDICT_FALSE(should_sample)) { - infoz = ForcedTrySample(SizeOfSlot, key_size, value_size, + infoz = ForcedTrySample(slot_size, key_size, value_size, SooEnabled ? SooCapacity() : 0); } const bool has_infoz = infoz.IsSampled(); RawHashSetLayout layout(c.capacity(), AlignOfSlot, has_infoz); char* mem = static_cast(Allocate( - &alloc, layout.alloc_size(SizeOfSlot))); + &alloc, layout.alloc_size(slot_size))); const GenerationType old_generation = c.generation(); c.set_generation_ptr( reinterpret_cast(mem + layout.generation_offset())); @@ -2180,18 +2217,31 @@ class HashSetResizeHelper { if (!had_soo_slot_) { c.set_control(new_ctrl); c.set_slots(new_slots); - ResetCtrl(c, SizeOfSlot); + ResetCtrl(c, slot_size); } else if (ABSL_PREDICT_TRUE(layout.capacity() <= MaxSmallAfterSooCapacity())) { if (TransferUsesMemcpy) { InsertOldSooSlotAndInitializeControlBytesSmall( - c, soo_slot_hash, new_ctrl, new_slots, SizeOfSlot, - [](void* target_slot, void* source_slot) { - std::memcpy(target_slot, source_slot, SizeOfSlot); + c, soo_slot_hash, new_ctrl, new_slots, slot_size, + [&](void* target_slot, void* source_slot) { + // Target slot is placed at index 1, but capacity is at + // minimum 3. So we are allowed to copy at least twice as much + // memory. + static_assert(SooSlotIndex() == 1, ""); + static_assert(SooSlotMemcpySize <= MaxSooSlotSize(), ""); + ABSL_SWISSTABLE_ASSERT(SooSlotMemcpySize != 0 && + SooSlotMemcpySize <= 2 * slot_size); + ABSL_SWISSTABLE_ASSERT(SooSlotMemcpySize >= slot_size); + void* next_slot = SlotAddress(target_slot, 1, slot_size); + SanitizerUnpoisonMemoryRegion(next_slot, + SooSlotMemcpySize - slot_size); + std::memcpy(target_slot, source_slot, SooSlotMemcpySize); + SanitizerPoisonMemoryRegion(next_slot, + SooSlotMemcpySize - slot_size); }); } else { InsertOldSooSlotAndInitializeControlBytesSmall( - c, soo_slot_hash, new_ctrl, new_slots, SizeOfSlot, + c, soo_slot_hash, new_ctrl, new_slots, slot_size, [&](void* target_slot, void* source_slot) { policy.transfer(&c, target_slot, source_slot); }); @@ -2209,14 +2259,14 @@ class HashSetResizeHelper { // SooEnabled implies that old_capacity_ != 0. if ((SooEnabled || old_capacity_ != 0) && grow_single_group) { if (TransferUsesMemcpy) { - GrowSizeIntoSingleGroupTransferable(c, SizeOfSlot); - DeallocateOld(alloc, SizeOfSlot); + GrowSizeIntoSingleGroupTransferable(c, slot_size); + DeallocateOld(alloc, slot_size); } else { GrowIntoSingleGroupShuffleControlBytes(c.control(), layout.capacity()); } } else { - ResetCtrl(c, SizeOfSlot); + ResetCtrl(c, slot_size); } } @@ -3843,7 +3893,10 @@ class raw_hash_set { // depending on the values of `transfer_uses_memcpy` and capacities. // Refer to the comment in `InitializeSlots` for more details. const bool grow_single_group = - resize_helper.InitializeSlots( common, CharAlloc(set->alloc_ref()), soo_slot_hash, diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index b7e1521a..36749ff6 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -576,6 +577,48 @@ template using SizedValuePolicy = ValuePolicy, /*kTransferable=*/true, kSoo>; +// Value is aligned as type T and contains N copies of it. +template +class AlignedValue { + public: + AlignedValue(int64_t v) { // NOLINT + for (int i = 0; i < N; ++i) { + vals_[i] = v; + if (sizeof(T) < sizeof(int64_t)) { + v >>= (8 * sizeof(T)); + } else { + v = 0; + } + } + } + AlignedValue() : AlignedValue(0) {} + AlignedValue(const AlignedValue&) = default; + AlignedValue& operator=(const AlignedValue&) = default; + + int64_t operator*() const { + if (sizeof(T) == sizeof(int64_t)) { + return vals_[0]; + } + int64_t result = 0; + for (int i = N - 1; i >= 0; --i) { + result <<= (8 * sizeof(T)); + result += vals_[i]; + } + return result; + } + explicit operator int() const { return **this; } + explicit operator int64_t() const { return **this; } + + template + friend H AbslHashValue(H h, AlignedValue sv) { + return H::combine(std::move(h), *sv); + } + bool operator==(const AlignedValue& rhs) const { return **this == *rhs; } + + private: + T vals_[N]; +}; + class StringPolicy { template class SmallTableResizeTest : public testing::Test {}; -using SmallTableTypes = - ::testing::Types; +using SmallTableTypes = ::testing::Types< + IntTable, TransferableIntTable, SooIntTable, + // int8 + ValueTable, + ValueTable, + // int16 + ValueTable, + ValueTable, + // int128 + ValueTable, /*kTransferable=*/true, /*kSoo=*/true>, + ValueTable, /*kTransferable=*/false, /*kSoo=*/true>, + // int192 + ValueTable, /*kTransferable=*/true, /*kSoo=*/true>, + ValueTable, /*kTransferable=*/false, /*kSoo=*/true>, + // Special tables. + MinimumAlignmentUint8Table, + CustomAllocIntTable, + BadTable, + // alignment 1, size 2. + ValueTable, /*kTransferable=*/true, /*kSoo=*/true>, + ValueTable, /*kTransferable=*/false, + /*kSoo=*/true>, + // alignment 1, size 7. + ValueTable, /*kTransferable=*/true, /*kSoo=*/true>, + ValueTable, /*kTransferable=*/false, + /*kSoo=*/true>, + // alignment 2, size 6. + ValueTable, /*kTransferable=*/true, + /*kSoo=*/true>, + ValueTable, /*kTransferable=*/false, + /*kSoo=*/true>, + // alignment 2, size 10. + ValueTable, /*kTransferable=*/true, + /*kSoo=*/true>, + ValueTable, /*kTransferable=*/false, + /*kSoo=*/true>>; TYPED_TEST_SUITE(SmallTableResizeTest, SmallTableTypes); TYPED_TEST(SmallTableResizeTest, InsertIntoSmallTable) {