sync bela/pugixml

Author: fcharlie

vendor/bela.lock

@@ -1 +1 @@
-https://github.com/fcharlie/bela/tree/96686cfb2173c52baad010be6832e0059824f7f8
+https://github.com/fcharlie/bela/tree/5565e770ec209510945a3c46a6bdfd33e231392c

vendor/bela/include/bela/__phmap/VERSION

@@ -1,2 +1,2 @@
 https://github.com/greg7mdp/parallel-hashmap.git
-60acfa4690303eeefde355175eaaad57c65a113d
+2d062fc53967b978cab766c76829e2fcd8817ccc

vendor/bela/include/bela/__phmap/phmap.h

@@ -329,11 +329,17 @@ static_assert(kDeleted == -2,
 // A single block of empty control bytes for tables without any slots allocated.
 // This enables removing a branch in the hot path of find().
 // --------------------------------------------------------------------------
+template <class std_alloc_t>
 inline ctrl_t* EmptyGroup() {
-  alignas(16) static constexpr ctrl_t empty_group[] = {
-      kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
-      kEmpty,    kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty};
-  return const_cast<ctrl_t*>(empty_group);
+  PHMAP_IF_CONSTEXPR (std_alloc_t::value) {
+      alignas(16) static constexpr ctrl_t empty_group[] = {
+          kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
+          kEmpty,    kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty};
+
+      return const_cast<ctrl_t*>(empty_group);
+  } else {
+       return nullptr;
+  }
 }
 
 // --------------------------------------------------------------------------
@@ -869,6 +875,8 @@ class raw_hash_set
     template <class K>
     using key_arg = typename KeyArgImpl::template type<K, key_type>;
 
+    using std_alloc_t = std::is_same<typename std::decay<Alloc>::type, phmap::priv::Allocator<value_type>>;
+
 private:
     // Give an early error when key_type is not hashable/eq.
     auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
@@ -918,12 +926,7 @@ class raw_hash_set
     using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>;
 
 public:
-    static_assert(std::is_same<pointer, value_type*>::value,
-                  "Allocators with custom pointer types are not supported");
-    static_assert(std::is_same<const_pointer, const value_type*>::value,
-                  "Allocators with custom pointer types are not supported");
-
-    class iterator 
+    class iterator
     {
         friend class raw_hash_set;
 
@@ -989,6 +992,11 @@ class raw_hash_set
         iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {}
 
         void skip_empty_or_deleted() {
+            PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
+                // ctrl_ could be nullptr
+                if (!ctrl_)
+                    return;
+            }
             while (IsEmptyOrDeleted(*ctrl_)) {
                 // ctrl is not necessarily aligned to Group::kWidth. It is also likely
                 // to read past the space for ctrl bytes and into slots. This is ok
@@ -1057,7 +1065,7 @@ class raw_hash_set
     explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(),
                           const key_equal& eq = key_equal(),
                           const allocator_type& alloc = allocator_type())
-        : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) {
+        : ctrl_(EmptyGroup<std_alloc_t>()), settings_(0, hashfn, eq, alloc) {
         if (bucket_cnt) {
             size_t new_capacity = NormalizeCapacity(bucket_cnt);
             reset_growth_left(new_capacity);
@@ -1180,7 +1188,7 @@ class raw_hash_set
         std::is_nothrow_copy_constructible<hasher>::value&&
         std::is_nothrow_copy_constructible<key_equal>::value&&
         std::is_nothrow_copy_constructible<allocator_type>::value)
-        : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())),
+        : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup<std_alloc_t>())),
         slots_(phmap::exchange(that.slots_, nullptr)),
         size_(phmap::exchange(that.size_, 0)),
         capacity_(phmap::exchange(that.capacity_, 0)),
@@ -1194,7 +1202,7 @@ class raw_hash_set
     }
 
     raw_hash_set(raw_hash_set&& that, const allocator_type& a)
-        : ctrl_(EmptyGroup()),
+        : ctrl_(EmptyGroup<std_alloc_t>()),
           slots_(nullptr),
           size_(0),
           capacity_(0),
@@ -1615,6 +1623,7 @@ class raw_hash_set
     // This overload is necessary because otherwise erase<K>(const K&) would be
     // a better match if non-const iterator is passed as an argument.
     iterator erase(iterator it) {
+        assert(it != end());
         auto res = it;
         ++res;
         _erase(it);
@@ -1738,7 +1747,8 @@ class raw_hash_set
 
     template <class K = key_type>
     void prefetch(const key_arg<K>& key) const {
-        prefetch_hash(this->hash(key));
+        PHMAP_IF_CONSTEXPR (std_alloc_t::value)
+            prefetch_hash(this->hash(key));
     }
 
     // The API of find() has two extensions.
@@ -1848,6 +1858,11 @@ class raw_hash_set
 
     template <class K = key_type>
     bool find_impl(const key_arg<K>& key, size_t hashval, size_t& offset) {
+        PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
+            // ctrl_ could be nullptr
+            if (!ctrl_)
+                return false;
+        }
         auto seq = probe(hashval);
         while (true) {
             Group g{ ctrl_ + seq.offset() };
@@ -1877,7 +1892,11 @@ class raw_hash_set
     {
         template <class K, class... Args>
         size_t operator()(const K& key, Args&&...) const {
+#if PHMAP_DISABLE_MIX
+            return h(key);
+#else
             return phmap_mix<sizeof(size_t)>()(h(key));
+#endif
         }
         const hasher& h;
     };
@@ -2025,7 +2044,7 @@ class raw_hash_set
         // Unpoison before returning the memory to the allocator.
         SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_);
         Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize());
-        ctrl_ = EmptyGroup();
+        ctrl_ = EmptyGroup<std_alloc_t>();
         slots_ = nullptr;
         size_ = 0;
         capacity_ = 0;
@@ -2135,6 +2154,11 @@ class raw_hash_set
     }
 
     bool has_element(const value_type& elem, size_t hashval) const {
+        PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
+            // ctrl_ could be nullptr
+            if (!ctrl_)
+                return false;
+        }
         auto seq = probe(hashval);
         while (true) {
             Group g{ctrl_ + seq.offset()};
@@ -2197,6 +2221,11 @@ class raw_hash_set
 protected:
     template <class K>
     size_t _find_key(const K& key, size_t hashval) {
+        PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
+            // ctrl_ could be nullptr
+            if (!ctrl_)
+                return (size_t)-1;
+        }
         auto seq = probe(hashval);
         while (true) {
             Group g{ctrl_ + seq.offset()};
@@ -2221,7 +2250,12 @@ class raw_hash_set
     }
 
     size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE {
-        auto target = find_first_non_full(hashval);
+        PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
+            // ctrl_ could be nullptr
+            if (!ctrl_)
+                rehash_and_grow_if_necessary();
+        }
+        FindInfo target = find_first_non_full(hashval);
         if (PHMAP_PREDICT_FALSE(growth_left() == 0 &&
                                !IsDeleted(ctrl_[target.offset]))) {
             rehash_and_grow_if_necessary();
@@ -2335,10 +2369,10 @@ class raw_hash_set
     // TODO(alkis): Investigate removing some of these fields:
     // - ctrl/slots can be derived from each other
     // - size can be moved into the slot array
-    ctrl_t* ctrl_ = EmptyGroup();    // [(capacity + 1) * ctrl_t]
-    slot_type* slots_ = nullptr;     // [capacity * slot_type]
-    size_t size_ = 0;                // number of full slots
-    size_t capacity_ = 0;            // total number of slots
+    ctrl_t* ctrl_ = EmptyGroup<std_alloc_t>();    // [(capacity + 1) * ctrl_t]
+    slot_type* slots_ = nullptr;                  // [capacity * slot_type]
+    size_t size_ = 0;                             // number of full slots
+    size_t capacity_ = 0;                         // total number of slots
     HashtablezInfoHandle infoz_;
     std::tuple<size_t /* growth_left */, hasher, key_equal, allocator_type>
         settings_{0, hasher{}, key_equal{}, allocator_type{}};
@@ -2582,7 +2616,6 @@ class parallel_hash_set
     using UniqueLock    = typename Lockable::UniqueLock;
     using SharedLock    = typename Lockable::SharedLock;
     using ReadWriteLock = typename Lockable::ReadWriteLock;
-    
 
     // --------------------------------------------------------------------
     struct Inner : public Lockable
@@ -3144,14 +3177,9 @@ class parallel_hash_set
     {
         Inner& inner   = sets_[subidx(hashval)];
         auto&  set     = inner.set_;
-        ReadWriteLock m(inner);
+        UniqueLock m(inner);
 
         size_t offset = set._find_key(key, hashval);
-        if (offset == (size_t)-1 && m.switch_to_unique()) {
-            // we did an unlock/lock, and another thread could have inserted the same key, so we need to
-            // do a find() again.
-            offset = set._find_key(key, hashval);
-        }
         if (offset == (size_t)-1) {
             offset = set.prepare_insert(hashval);
             set.emplace_at(offset, std::forward<Args>(args)...);
@@ -3234,13 +3262,8 @@ class parallel_hash_set
     iterator lazy_emplace_with_hash(const key_arg<K>& key, size_t hashval, F&& f) {
         Inner& inner = sets_[subidx(hashval)];
         auto&  set   = inner.set_;
-        ReadWriteLock m(inner);
+        UniqueLock m(inner);
         size_t offset = set._find_key(key, hashval);
-        if (offset == (size_t)-1 && m.switch_to_unique()) {
-            // we did an unlock/lock, and another thread could have inserted the same key, so we need to
-            // do a find() again.
-            offset = set._find_key(key, hashval);
-        }
         if (offset == (size_t)-1) {
             offset = set.prepare_insert(hashval);
             set.lazy_emplace_at(offset, std::forward<F>(f));
@@ -3355,7 +3378,7 @@ class parallel_hash_set
     template <class K = key_type, class FExists, class FEmplace>
     bool lazy_emplace_l(const key_arg<K>& key, FExists&& fExists, FEmplace&& fEmplace) {
         size_t hashval = this->hash(key);
-        ReadWriteLock m;
+        UniqueLock m;
         auto res = this->find_or_prepare_insert_with_hash(hashval, key, m);
         Inner* inner = std::get<0>(res);
         if (std::get<2>(res)) {
@@ -3730,7 +3753,11 @@ class parallel_hash_set
     {
         template <class K, class... Args>
         size_t operator()(const K& key, Args&&...) const {
+#if PHMAP_DISABLE_MIX
+            return h(key);
+#else
             return phmap_mix<sizeof(size_t)>()(h(key));
+#endif
         }
         const hasher& h;
     };
@@ -3809,16 +3836,11 @@ class parallel_hash_set
 
     template <class K>
     std::tuple<Inner*, size_t, bool> 
-    find_or_prepare_insert_with_hash(size_t hashval, const K& key, ReadWriteLock &mutexlock) {
+    find_or_prepare_insert_with_hash(size_t hashval, const K& key, UniqueLock &mutexlock) {
         Inner& inner = sets_[subidx(hashval)];
         auto&  set   = inner.set_;
-        mutexlock    = std::move(ReadWriteLock(inner));
+        mutexlock    = std::move(UniqueLock(inner));
         size_t offset = set._find_key(key, hashval);
-        if (offset == (size_t)-1 && mutexlock.switch_to_unique()) {
-            // we did an unlock/lock, and another thread could have inserted the same key, so we need to
-            // do a find() again.
-            offset = set._find_key(key, hashval);
-        }
         if (offset == (size_t)-1) {
             offset = set.prepare_insert(hashval);
             return std::make_tuple(&inner, offset, true);
@@ -3828,7 +3850,7 @@ class parallel_hash_set
 
     template <class K>
     std::tuple<Inner*, size_t, bool> 
-    find_or_prepare_insert(const K& key, ReadWriteLock &mutexlock) {
+    find_or_prepare_insert(const K& key, UniqueLock &mutexlock) {
         return find_or_prepare_insert_with_hash<K>(this->hash(key), key, mutexlock);
     }
 
@@ -4050,7 +4072,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash
     template <class K = key_type, class F, class... Args>
     bool try_emplace_l(K&& k, F&& f, Args&&... args) {
         size_t hashval = this->hash(k);
-        ReadWriteLock m;
+        UniqueLock m;
         auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
         typename Base::Inner *inner = std::get<0>(res);
         if (std::get<2>(res)) {
@@ -4071,7 +4093,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash
     template <class K = key_type, class... Args>
     std::pair<typename parallel_hash_map::parallel_hash_set::pointer, bool> try_emplace_p(K&& k, Args&&... args) {
         size_t hashval = this->hash(k);
-        ReadWriteLock m;
+        UniqueLock m;
         auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
         typename Base::Inner *inner = std::get<0>(res);
         if (std::get<2>(res)) {
@@ -4101,7 +4123,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash
     template <class K, class V>
     std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) {
         size_t hashval = this->hash(k);
-        ReadWriteLock m;
+        UniqueLock m;
         auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
         typename Base::Inner *inner = std::get<0>(res);
         if (std::get<2>(res)) {
@@ -4121,7 +4143,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash
 
     template <class K = key_type, class... Args>
     std::pair<iterator, bool> try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) {
-        ReadWriteLock m;
+        UniqueLock m;
         auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
         typename Base::Inner *inner = std::get<0>(res);
         if (std::get<2>(res)) {
@@ -4576,6 +4598,8 @@ struct HashtableDebugAccess<Set, typename std::enable_if<has_member_type_raw_has
 
     static size_t GetNumProbes(const Set& set,
                                const typename Set::key_type& key) {
+        if (!set.ctrl_)
+            return 0;
         size_t num_probes = 0;
         size_t hashval = set.hash(key); 
         auto seq = set.probe(hashval);

vendor/bela/include/bela/__phmap/phmap_base.h

@@ -2774,7 +2774,9 @@ class node_handle_base
     node_handle_base& operator=(node_handle_base&& other) noexcept {
         destroy();
         if (!other.empty()) {
-            alloc_ = other.alloc_;
+            if (other.alloc_) {
+               alloc_.emplace(other.alloc_.value());
+            }
             PolicyTraits::transfer(alloc(), slot(), other.slot());
             other.reset();
         }
@@ -4189,7 +4191,7 @@ void* Allocate(Alloc* alloc, size_t n) {
   using A = typename phmap::allocator_traits<Alloc>::template rebind_alloc<M>;
   using AT = typename phmap::allocator_traits<Alloc>::template rebind_traits<M>;
   A mem_alloc(*alloc);
-  void* p = AT::allocate(mem_alloc, (n + sizeof(M) - 1) / sizeof(M));
+  void* p = &*AT::allocate(mem_alloc, (n + sizeof(M) - 1) / sizeof(M)); // `&*` to support custom pointers such as boost offset_ptr.
   assert(reinterpret_cast<uintptr_t>(p) % Alignment == 0 &&
          "allocator does not respect alignment");
   return p;

vendor/bela/include/bela/__phmap/phmap_config.h

@@ -36,7 +36,7 @@
 
 #define PHMAP_VERSION_MAJOR 1
 #define PHMAP_VERSION_MINOR 4
-#define PHMAP_VERSION_PATCH 0
+#define PHMAP_VERSION_PATCH 1
 
 // Included for the __GLIBC__ macro (or similar macros on other systems).
 #include <limits.h>

vendor/bela/include/bela/__phmap/phmap_dump.h

@@ -94,6 +94,8 @@ bool raw_hash_set<Policy, Hash, Eq, Alloc>::phmap_load(InputArchive& ar) {
     if (version >= s_version_base) {
         // growth_left should be restored after calling initialize_slots() which resets it.
         ar.loadBinary(&growth_left(), sizeof(size_t));
+    } else {
+       drop_deletes_without_resize();
     }
     return true;
 }

vendor/bela/include/bela/__strings/int128_have_intrinsic.inc

@@ -173,27 +173,29 @@ inline int128::operator float() const {
   // complement overwhelms the precision of the mantissa.
   //
   // Also check to make sure we don't negate Int128Min()
+  constexpr float pow_2_64 = 18446744073709551616.0f;
   return v_ < 0 && *this != Int128Min()
              ? -static_cast<float>(-*this)
              : static_cast<float>(Int128Low64(*this)) +
-                   std::ldexp(static_cast<float>(Int128High64(*this)), 64);
+                    static_cast<float>(Int128High64(*this)) * pow_2_64;
 }
 
 inline int128::operator double() const {
   // See comment in int128::operator float() above.
+   constexpr double pow_2_64 = 18446744073709551616.0;
   return v_ < 0 && *this != Int128Min()
              ? -static_cast<double>(-*this)
              : static_cast<double>(Int128Low64(*this)) +
-                   std::ldexp(static_cast<double>(Int128High64(*this)), 64);
+                   static_cast<double>(Int128High64(*this)) * pow_2_64;
 }
 
 inline int128::operator long double() const {
   // See comment in int128::operator float() above.
+  constexpr long double pow_2_64 = 18446744073709551616.0L;
   return v_ < 0 && *this != Int128Min()
              ? -static_cast<long double>(-*this)
              : static_cast<long double>(Int128Low64(*this)) +
-                   std::ldexp(static_cast<long double>(Int128High64(*this)),
-                              64);
+                    static_cast<long double>(Int128High64(*this)) * pow_2_64;
 }
 #endif  // Clang on PowerPC